File: runserver.py

from explainshell import config
from explainshell.web import app

import logging.config
logging.config.dictConfig(config.LOGGING_DICT)

if __name__ == '__main__':
    if config.HOST_IP:
        app.run(debug=config.DEBUG, host=config.HOST_IP)
    else:
        app.run(debug=config.DEBUG)



File: tools/shellbuiltins.py

# -*- coding: utf-8 -*-

'''manually define and add shell builtins into the store

unfortunately the bash section for builtins isn't written in a way
explainshell can understannd, so we have to resort to manually
writing these down and adding them.'''

import textwrap
from explainshell import store, config

sp = store.paragraph
so = store.option
sm = store.manpage

BUILTINS = {}

def _add(names, synopsis, options):
    name = names[0]
    # hack: fake a source man page (this breaks the outgoing links from
    # explainshell, oh well)
    names.append('bash-%s' % name)
    BUILTINS[name] = sm('bash-%s.1.gz' % name, name, synopsis, options, [(name, 20) for name in names])

_add([':'], 'the command does nothing', [so(sp(0, '''No  effect;  the command does nothing beyond expanding arguments and performing any specified redirections.  A zero
exit code  is returned.''', '', True), [], [], False, True, False)])

source = textwrap.dedent('''       <b>.</b>  <u>filename</u> [<u>arguments</u>]
       <b>source</b> <u>filename</u> [<u>arguments</u>]
              Read  and  execute  commands  from <u>filename</u> in the current shell environment and return the  exit  status  of
              the  last  command executed  from  <u>filename</u>.  If <u>filename</u> does not contain a slash, filenames in <b>PATH</b> are used
              to  find  the  directory  containing <u>filename</u>.  The file searched for in <b>PATH</b> need not be executable.  When
              <b>bash</b> is not  in  <u>posix</u>  <u>mode</u>,  the  current  directory  is searched  if no file is found in <b>PATH</b>.  If the
              <b>sourcepath</b> option to the <b>shopt</b> builtin command is turned  off,  the  <b>PATH</b>  is  not searched.   If  any
              <u>arguments</u>  are  supplied,  they  become the positional parameters when <u>filename</u> is executed.  Otherwise
              the positional  parameters  are unchanged.  The return status is the status of the last command exited within
              the  script  (0  if  no commands  are  executed), and false if <u>filename</u> is not found or cannot be read.''')
_add(['source', '.'], 'read and execute commands in the current shell', [so(sp(0, source, '', True), [], [], False, True, False)])

_add(['break'], 'exit from within a for, while, until, or select loop',
     [so(sp(0, '''If <u>n</u> is specified, break <u>n</u> levels.  <u>n</u> must be ≥ 1.  If <u>n</u> is greater than the  number  of enclosing loops, all enclosing loops are exited.  The return value is 0 unless <u>n</u> is not greater than or  equal  to 1.''', '', True), [], [], False, True, False)])

_add(['history'], 'display the  command  history  list  with  line numbers',
    [so(sp(0, '''<b>history</b> <b>[</b><u>n</u><b>]</b>
<b>history</b> <b>-c</b>
<b>history</b> <b>-d</b> <u>offset</u>
<b>history</b> <b>-anrw</b> [<u>filename</u>]
<b>history</b> <b>-p</b> <u>arg</u> [<u>arg</u> <u>...</u>]
<b>history</b> <b>-s</b> <u>arg</u> [<u>arg</u> <u>...</u>]

With no options, display the  command  history  list  with  line numbers.  Lines listed with a <b>*</b> have been modified.
An argument of <u>n</u> lists only  the  last  <u>n</u>  lines.   If  the  shell  variable <b>HISTTIMEFORMAT</b>  is  set
and  not  null,  it is used as a format string for <u>strftime</u>(3) to display the time stamp associated with each
displayed  history entry.  No intervening blank is printed between the formatted time  stamp  and  the  history
line.   If <u>filename</u>  is  supplied,  it  is  used as the name of the history file; if not, the  value  of
<b>HISTFILE</b>  is  used.''', '', True), [], [], False, True, False),
     so(sp(1, '<b>-c</b>     Clear the history list by deleting all the entries.', '', True), ['-c'], [], False, False, False),
     so(sp(2, textwrap.dedent('''              <b>-d</b> <u>offset</u>
                     Delete the history entry at position <u>offset</u>.'''), '', True), ['-d'], [], 'offset', False, False),
     so(sp(3, textwrap.dedent('''              <b>-a</b>     Append  the  ``new'' history lines (history lines entered since the beginning of the current <b>bash</b> session)
                     to  the history file.'''), '', True), ['-a'], [], False, False, False),
     so(sp(4, textwrap.dedent('''              <b>-n</b>     Read  the history lines not already read from the history file into the current  history  list.   These  are
                     lines appended  to  the history file since the beginning of the current <b>bash</b> session.'''), '', True), ['-n'], [], False, False, False),
     so(sp(5, textwrap.dedent('''              <b>-r</b>     Read the contents of the history file and append them  to the current history list.'''), '', True), ['-r'], [], False, False, False),
     so(sp(6, textwrap.dedent('''              <b>-w</b>     Write  the  current  history  list  to  the history file, overwriting the history file's contents.'''), '', True), ['-w'], [], 'filename', False, False),
     so(sp(7, textwrap.dedent('''              <b>-p</b>     Perform history substitution on the  following  <u>args</u>  and display  the  result  on  the  standard output.
                     Does not store the results in the history list.  Each <u>arg</u> must  be quoted to disable normal history expansion.'''), '', True), ['-p'], [], 'arg', True, False),
     so(sp(8, textwrap.dedent('''              <b>-s</b>     Store  the  <u>args</u>  in  the history list as a single entry.  The last command in the history list  is
                     removed  before the <u>args</u> are added.'''), '', True), ['-s'], [], 'arg', False, False)])

if __name__ == '__main__':
    import logging.config
    logging.config.dictConfig(config.LOGGING_DICT)

    s = store.store('explainshell', config.MONGO_URI)
    for m in BUILTINS.itervalues():
        s.addmanpage(m)



File: explainshell/options.py

import re, collections, logging

from explainshell import store

tokenstate = collections.namedtuple('tokenstate', 'startpos endpos token')

logger = logging.getLogger(__name__)

def extract(manpage):
    '''extract options from all paragraphs that have been classified as containing
    options'''
    for i, p in enumerate(manpage.paragraphs):
        if p.is_option:
            s, l = extract_option(p.cleantext())
            if s or l:
                expectsarg = any(x.expectsarg for x in s + l)
                s = [x.flag for x in s]
                l = [x.flag for x in l]
                manpage.paragraphs[i] = store.option(p, s, l, expectsarg)
            else:
                logger.error("no options could be extracted from paragraph %r", p)

opt_regex = re.compile(r'''
    (?P<opt>--?(?:\?|\#|(?:\w+-)*\w+))  # option starts with - or -- and can have - in the middle but not at the end, also allow '-?'
    (?:
     (?:\s?(=)?\s?)           # -a=
     (?P<argoptional>[<\[])?  # -a=< or -a=[
     (?:\s?(=)?\s?)           # or maybe -a<=
     (?P<arg>
      (?(argoptional)         # if we think we have an arg (we saw [ or <)
       [^\]>]+                # either read everything until the closing ] or >
       |
       (?(2)
        [-a-zA-Z]+             # or if we didn't see [ or < but just saw =, read all letters, e.g. -a=abc
        |
        [A-Z]+                # but if we didn't have =, only allow uppercase letters, e.g. -a FOO
       )
      )
     )
     (?(argoptional)(?P<argoptionalc>[\]>])) # read closing ] or > if we have an arg
    )?                        # the whole arg thing is optional
    (?P<ending>,\s*|\s+|\Z|/|\|)''', re.X)  # read any trailing whitespace or the end of the string

opt2_regex = re.compile(r'''
        (?P<opt>\w+)    # an option that doesn't start with any of the usual characters, e.g. options from 'dd' like bs=BYTES
        (?:
         (?:\s*=\s*)    # an optional arg, e.g. bs=BYTES
         (?P<arg>\w+)
        )
        (?:,\s*|\s+|\Z)''', re.X) # end with , or whitespace or the end of the string

def _flag(s, pos=0):
    '''
    >>> _flag('a=b').groupdict()
    {'opt': 'a', 'arg': 'b'}
    >>> bool(_flag('---c-d'))
    False
    >>> bool(_flag('foobar'))
    False
    '''
    m = opt2_regex.match(s, pos)
    return m

def _option(s, pos=0):
    '''
    >>> bool(_option('-'))
    False
    >>> bool(_option('--'))
    False
    >>> bool(_option('---'))
    False
    >>> bool(_option('-a-'))
    False
    >>> bool(_option('--a-'))
    False
    >>> bool(_option('--a-b-'))
    False
    >>> sorted(_option('-a').groupdict().iteritems())
    [('arg', None), ('argoptional', None), ('argoptionalc', None), ('ending', ''), ('opt', '-a')]
    >>> sorted(_option('--a').groupdict().iteritems())
    [('arg', None), ('argoptional', None), ('argoptionalc', None), ('ending', ''), ('opt', '--a')]
    >>> sorted(_option('-a<b>').groupdict().iteritems())
    [('arg', 'b'), ('argoptional', '<'), ('argoptionalc', '>'), ('ending', ''), ('opt', '-a')]
    >>> sorted(_option('-a=[foo]').groupdict().iteritems())
    [('arg', 'foo'), ('argoptional', '['), ('argoptionalc', ']'), ('ending', ''), ('opt', '-a')]
    >>> sorted(_option('-a=<foo>').groupdict().iteritems())
    [('arg', 'foo'), ('argoptional', '<'), ('argoptionalc', '>'), ('ending', ''), ('opt', '-a')]
    >>> sorted(_option('-a=<foo bar>').groupdict().iteritems())
    [('arg', 'foo bar'), ('argoptional', '<'), ('argoptionalc', '>'), ('ending', ''), ('opt', '-a')]
    >>> sorted(_option('-a=foo').groupdict().iteritems())
    [('arg', 'foo'), ('argoptional', None), ('argoptionalc', None), ('ending', ''), ('opt', '-a')]
    >>> bool(_option('-a=[foo>'))
    False
    >>> bool(_option('-a=[foo bar'))
    False
    >>> _option('-a foo').end(0)
    3
    '''
    m = opt_regex.match(s, pos)
    if m:
        if m.group('argoptional'):
            c = m.group('argoptional')
            cc = m.group('argoptionalc')
            if (c == '[' and cc == ']') or (c == '<' and cc == '>'):
                return m
            else:
                return
    return m

_eatbetweenregex = re.compile(r'\s*(?:or|,|\|)\s*')

def _eatbetween(s, pos):
    '''
    >>> _eatbetween('foo', 0)
    0
    >>> _eatbetween('a, b', 1)
    3
    >>> _eatbetween('a|b', 1)
    2
    >>> _eatbetween('a or b', 1)
    5
    '''
    m = _eatbetweenregex.match(s, pos)
    if m:
        return m.end(0)
    return pos

class extractedoption(collections.namedtuple('extractedoption', 'flag expectsarg')):
    def __eq__(self, other):
        if isinstance(other, str):
            return self.flag == other
        else:
            return super(extractedoption, self).__eq__(other)

    def __str__(self):
        return self.flag

def extract_option(txt):
    '''this is where the magic is (suppose) to happen. try and find options
    using a regex'''
    startpos = currpos = len(txt) - len(txt.lstrip())
    short, long = [], []

    m = _option(txt, currpos)

    # keep going as long as options are found
    while m:
        s = m.group('opt')
        po = extractedoption(s, m.group('arg'))
        if s.startswith('--'):
            long.append(po)
        else:
            short.append(po)
        currpos = m.end(0)
        currpos = _eatbetween(txt, currpos)
        if m.group('ending') == '|':
            m = _option(txt, currpos)
            if not m:
                startpos = currpos
                while currpos < len(txt) and not txt[currpos].isspace():
                    if txt[currpos] == '|':
                        short.append(extractedoption(txt[startpos:currpos], None))
                        startpos = currpos
                    currpos += 1
                leftover = txt[startpos:currpos]
                if leftover:
                    short.append(extractedoption(leftover, None))
        else:
            m = _option(txt, currpos)

    if currpos == startpos:
        m = _flag(txt, currpos)
        while m:
            s = m.group('opt')
            po = extractedoption(s, m.group('arg'))
            long.append(po)
            currpos = m.end(0)
            currpos = _eatbetween(txt, currpos)
            m = _flag(txt, currpos)

    return short, long



File: explainshell/store.py

'''data objects to save processed man pages to mongodb'''
import pymongo, collections, re, logging

from explainshell import errors, util, helpconstants, config

logger = logging.getLogger(__name__)

class classifiermanpage(collections.namedtuple('classifiermanpage', 'name paragraphs')):
    '''a man page that had its paragraphs manually tagged as containing options
    or not'''
    @staticmethod
    def from_store(d):
        m = classifiermanpage(d['name'], [paragraph.from_store(p) for p in d['paragraphs']])
        return m

    def to_store(self):
        return {'name' : self.name,
                'paragraphs' : [p.to_store() for p in self.paragraphs]}

class paragraph(object):
    '''a paragraph inside a man page is text that ends with two new lines'''
    def __init__(self, idx, text, section, is_option):
        self.idx = idx
        self.text = text
        self.section = section
        self.is_option = is_option

    def cleantext(self):
        t = re.sub(r'<[^>]+>', '', self.text)
        t = re.sub('&lt;', '<', t)
        t = re.sub('&gt;', '>', t)
        return t

    @staticmethod
    def from_store(d):
        p = paragraph(d.get('idx', 0), d['text'].encode('utf8'), d['section'], d['is_option'])
        return p

    def to_store(self):
        return {'idx' : self.idx, 'text' : self.text, 'section' : self.section,
                'is_option' : self.is_option}

    def __repr__(self):
        t = self.cleantext()
        t = t[:min(20, t.find('\n'))].lstrip()
        return '<paragraph %d, %s: %r>' % (self.idx, self.section, t)

    def __eq__(self, other):
        if not other:
            return False
        return self.__dict__ == other.__dict__

class option(paragraph):
    '''a paragraph that contains extracted options

    short - a list of short options (-a, -b, ..)
    long - a list of long options (--a, --b)
    expectsarg - specifies if one of the short/long options expects an additional argument
    argument - specifies if to consider this as positional arguments
    nestedcommand - specifies if the arguments to this option can start a nested command
    '''
    def __init__(self, p, short, long, expectsarg, argument=None, nestedcommand=False):
        paragraph.__init__(self, p.idx, p.text, p.section, p.is_option)
        self.short = short
        self.long = long
        self._opts = self.short + self.long
        self.argument = argument
        self.expectsarg = expectsarg
        self.nestedcommand = nestedcommand
        if nestedcommand:
            assert expectsarg, 'an option that can nest commands must expect an argument'

    @property
    def opts(self):
        return self._opts

    @classmethod
    def from_store(cls, d):
        p = paragraph.from_store(d)

        return cls(p, d['short'], d['long'], d['expectsarg'], d['argument'],
                   d.get('nestedcommand'))

    def to_store(self):
        d = paragraph.to_store(self)
        assert d['is_option']
        d['short'] = self.short
        d['long'] = self.long
        d['expectsarg'] = self.expectsarg
        d['argument'] = self.argument
        d['nestedcommand'] = self.nestedcommand
        return d

    def __str__(self):
        return '(%s)' % ', '.join([str(x) for x in self.opts])

    def __repr__(self):
        return '<options for paragraph %d: %s>' % (self.idx, str(self))

class manpage(object):
    '''processed man page

    source - the path to the original source man page
    name - the name of this man page as extracted by manpage.manpage
    synopsis - the synopsis of this man page as extracted by manpage.manpage
    paragraphs - a list of paragraphs (and options) that contain all of the text and options
        extracted from this man page
    aliases - a list of aliases found for this man page
    partialmatch - allow interperting options without a leading '-'
    multicommand - consider sub commands when explaining a command with this man page,
        e.g. git -> git commit
    updated - whether this man page was manually updated
    nestedcommand - specifies if positional arguments to this program can start a nested command,
        e.g. sudo, xargs
    '''
    def __init__(self, source, name, synopsis, paragraphs, aliases,
                 partialmatch=False, multicommand=False, updated=False,
                 nestedcommand=False):
        self.source = source
        self.name = name
        self.synopsis = synopsis
        self.paragraphs = paragraphs
        self.aliases = aliases
        self.partialmatch = partialmatch
        self.multicommand = multicommand
        self.updated = updated
        self.nestedcommand = nestedcommand

    def removeoption(self, idx):
        for i, p in self.paragraphs:
            if p.idx == idx:
                if not isinstance(p, option):
                    raise ValueError("paragraph %d isn't an option" % idx)
                self.paragraphs[i] = paragraph(p.idx, p.text, p.section, False)
                return
        raise ValueError('idx %d not found' % idx)

    @property
    def namesection(self):
        name, section = util.namesection(self.source[:-3])
        return '%s(%s)' % (name, section)

    @property
    def section(self):
        name, section = util.namesection(self.source[:-3])
        return section

    @property
    def options(self):
        return [p for p in self.paragraphs if isinstance(p, option)]

    @property
    def arguments(self):
        # go over all paragraphs and look for those with the same 'argument'
        # field
        groups = collections.OrderedDict()
        for opt in self.options:
            if opt.argument:
                groups.setdefault(opt.argument, []).append(opt)

        # merge all the paragraphs under the same argument to a single string
        for k, l in groups.iteritems():
            groups[k] = '\n\n'.join([p.text for p in l])

        return groups

    @property
    def synopsisnoname(self):
        return re.match(r'[\w|-]+ - (.*)$', self.synopsis).group(1)

    def find_option(self, flag):
        for option in self.options:
            for o in option.opts:
                if o == flag:
                    return option

    def to_store(self):
        return {'source' : self.source, 'name' : self.name, 'synopsis' : self.synopsis,
                'paragraphs' : [p.to_store() for p in self.paragraphs],
                'aliases' : self.aliases, 'partialmatch' : self.partialmatch,
                'multicommand' : self.multicommand, 'updated' : self.updated,
                'nestedcommand' : self.nestedcommand}

    @staticmethod
    def from_store(d):
        paragraphs = []
        for pd in d.get('paragraphs', []):
            pp = paragraph.from_store(pd)
            if pp.is_option == True and 'short' in pd:
                pp = option.from_store(pd)
            paragraphs.append(pp)

        synopsis = d['synopsis']
        if synopsis:
            synopsis = synopsis.encode('utf8')
        else:
            synopsis = helpconstants.NOSYNOPSIS

        return manpage(d['source'], d['name'], synopsis, paragraphs,
                       [tuple(x) for x in d['aliases']], d['partialmatch'],
                       d['multicommand'], d['updated'], d.get('nestedcommand'))

    @staticmethod
    def from_store_name_only(name, source):
        return manpage(source, name, None, [], [], None, None, None)

    def __repr__(self):
        return '<manpage %r(%s), %d options>' % (self.name, self.section, len(self.options))

class store(object):
    '''read/write processed man pages from mongodb

    we use three collections:
    1) classifier - contains manually tagged paragraphs from man pages
    2) manpage - contains a processed man page
    3) mapping - contains (name, manpageid, score) tuples
    '''
    def __init__(self, db='explainshell', host=config.MONGO_URI):
        logger.info('creating store, db = %r, host = %r', db, host)
        self.connection = pymongo.MongoClient(host)
        self.db = self.connection[db]
        self.classifier = self.db['classifier']
        self.manpage = self.db['manpage']
        self.mapping = self.db['mapping']

    def close(self):
        self.connection.disconnect()
        self.classifier = self.manpage = self.mapping = self.db = None

    def drop(self, confirm=False):
        if not confirm:
            return

        logger.info('dropping mapping, manpage, collections')
        self.mapping.drop()
        self.manpage.drop()

    def trainingset(self):
        for d in self.classifier.find():
            yield classifiermanpage.from_store(d)

    def __contains__(self, name):
        c = self.mapping.find({'src' : name}).count()
        return c > 0

    def __iter__(self):
        for d in self.manpage.find():
            yield manpage.from_store(d)

    def findmanpage(self, name):
        '''find a man page by its name, everything following the last dot (.) in name,
        is taken as the section of the man page

        we return the man page found with the highest score, and a list of
        suggestions that also matched the given name (only the first item
        is prepopulated with the option data)'''
        if name.endswith('.gz'):
            logger.info('name ends with .gz, looking up an exact match by source')
            d = self.manpage.find_one({'source':name})
            if not d:
                raise errors.ProgramDoesNotExist(name)
            m = manpage.from_store(d)
            logger.info('returning %s', m)
            return [m]

        section = None
        origname = name

        # don't try to look for a section if it's . (source)
        if name != '.':
            splitted = name.rsplit('.', 1)
            name = splitted[0]
            if len(splitted) > 1:
                section = splitted[1]

        logger.info('looking up manpage in mapping with src %r', name)
        cursor = self.mapping.find({'src' : name})
        count = cursor.count()
        if not count:
            raise errors.ProgramDoesNotExist(name)

        dsts = dict(((d['dst'], d['score']) for d in cursor))
        cursor = self.manpage.find({'_id' : {'$in' : list(dsts.keys())}}, {'name' : 1, 'source' : 1})
        if cursor.count() != len(dsts):
            logger.error('one of %r mappings is missing in manpage collection '
                         '(%d mappings, %d found)', dsts, len(dsts), cursor.count())
        results = [(d.pop('_id'), manpage.from_store_name_only(**d)) for d in cursor]
        results.sort(key=lambda x: dsts.get(x[0], 0), reverse=True)
        logger.info('got %s', results)
        if section is not None:
            if len(results) > 1:
                results.sort(key=lambda (oid, m): m.section == section, reverse=True)
                logger.info(r'sorting %r so %s is first', results, section)
            if not results[0][1].section == section:
                raise errors.ProgramDoesNotExist(origname)
            results.extend(self._discovermanpagesuggestions(results[0][0], results))

        oid = results[0][0]
        results = [x[1] for x in results]
        results[0] = manpage.from_store(self.manpage.find_one({'_id' : oid}))
        return results

    def _discovermanpagesuggestions(self, oid, existing):
        '''find suggestions for a given man page

        oid is the objectid of the man page in question,
        existing is a list of (oid, man page) of suggestions that were
        already discovered
        '''
        skip = set([oid for oid, m in existing])
        cursor = self.mapping.find({'dst' : oid})
        # find all srcs that point to oid
        srcs = [d['src'] for d in cursor]
        # find all dsts of srcs
        suggestionoids = self.mapping.find({'src' : {'$in' : srcs}}, {'dst' : 1})
        # remove already discovered
        suggestionoids = [d['dst'] for d in suggestionoids if d['dst'] not in skip]
        if not suggestionoids:
            return []

        # get just the name and source of found suggestions
        suggestionoids = self.manpage.find({'_id' : {'$in' : suggestionoids}},
                                           {'name' : 1, 'source' : 1})
        return [(d.pop('_id'), manpage.from_store_name_only(**d)) for d in suggestionoids]

    def addmapping(self, src, dst, score):
        self.mapping.insert({'src' : src, 'dst' : dst, 'score' : score})

    def addmanpage(self, m):
        '''add m into the store, if it exists first remove it and its mappings

        each man page may have aliases besides the name determined by its
        basename'''
        d = self.manpage.find_one({'source' : m.source})
        if d:
            logger.info('removing old manpage %s (%s)', m.source, d['_id'])
            self.manpage.remove(d['_id'])

            # remove old mappings if there are any
            c = self.mapping.count()
            self.mapping.remove({'dst' : d['_id']})
            c -= self.mapping.count()
            logger.info('removed %d mappings for manpage %s', c, m.source)

        o = self.manpage.insert(m.to_store())

        for alias, score in m.aliases:
            self.addmapping(alias, o, score)
            logger.info('inserting mapping (alias) %s -> %s (%s) with score %d', alias, m.name, o, score)
        return m

    def updatemanpage(self, m):
        '''update m and add new aliases if necessary

        change updated attribute so we don't overwrite this in the future'''
        logger.info('updating manpage %s', m.source)
        m.updated = True
        self.manpage.update({'source' : m.source}, m.to_store())
        _id = self.manpage.find_one({'source' : m.source}, fields={'_id':1})['_id']
        for alias, score in m.aliases:
            if alias not in self:
                self.addmapping(alias, _id, score)
                logger.info('inserting mapping (alias) %s -> %s (%s) with score %d', alias, m.name, _id, score)
            else:
                logger.debug('mapping (alias) %s -> %s (%s) already exists', alias, m.name, _id)
        return m

    def verify(self):
        # check that everything in manpage is reachable
        mappings = list(self.mapping.find())
        reachable = set([m['dst'] for m in mappings])
        manpages = set([m['_id'] for m in self.manpage.find(fields={'_id':1})])

        ok = True
        unreachable = manpages - reachable
        if unreachable:
            logger.error('manpages %r are unreachable (nothing maps to them)', unreachable)
            unreachable = [self.manpage.find_one({'_id' : u})['name'] for u in unreachable]
            ok = False

        notfound = reachable - manpages
        if notfound:
            logger.error('mappings to inexisting manpages: %r', notfound)
            ok = False

        return ok, unreachable, notfound

    def names(self):
        cursor = self.manpage.find(fields={'name':1})
        for d in cursor:
            yield d['_id'], d['name']

    def mappings(self):
        cursor = self.mapping.find(fields={'src':1})
        for d in cursor:
            yield d['src'], d['_id']

    def setmulticommand(self, manpageid):
        self.manpage.update({'_id' : manpageid}, {'$set' : {'multicommand' : True}})



File: explainshell/fixer.py

import textwrap, logging

from explainshell import util

class basefixer(object):
    '''The base fixer class which other fixers inherit from.

    Subclasses override the base methods in order to fix manpage content during
    different parts of the parsing/classifying/saving process.'''
    runbefore = []
    runlast = False

    def __init__(self, mctx):
        self.mctx = mctx
        self.run = True
        self.logger = logging.getLogger(self.__class__.__name__)

    def pre_get_raw_manpage(self):
        pass

    def pre_parse_manpage(self):
        pass

    def post_parse_manpage(self):
        pass

    def pre_classify(self):
        pass

    def post_classify(self):
        pass

    def post_option_extraction(self):
        pass

    def pre_add_manpage(self):
        pass

fixerscls = []
fixerspriority = {}

class runner(object):
    '''The runner coordinates the fixers.'''
    def __init__(self, mctx):
        self.mctx = mctx
        self.fixers = [f(mctx) for f in fixerscls]

    def disable(self, name):
        before = len(self.fixers)
        self.fixers = [f for f in self.fixers if f.__class__.__name__ != name]
        if before == len(self.fixers):
            raise ValueError('fixer %r not found' % name)

    def _fixers(self):
        return (f for f in self.fixers if f.run)

    def pre_get_raw_manpage(self):
        for f in self._fixers():
            f.pre_get_raw_manpage()

    def pre_parse_manpage(self):
        for f in self._fixers():
            f.pre_parse_manpage()

    def post_parse_manpage(self):
        for f in self._fixers():
            f.post_parse_manpage()

    def pre_classify(self):
        for f in self._fixers():
            f.pre_classify()

    def post_classify(self):
        for f in self._fixers():
            f.post_classify()

    def post_option_extraction(self):
        for f in self._fixers():
            f.post_option_extraction()

    def pre_add_manpage(self):
        for f in self._fixers():
            f.pre_add_manpage()

def register(fixercls):
    fixerscls.append(fixercls)
    for f in fixercls.runbefore:
        if not hasattr(f, '_parents'):
            f._parents = []
        f._parents.append(fixercls)
    return fixercls

@register
class bulletremover(basefixer):
    '''remove list bullets from paragraph start, see mysqlslap.1'''
    def post_parse_manpage(self):
        toremove = []
        for i, p in enumerate(self.mctx.manpage.paragraphs):
            try:
                idx = p.text.index('\xc2\xb7')
                p.text = p.text[:idx] + p.text[idx+2:]
                if not p.text.strip():
                    toremove.append(i)
            except ValueError:
                pass
        for i in reversed(toremove):
            del self.mctx.manpage.paragraphs[i]

@register
class leadingspaceremover(basefixer):
    '''go over all known option paragraphs and remove their leading spaces
    by the amount of spaces in the first line'''

    def post_option_extraction(self):
        for i, p in enumerate(self.mctx.manpage.options):
            text = self._removewhitespace(p.text)
            p.text = text

    def _removewhitespace(self, text):
        '''
        >>> f = leadingspaceremover(None)
        >>> f._removewhitespace(' a\\n  b ')
        'a\\n b'
        >>> f._removewhitespace('\\t a\\n\\t \\tb')
        'a\\n\\tb'
        '''
        return textwrap.dedent(text).rstrip()

@register
class tarfixer(basefixer):
    def __init__(self, *args):
        super(tarfixer, self).__init__(*args)
        self.run = self.mctx.name == 'tar'

    def pre_add_manpage(self):
        self.mctx.manpage.partialmatch = True

@register
class paragraphjoiner(basefixer):
    runbefore = [leadingspaceremover]
    maxdistance = 5

    def post_option_extraction(self):
        options = [p for p in self.mctx.manpage.paragraphs if p.is_option]
        self._join(self.mctx.manpage.paragraphs, options)

    def _join(self, paragraphs, options):
        def _paragraphsbetween(op1, op2):
            assert op1.idx < op2.idx
            r = []
            start = None
            for i, p in enumerate(paragraphs):
                if op1.idx < p.idx < op2.idx:
                    if not r:
                        start = i
                    r.append(p)
            return r, start

        totalmerged = 0
        for curr, next in util.pairwise(options):
            between, start = _paragraphsbetween(curr, next)
            if curr.section == next.section and 1 <= len(between) < self.maxdistance:
                self.logger.info('merging paragraphs %d through %d (inclusive)', curr.idx, next.idx-1)
                newdesc = [curr.text.rstrip()]
                newdesc.extend([p.text.rstrip() for p in between])
                curr.text = '\n\n'.join(newdesc)
                del paragraphs[start:start+len(between)]
                totalmerged += len(between)
        return totalmerged

@register
class optiontrimmer(basefixer):
    runbefore = [paragraphjoiner]

    d = {'git-rebase' : (50, -1)}

    def __init__(self, mctx):
        super(optiontrimmer, self).__init__(mctx)
        self.run = self.mctx.name in self.d

    def post_classify(self):
        start, end = self.d[self.mctx.name]
        classifiedoptions = [p for p in self.mctx.manpage.paragraphs if p.is_option]
        assert classifiedoptions
        if end == -1:
            end = classifiedoptions[-1].idx
        else:
            assert start > end

        for p in classifiedoptions:
            if not (start <= p.idx <= end):
                p.is_option = False
                self.logger.info('removing option %r', p)

def _parents(fixercls):
    p = getattr(fixercls, '_parents', [])
    last = fixercls.runlast

    if last and p:
        raise ValueError("%s can't be last and also run before someone else" % fixercls.__name__)

    if last:
        return [f for f in fixerscls if f is not fixercls]
    return p

fixerscls = util.toposorted(fixerscls, _parents)



File: explainshell/manpage.py

import os, subprocess, re, logging, collections, urllib

from explainshell import config, store, errors

devnull = open(os.devnull, 'w')
SPLITSYNOP = re.compile(r'([^ ]+) - (.*)$')

ENV = dict(os.environ)
ENV["W3MMAN_MAN"] = "man --no-hyphenation"
ENV["MAN_KEEP_FORMATTING"] = "1"
ENV["MANWIDTH"] = "115"
ENV["LC_ALL"] = "en_US.UTF-8"

logger = logging.getLogger(__name__)

def extractname(gzname):
    '''
    >>> extractname('ab.1.gz')
    'ab'
    >>> extractname('ab.1.1.gz')
    'ab.1'
    >>> extractname('ab.1xyz.gz')
    'ab'
    >>> extractname('ab.1.1xyz.gz')
    'ab.1'
    >>> extractname('a/b/c/ab.1.1xyz.gz')
    'ab.1'
    '''
    if '/' in gzname:
        gzname = os.path.basename(gzname)
    return gzname.rsplit('.', 2)[0]

def bold(l):
    '''
    >>> bold('a')
    ([], ['a'])
    >>> bold('<b>a</b>')
    (['a'], [])
    >>> bold('a<b>b</b>c')
    (['b'], ['a', 'c'])
    >>> bold('<b>first</b> <b>second:</b>')
    (['first', 'second:'], [])
    '''
    inside = []
    for m in _section.finditer(l):
        inside.append(m.span(0))

    current = 0
    outside = []
    for start, end in inside:
        outside.append((current, start))
        current = end
    outside.append((current, len(l)))

    inside = [l[s:e] for s, e in inside]
    inside = [s.replace('<b>', '').replace('</b>', '') for s in inside]

    outside = [l[s:e] for s, e in outside]
    outside = [l for l in outside if l and not l.isspace()]
    return inside, outside

# w3mman2html.cgi (the tool we're using to output html from a man page) does
# some strange escaping which causes it to output invalid utf8. we look these
# up and fix them manually
_replacementsprefix = [
        ('\xe2\x80\xe2\x80\x98', None, True), # left single quote
        ('\xe2\x80\xe2\x80\x99', None, True), # right single quote
        ('\xe2\x80\xe2\x80\x9c', None, True), # left double quote
        ('\xe2\x80\xe2\x80\x9d', None, True), # right double quote
        ('\xe2\x94\xe2\x94\x82', '|', False), # pipe
        ('\xe2\x8e\xe2\x8e\xaa', None, False), # pipe 2
        ('\xe2\x80\xe2\x80\x90', None, True), # hyphen
        ('\xe2\x80\xe2\x80\x94', None, True), # hyphen 2
        ('\xe2\x80\xc2\xbd', None, True), # half
        ('\xe2\x88\xe2\x88\x97', None, True), # asterisk
        ('\xe2\x86\xe2\x86\x92', None, True), # right arrow
        ('\xe2\x88\xe2\x88\x92', None, True), # minus sign
        ('\xe2\x80\xe2\x80\x93', None, True), # en dash
        ('\xe2\x80\xe2\x80\xb2', None, False), # prime
        ('\xe2\x88\xe2\x88\xbc', None, False), # tilde operator
        ('\xe2\x86\xe2\x86\xb5', None, False), # downwards arrow with corner leftwards
        ('\xef\xbf\xef\xbf\xbd', None, False) # replacement char
        ]

_replacements = []

for searchfor, replacewith, underline in _replacementsprefix:
    if replacewith is None:
        replacewith = searchfor[2:]
    _replacements.append((searchfor, replacewith))
    if underline:
        x = list(replacewith)
        x.insert(1, '</u>')
        x = ''.join(x)
        _replacements.append((x, '%s</u>' % replacewith))

_replacementsnoprefix = ['\xc2\xb7', # bullet
        '\xc2\xb4', # apostrophe
        '\xc2\xa0', # no break space
        '\xc3\xb8', '\xe4\xbd\xa0', '\xe5\xa5\xbd', # gibberish
        '\xc2\xa7', # section sign
        '\xef\xbf\xbd', # replacement char
        '\xc2\xa4',  # latin small letter a with diaeresis
        '\xc3\xa4', # latin small letter a with diaeresis
        '\xc4\xa4',  # latin small letter a with diaeresis
        '\xc3\xaa',  # latin small letter e with circumflex
        ]

for s in _replacementsnoprefix:
    x = list(s)
    x.insert(1, '</u>')
    x = ''.join(x)
    _replacements.append((x, '%s</u>' % s))

_href = re.compile(r'<a href="file:///[^\?]*\?([^\(]*)\(([^\)]*)\)">')
_section = re.compile(r'<b>([^<]+)</b>')

def _parsetext(lines):
    paragraphlines = []
    section = None
    i = 0
    for l in lines:
        l = re.sub(_href, r'<a href="http://manpages.ubuntu.com/manpages/precise/en/man\2/\1.\2.html">', l)
        for lookfor, replacewith in _replacements:
            l = re.sub(lookfor, replacewith, l)
        # confirm the line is valid utf8
        lreplaced = l.decode('utf8', 'ignore').encode('utf8')
        if lreplaced != l:
            logger.error('line %r contains invalid utf8', l)
            l = lreplaced
            raise ValueError
        if l.startswith('<b>'): # section
            section = re.sub(_section, r'\1', l)
        else:
            foundsection = False
            if l.strip().startswith('<b>'):
                inside, outside = bold(l.strip())
                if not outside and inside[-1][-1] == ':':
                    foundsection = True
                    section = ' '.join(inside)[:-1]
            if not foundsection:
                if not l.strip() and paragraphlines:
                    yield store.paragraph(i, '\n'.join(paragraphlines), section, False)
                    i += 1
                    paragraphlines = []
                elif l.strip():
                    paragraphlines.append(l)
    if paragraphlines:
        yield store.paragraph(i, '\n'.join(paragraphlines), section, False)

def _parsesynopsis(base, synopsis):
    '''
    >>> _parsesynopsis('/a/b/c', '/a/b/c: "p-r+o++g - foo bar."')
    ('p-r+o++g', 'foo bar')
    '''
    synopsis = synopsis[len(base)+3:-1]
    if synopsis[-1] == '.':
        synopsis = synopsis[:-1]
    return SPLITSYNOP.match(synopsis).groups()

class manpage(object):
    '''read the man page at path by executing w3mman2html.cgi and find its
    synopsis with lexgrog

    since some man pages share the same name (different versions), each
    alias of a man page has a score that's determined in this simple fashion:
    - name of man page source file is given a score of 10
    - all other names found for a particular man page are given a score of 1
      (other names are found by scanning the output of lexgrog)
    '''
    def __init__(self, path):
        self.path = path
        self.shortpath = os.path.basename(self.path)
        self.name = extractname(self.path)
        self.aliases = set([self.name])
        self.synopsis = None
        self.paragraphs = None
        self._text = None

    def read(self):
        '''Read the content from a local manpage file and store it in usable formats
        on the class instance.'''
        cmd = [config.MAN2HTML, urllib.urlencode({'local' : os.path.abspath(self.path)})]
        logger.info('executing %r', ' '.join(cmd))
        self._text = subprocess.check_output(cmd, stderr=devnull, env=ENV)
        try:
            self.synopsis = subprocess.check_output(['lexgrog', self.path], stderr=devnull).rstrip()
        except subprocess.CalledProcessError:
            logger.error('failed to extract synopsis for %s', self.name)

    def parse(self):
        self.paragraphs = list(_parsetext(self._text.splitlines()[7:-3]))
        if not self.paragraphs:
            raise errors.EmptyManpage(self.shortpath)
        if self.synopsis:
            self.synopsis = [_parsesynopsis(self.path, l) for l in self.synopsis.splitlines()]

            # figure out aliases from the synopsis
            d = collections.OrderedDict()
            for prog, text in self.synopsis:
                d.setdefault(text, []).append(prog)
            text, progs = d.items()[0]
            self.synopsis = text
            self.aliases.update(progs)
        self.aliases.remove(self.name)

        # give the name of the man page the highest score
        self.aliases = [(self.name, 10)] + [(x, 1) for x in self.aliases]



File: explainshell/config.py

import os

_currdir = os.path.dirname(os.path.dirname(__file__))

MANPAGEDIR = os.path.join(_currdir, 'manpages')
CLASSIFIER_CUTOFF = 0.7
TOOLSDIR = os.path.join(_currdir, 'tools')

MAN2HTML = os.path.join(TOOLSDIR, 'w3mman2html.cgi')

# host to pass into Flask's app.run.
HOST_IP = os.getenv('HOST_IP', False)
MONGO_URI = os.getenv('MONGO_URI', 'mongodb://localhost')
DEBUG = True

LOGGING_DICT = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
        },
    },
    'handlers': {
        'console': {
            'level' : 'INFO',
            'class' : 'logging.StreamHandler',
            'formatter': 'standard',
        },
        'file': {
            'class': 'logging.FileHandler',
            'level': 'INFO',
            'formatter': 'standard',
            'filename': 'application.log',
            'mode': 'a',
        },
    },
    'loggers': {
        'explainshell': {
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': False
        }
    }
}



File: explainshell/matcher.py

import collections, logging, itertools

import bashlex.parser
import bashlex.ast

from explainshell import errors, util, helpconstants

class matchgroup(object):
    '''a class to group matchresults together

    we group all shell results in one group and create a new group for every
    command'''
    def __init__(self, name):
        self.name = name
        self.results = []

    def __repr__(self):
        return '<matchgroup %r with %d results>' % (self.name, len(self.results))

class matchresult(collections.namedtuple('matchresult', 'start end text match')):
    @property
    def unknown(self):
        return self.text is None

matchwordexpansion = collections.namedtuple('matchwordexpansion',
                                            'start end kind')

logger = logging.getLogger(__name__)

class matcher(bashlex.ast.nodevisitor):
    '''parse a command line and return a list of matchresults describing
    each token.
    '''
    def __init__(self, s, store):
        self.s = s.encode('latin1', 'replace')
        self.store = store
        self._prevoption = self._currentoption = None
        self.groups = [matchgroup('shell')]

        # a list of matchwordexpansions where expansions happened during word
        # expansion
        self.expansions = []

        # a stack to manage nested command groups: whenever a new simple
        # command is started, we push a tuple with:
        # - the node that started this group. this is used to find it when
        #   a command ends (see visitnodeend)
        # - its matchgroup. new matchresults will be added to it.
        # - a word used to end the top-most command. this is used when a flag
        #   starts a new command, e.g.  find -exec.
        self.groupstack = [(None, self.groups[-1], None)]

        # keep a stack of the currently visited compound command (if/for..)
        # to provide context when matching reserved words, since for example
        # the keyword 'done' can appear in a for, while..
        self.compoundstack = []

        # a set of functions defined in the current input, we will try to match
        # commands against them so if one refers to defined function, it won't
        # show up as unknown or be taken from the db
        self.functions = set()

    def _generatecommandgroupname(self):
        existing = len([g for g in self.groups if g.name.startswith('command')])
        return 'command%d' % existing

    @property
    def matches(self):
        '''return the list of results from the most recently created group'''
        return self.groupstack[-1][1].results

    @property
    def allmatches(self):
        return list(itertools.chain.from_iterable(g.results for g in self.groups))

    @property
    def manpage(self):
        group = self.groupstack[-1][1]
        # we do not have a manpage if the top of the stack is the shell group.
        # this can happen if the first argument is a command substitution
        # and we're not treating it as a "man page not found"
        if group.name != 'shell':
            return group.manpage

    def find_option(self, opt):
        self._currentoption = self.manpage.find_option(opt)
        logger.debug('looking up option %r, got %r', opt, self._currentoption)
        return self._currentoption

    def findmanpages(self, prog):
        prog = prog.decode('latin1')
        logger.info('looking up %r in store', prog)
        manpages = self.store.findmanpage(prog)
        logger.info('found %r in store, got: %r, using %r', prog, manpages, manpages[0])
        return manpages

    def unknown(self, token, start, end):
        logger.debug('nothing to do with token %r', token)
        return matchresult(start, end, None, None)

    def visitreservedword(self, node, word):
        # first try the compound reserved words
        helptext = None
        if self.compoundstack:
            currentcompound = self.compoundstack[-1]
            helptext = helpconstants.COMPOUNDRESERVEDWORDS.get(currentcompound, {}).get(word)

        # try these if we don't have anything specific
        if not helptext:
            helptext = helpconstants.RESERVEDWORDS[word]

        self.groups[0].results.append(matchresult(node.pos[0], node.pos[1], helptext, None))

    def visitoperator(self, node, op):
        helptext = None
        if self.compoundstack:
            currentcompound = self.compoundstack[-1]
            helptext = helpconstants.COMPOUNDRESERVEDWORDS.get(currentcompound, {}).get(op)

        if not helptext:
            helptext = helpconstants.OPERATORS[op]

        self.groups[0].results.append(matchresult(node.pos[0], node.pos[1], helptext, None))

    def visitpipe(self, node, pipe):
        self.groups[0].results.append(
                matchresult(node.pos[0], node.pos[1], helpconstants.PIPELINES, None))

    def visitredirect(self, node, input, type, output, heredoc):
        helptext = [helpconstants.REDIRECTION]

        if type == '>&' and isinstance(output, int):
            type = type[:-1]

        if type in helpconstants.REDIRECTION_KIND:
            helptext.append(helpconstants.REDIRECTION_KIND[type])

        self.groups[0].results.append(
                matchresult(node.pos[0], node.pos[1], '\n\n'.join(helptext), None))

        # the output might contain a wordnode, visiting it will confuse the
        # matcher who'll think it's an argument, instead visit the expansions
        # directly, if we have any
        if isinstance(output, bashlex.ast.node):
            for part in output.parts:
                self.visit(part)

        return False

    def visitcommand(self, node, parts):
        assert parts

        # look for the first WordNode, which might not be at parts[0]
        idxwordnode = bashlex.ast.findfirstkind(parts, 'word')
        if idxwordnode == -1:
            logger.info('no words found in command (probably contains only redirects)')
            return

        wordnode = parts[idxwordnode]

        # check if this refers to a previously defined function
        if wordnode.word in self.functions:
            logger.info('word %r is a function, not trying to match it or its '
                        'arguments', wordnode)

            # first, add a matchresult for the function call
            mr = matchresult(wordnode.pos[0], wordnode.pos[1],
                             helpconstants._functioncall % wordnode.word, None)
            self.matches.append(mr)

            # this is a bit nasty: if we were to visit the command like we
            # normally do it would try to match it against a manpage. but
            # we don't have one here, we just want to take all the words and
            # consider them part of the function call
            for part in parts:
                # maybe it's a redirect...
                if part.kind != 'word':
                    self.visit(part)
                else:
                    # this is an argument to the function
                    if part is not wordnode:
                        mr = matchresult(part.pos[0], part.pos[1],
                                         helpconstants._functionarg % wordnode.word,
                                         None)
                        self.matches.append(mr)

                        # visit any expansions in there
                        for ppart in part.parts:
                            self.visit(ppart)

            # we're done with this commandnode, don't visit its children
            return False


        self.startcommand(node, parts, None)

    def visitif(self, *args):
        self.compoundstack.append('if')
    def visitfor(self, node, parts):
        self.compoundstack.append('for')

        for part in parts:
            # don't visit words since they're not part of the current command,
            # instead consider them part of the for construct
            if part.kind == 'word':
                mr = matchresult(part.pos[0], part.pos[1], helpconstants._for, None)
                self.groups[0].results.append(mr)

                # but we do want to visit expanions
                for ppart in part.parts:
                    self.visit(ppart)
            else:
                self.visit(part)

        return False

    def visitwhile(self, *args):
        self.compoundstack.append('while')
    def visituntil(self, *args):
        self.compoundstack.append('until')

    def visitnodeend(self, node):
        if node.kind == 'command':
            # it's possible for visitcommand/end to be called without a command
            # group being pushed if it contains only redirect nodes
            if len(self.groupstack) > 1:
                logger.info('visitnodeend %r, groups %d', node, len(self.groupstack))

                while self.groupstack[-1][0] is not node:
                    logger.info('popping groups that are a result of nested commands')
                    self.endcommand()
                self.endcommand()
        elif node.kind in ('if', 'for', 'while', 'until'):
            kind = self.compoundstack.pop()
            assert kind == node.kind

    def startcommand(self, commandnode, parts, endword, addgroup=True):
        logger.info('startcommand commandnode=%r parts=%r, endword=%r, addgroup=%s',
                    commandnode, parts, endword, addgroup)
        idxwordnode = bashlex.ast.findfirstkind(parts, 'word')
        assert idxwordnode != -1

        wordnode = parts[idxwordnode]
        if wordnode.parts:
            logger.info('node %r has parts (it was expanded), no point in looking'
                        ' up a manpage for it', wordnode)

            if addgroup:
                mg = matchgroup(self._generatecommandgroupname())
                mg.manpage = None
                mg.suggestions = None
                self.groups.append(mg)
                self.groupstack.append((commandnode, mg, endword))

            return False

        startpos, endpos = wordnode.pos

        try:
            mps = self.findmanpages(wordnode.word)
            # we consume this node here, pop it from parts so we
            # don't visit it again as an argument
            parts.pop(idxwordnode)
        except errors.ProgramDoesNotExist, e:
            if addgroup:
                # add a group for this command, we'll mark it as unknown
                # when visitword is called
                logger.info('no manpage found for %r, adding a group for it',
                            wordnode.word)

                mg = matchgroup(self._generatecommandgroupname())
                mg.error = e
                mg.manpage = None
                mg.suggestions = None
                self.groups.append(mg)
                self.groupstack.append((commandnode, mg, endword))

            return False

        manpage = mps[0]
        idxnextwordnode = bashlex.ast.findfirstkind(parts, 'word')

        # check the next word for a possible multicommand if:
        # - the matched manpage says so
        # - we have another word node
        # - the word node has no expansions in it
        if manpage.multicommand and idxnextwordnode != -1 and not parts[idxnextwordnode].parts:
            nextwordnode = parts[idxnextwordnode]
            try:
                multi = '%s %s' % (wordnode.word, nextwordnode.word)
                logger.info('%r is a multicommand, trying to get another token and look up %r', manpage, multi)
                mps = self.findmanpages(multi)
                manpage = mps[0]
                # we consume this node here, pop it from parts so we
                # don't visit it again as an argument
                parts.pop(idxnextwordnode)
                endpos = nextwordnode.pos[1]
            except errors.ProgramDoesNotExist:
                logger.info('no manpage %r for multicommand %r', multi, manpage)

        # create a new matchgroup for the current command
        mg = matchgroup(self._generatecommandgroupname())
        mg.manpage = manpage
        mg.suggestions = mps[1:]
        self.groups.append(mg)
        self.groupstack.append((commandnode, mg, endword))

        self.matches.append(matchresult(startpos, endpos,
                            manpage.synopsis or helpconstants.NOSYNOPSIS, None))
        return True

    def endcommand(self):
        '''end the most recently created command group by popping it from the
        group stack. groups are created by visitcommand or a nested command'''
        assert len(self.groupstack) >= 2, 'groupstack must contain shell and command groups'
        g = self.groupstack.pop()
        logger.info('ending group %s', g)

    def visitcommandsubstitution(self, node, command):
        kind = self.s[node.pos[0]]
        substart = 2 if kind == '$' else 1

        # start the expansion after the $( or `
        self.expansions.append(matchwordexpansion(node.pos[0] + substart,
                                                  node.pos[1] - 1,
                                                  'substitution'))

        # do not try to match the child nodes
        return False

    def visitprocesssubstitution(self, node, command):
        # don't include opening <( and closing )
        self.expansions.append(matchwordexpansion(node.pos[0] + 2,
                                                  node.pos[1] - 1,
                                                  'substitution'))

        # do not try to match the child nodes
        return False

    def visitassignment(self, node, word):
        helptext = helpconstants.ASSIGNMENT
        self.groups[0].results.append(matchresult(node.pos[0], node.pos[1], helptext, None))

    def visitword(self, node, word):
        def attemptfuzzy(chars):
            m = []
            if chars[0] == '-':
                tokens = [chars[0:2]] + list(chars[2:])
                considerarg = True
            else:
                tokens = list(chars)
                considerarg = False

            pos = node.pos[0]
            prevoption = None
            for i, t in enumerate(tokens):
                op = t if t[0] == '-' else '-' + t
                option = self.find_option(op)
                if option:
                    if considerarg and not m and option.expectsarg:
                        logger.info('option %r expected an arg, taking the rest too', option)
                        # reset the current option if we already took an argument,
                        # this prevents the next word node to also consider itself
                        # as an argument
                        self._currentoption = None
                        return [matchresult(pos, pos+len(chars), option.text, None)]

                    mr = matchresult(pos, pos+len(t), option.text, None)
                    m.append(mr)
                # if the previous option expected an argument and we couldn't
                # match the current token, take the rest as its argument, this
                # covers a series of short options where the last one has an argument
                # with no space between it, such as 'xargs -r0n1'
                elif considerarg and prevoption and prevoption.expectsarg:
                    pmr = m[-1]
                    mr = matchresult(pmr.start, pmr.end+(len(tokens)-i), pmr.text, None)
                    m[-1] = mr
                    # reset the current option if we already took an argument,
                    # this prevents the next word node to also consider itself
                    # as an argument
                    self._currentoption = None
                    break
                else:
                    m.append(self.unknown(t, pos, pos+len(t)))
                pos += len(t)
                prevoption = option
            return m

        def _visitword(node, word):
            if not self.manpage:
                logger.info('inside an unknown command, giving up on %r', word)
                self.matches.append(self.unknown(word, node.pos[0], node.pos[1]))
                return

            logger.info('trying to match token: %r', word)

            self._prevoption = self._currentoption
            if word.startswith('--'):
                word = word.split('=', 1)[0]
            option = self.find_option(word)
            if option:
                logger.info('found an exact match for %r: %r', word, option)
                mr = matchresult(node.pos[0], node.pos[1], option.text, None)
                self.matches.append(mr)

                # check if we splitted the word just above, if we did then reset
                # the current option so the next word doesn't consider itself
                # an argument
                if word != node.word:
                    self._currentoption = None
            else:
                word = node.word

                # check if we're inside a nested command and this word marks the end
                if isinstance(self.groupstack[-1][-1], list) and word in self.groupstack[-1][-1]:
                    logger.info('token %r ends current nested command', word)
                    self.endcommand()
                    mr = matchresult(node.pos[0], node.pos[1], self.matches[-1].text, None)
                    self.matches.append(mr)
                elif word != '-' and word.startswith('-') and not word.startswith('--'):
                    logger.debug('looks like a short option')
                    if len(word) > 2:
                        logger.info("trying to split it up")
                        self.matches.extend(attemptfuzzy(word))
                    else:
                        self.matches.append(self.unknown(word, node.pos[0], node.pos[1]))
                elif self._prevoption and self._prevoption.expectsarg:
                    logger.info("previous option possibly expected an arg, and we can't"
                            " find an option to match the current token, assuming it's an arg")
                    ea = self._prevoption.expectsarg
                    possibleargs = ea if isinstance(ea, list) else []
                    take = True
                    if possibleargs and word not in possibleargs:
                        take = False
                        logger.info('token %r not in list of possible args %r for %r',
                                    word, possibleargs, self._prevoption)
                    if take:
                        if self._prevoption.nestedcommand:
                            logger.info('option %r can nest commands', self._prevoption)
                            if self.startcommand(None, [node], self._prevoption.nestedcommand, addgroup=False):
                                self._currentoption = None
                                return

                        pmr = self.matches[-1]
                        mr = matchresult(pmr.start, node.pos[1], pmr.text, None)
                        self.matches[-1] = mr
                    else:
                        self.matches.append(self.unknown(word, node.pos[0], node.pos[1]))
                else:
                    if self.manpage.partialmatch:
                        logger.info('attemping to do a partial match')

                        m = attemptfuzzy(word)
                        if not any(mm.unknown for mm in m):
                            logger.info('found a match for everything, taking it')
                            self.matches.extend(m)
                            return

                    if self.manpage.arguments:
                        if self.manpage.nestedcommand:
                            logger.info('manpage %r can nest commands', self.manpage)
                            if self.startcommand(None, [node], self.manpage.nestedcommand, addgroup=False):
                                self._currentoption = None
                                return

                        d = self.manpage.arguments
                        k = list(d.keys())[0]
                        logger.info('got arguments, using %r', k)
                        text = d[k]
                        mr = matchresult(node.pos[0], node.pos[1], text, None)
                        self.matches.append(mr)
                        return

                    # if all of that failed, we can't explain it so mark it unknown
                    self.matches.append(self.unknown(word, node.pos[0], node.pos[1]))

        _visitword(node, word)

    def visitfunction(self, node, name, body, parts):
        self.functions.add(name.word)

        def _iscompoundopenclosecurly(compound):
            first, last = compound.list[0], compound.list[-1]
            if (first.kind == 'reservedword' and last.kind == 'reservedword' and
                first.word == '{' and last.word == '}'):
                return True

        # if the compound command we have there is { }, let's include the
        # {} as part of the function declaration. normally it would be
        # treated as a group command, but that seems less informative in this
        # context
        if _iscompoundopenclosecurly(body):
            # create a matchresult until after the first {
            mr = matchresult(node.pos[0], body.list[0].pos[1],
                             helpconstants._function, None)
            self.groups[0].results.append(mr)

            # create a matchresult for the closing }
            mr = matchresult(body.list[-1].pos[0], body.list[-1].pos[1],
                             helpconstants._function, None)
            self.groups[0].results.append(mr)

            # visit anything in between the { }
            for part in body.list[1:-1]:
                self.visit(part)
        else:
            beforebody = bashlex.ast.findfirstkind(parts, 'compound') - 1
            assert beforebody > 0
            beforebody = parts[beforebody]

            # create a matchresult ending at the node before body
            mr = matchresult(node.pos[0], beforebody.pos[1],
                             helpconstants._function, None)
            self.groups[0].results.append(mr)

            self.visit(body)

        return False

    def visittilde(self, node, value):
        self.expansions.append(matchwordexpansion(node.pos[0], node.pos[1],
                                                  'tilde'))

    def visitparameter(self, node, value):
        try:
            int(value)
            kind = 'digits'
        except ValueError:
            kind = helpconstants.parameters.get(value, 'param')

        self.expansions.append(matchwordexpansion(node.pos[0], node.pos[1],
                                                  'parameter-%s' % kind))

    def match(self):
        logger.info('matching string %r', self.s)

        # limit recursive parsing to a depth of 1
        self.ast = bashlex.parser.parsesingle(self.s, expansionlimit=1,
                                              strictmode=False)
        if self.ast:
            self.visit(self.ast)
            assert len(self.groupstack) == 1, 'groupstack should contain only shell group after matching'

            # if we only have one command in there and no shell results/expansions,
            # reraise the original exception
            if (len(self.groups) == 2 and not self.groups[0].results and
                self.groups[1].manpage is None and not self.expansions):
                raise self.groups[1].error
        else:
            logger.warn('no AST generated for %r', self.s)

        def debugmatch():
            s = '\n'.join(['%d) %r = %r' % (i, self.s[m.start:m.end], m.text) for i, m in enumerate(self.allmatches)])
            return s

        self._markunparsedunknown()

        # fix each matchgroup seperately
        for group in self.groups:
            if group.results:
                if getattr(group, 'manpage', None):
                    # ensure that the program part isn't unknown (i.e. it has
                    # something as its synopsis)
                    assert not group.results[0].unknown

                group.results = self._mergeadjacent(group.results)

                # add matchresult.match to existing matches
                for i, m in enumerate(group.results):
                    assert m.end <= len(self.s), '%d %d' % (m.end, len(self.s))

                    portion = self.s[m.start:m.end].decode('latin1')
                    group.results[i] = matchresult(m.start, m.end, m.text, portion)

        logger.debug('%r matches:\n%s', self.s, debugmatch())

        # not strictly needed, but doesn't hurt
        self.expansions.sort()

        return self.groups

    def _markunparsedunknown(self):
        '''the parser may leave a remainder at the end of the string if it doesn't
        match any of the rules, mark them as unknowns'''
        parsed = [False]*len(self.s)

        # go over all existing matches to see if we've covered the
        # current position
        for start, end, _, _ in self.allmatches:
            for i in range(start, end):
                parsed[i] = True

        for i in range(len(parsed)):
            c = self.s[i]
            # whitespace is always 'unparsed'
            if c.isspace():
                parsed[i] = True

            # the parser ignores comments but we can use a trick to see if this
            # starts a comment and is beyond the ending index of the parsed
            # portion of the inpnut
            if (not self.ast or i > self.ast.pos[1]) and c == '#':
                comment = matchresult(i, len(parsed), helpconstants.COMMENT, None)
                self.groups[0].results.append(comment)
                break

            if not parsed[i]:
                # add unparsed results to the 'shell' group
                self.groups[0].results.append(self.unknown(c, i, i+1))

        # there are no overlaps, so sorting by the start is enough
        self.groups[0].results.sort(key=lambda mr: mr.start)

    def _resultindex(self):
        '''return a mapping of matchresults to their index among all
        matches, sorted by the start position of the matchresult'''
        d = {}
        i = 0
        for result in sorted(self.allmatches, key=lambda mr: mr.start):
            d[result] = i
            i += 1
        return d

    def _mergeadjacent(self, matches):
        merged = []
        resultindex = self._resultindex()
        sametext = itertools.groupby(matches, lambda m: m.text)
        for text, ll in sametext:
            for l in util.groupcontinuous(ll, key=lambda m: resultindex[m]):
                if len(l) == 1:
                    merged.append(l[0])
                else:
                    start = l[0].start
                    end = l[-1].end
                    endindex = resultindex[l[-1]]
                    for mr in l:
                        del resultindex[mr]
                    merged.append(matchresult(start, end, text, None))
                    resultindex[merged[-1]] = endindex
        return merged



File: explainshell/util.py

import itertools
from operator import itemgetter

def consecutive(l, fn):
    '''yield consecutive items from l that fn returns True for them

    >>> even = lambda x: x % 2 == 0
    >>> list(consecutive([], even))
    []
    >>> list(consecutive([1], even))
    [[1]]
    >>> list(consecutive([1, 2], even))
    [[1], [2]]
    >>> list(consecutive([2, 4], even))
    [[2, 4]]
    >>> list(consecutive([1, 2, 4], even))
    [[1], [2, 4]]
    >>> list(consecutive([1, 2, 4, 5, 7, 8, 10], even))
    [[1], [2, 4], [5], [7], [8, 10]]
    '''
    it = iter(l)
    ll = []
    try:
        while True:
            x = it.next()
            if fn(x):
                ll.append(x)
            else:
                if ll:
                    yield ll
                    ll = []
                yield [x]
    except StopIteration:
        if ll:
            yield ll

def groupcontinuous(l, key=None):
    '''
    >>> list(groupcontinuous([1, 2, 4, 5, 7, 8, 10]))
    [[1, 2], [4, 5], [7, 8], [10]]
    >>> list(groupcontinuous(range(5)))
    [[0, 1, 2, 3, 4]]
    '''
    if key is None:
        key = lambda x: x
    for k, g in itertools.groupby(enumerate(l), lambda (i, x): i-key(x)):
        yield map(itemgetter(1), g)

def toposorted(graph, parents):
    """
    Returns vertices of a DAG in topological order.

    Arguments:
    graph -- vetices of a graph to be toposorted
    parents -- function (vertex) -> vertices to preceed
               given vertex in output
    """
    result = []
    used = set()
    def use(v, top):
        if id(v) in used:
            return
        for parent in parents(v):
            if parent is top:
                raise ValueError('graph is cyclical', graph)
            use(parent, v)
        used.add(id(v))
        result.append(v)
    for v in graph:
        use(v, v)
    return result

def pairwise(iterable):
    a, b = itertools.tee(iterable)
    next(b, None)
    return itertools.izip(a, b)

class peekable(object):
    '''
    >>> it = peekable(iter('abc'))
    >>> it.index, it.peek(), it.index, it.peek(), it.next(), it.index, it.peek(), it.next(), it.next(), it.index
    (0, 'a', 0, 'a', 'a', 1, 'b', 'b', 'c', 3)
    >>> it.peek()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration
    >>> it.peek()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration
    >>> it.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration
    '''
    def __init__(self, it):
        self.it = it
        self._peeked = False
        self._peekvalue = None
        self._idx = 0
    def __iter__(self):
        return self
    def next(self):
        if self._peeked:
            self._peeked = False
            self._idx += 1
            return self._peekvalue
        n = self.it.next()
        self._idx += 1
        return n
    def hasnext(self):
        try:
            self.peek()
            return True
        except StopIteration:
            return False
    def peek(self):
        if self._peeked:
            return self._peekvalue
        else:
            self._peekvalue = self.it.next()
            self._peeked = True
            return self._peekvalue
    @property
    def index(self):
        '''return the index of the next item returned by next()'''
        return self._idx

def namesection(path):
    assert '.gz' not in path
    name, section = path.rsplit('.', 1)
    return name, section

class propertycache(object):
    def __init__(self, func):
        self.func = func
        self.name = func.__name__

    def __get__(self, obj, type=None):
        result = self.func(obj)
        self.cachevalue(obj, result)
        return result

    def cachevalue(self, obj, value):
        setattr(obj, self.name, value)



File: explainshell/helpconstants.py

# -*- coding: utf-8 -*-

import textwrap

NOSYNOPSIS = 'no synopsis found'

PIPELINES = textwrap.dedent('''   <b>Pipelines</b>
       A  <u>pipeline</u> is a sequence of one or more commands separated by one of the control operators <b>|</b> or <b>|&amp;</b>.  The
       format for a pipeline is:

              [<b>time</b> [<b>-p</b>]] [ ! ] <u>command</u> [ [<b>|</b>⎪<b>|&amp;</b>] <u>command2</u> ... ]

       The standard output of <u>command</u> is connected  via  a  pipe  to  the  standard  input  of  <u>command2</u>.   This
       connection  is performed before any redirections specified by the command (see <b>REDIRECTION</b> below).  If <b>|&amp;</b>
       is used, the standard error of <u>command</u> is connected to <u>command2</u>'s standard input through the pipe; it  is
       shorthand  for  <b>2&gt;&amp;1</b>  <b>|</b>.   This  implicit  redirection  of  the  standard  error  is  performed after any
       redirections specified by the command.

       The return status of a pipeline is the exit status of the last command, unless  the  <b>pipefail</b>  option  is
       enabled.   If  <b>pipefail</b>  is  enabled,  the  pipeline's return status is the value of the last (rightmost)
       command to exit with a non-zero status, or zero if all commands exit successfully.  If the reserved  word
       <b>!</b>   precedes  a  pipeline, the exit status of that pipeline is the logical negation of the exit status as
       described above.  The shell waits for all commands in the pipeline to terminate before returning a value.

       If the <b>time</b> reserved word precedes a pipeline, the elapsed as well as user and system  time  consumed  by
       its execution are reported when the pipeline terminates.  The <b>-p</b> option changes the output format to that
       specified by POSIX.  When the shell is in <u>posix</u> <u>mode</u>, it does not recognize <b>time</b> as a  reserved  word  if
       the  next  token begins with a `-'.  The <b>TIMEFORMAT</b> variable may be set to a format string that specifies
       how the timing information should be displayed; see the description of <b>TIMEFORMAT</b> under  <b>Shell</b>  <b>Variables</b>
       below.

       When the shell is in <u>posix</u> <u>mode</u>, <b>time</b> may be followed by a newline.  In this case, the shell displays the
       total user and system time consumed by the shell and its children.  The <b>TIMEFORMAT</b> variable may  be  used
       to specify the format of the time information.

       Each command in a pipeline is executed as a separate process (i.e., in a subshell).''')

OPSEMICOLON = textwrap.dedent('''       Commands separated  by  a <b>;</b> are executed sequentially; the shell waits for each command to terminate in turn.  The
       return status is the exit status of the last command executed.''')

OPBACKGROUND = textwrap.dedent('''       If a command is terminated by the control operator <b>&amp;</b>, the shell executes the command in the <u>background</u> in
       a subshell.  The shell does not wait for the command to finish, and the return  status  is  0.''')

OPANDOR = textwrap.dedent('''       AND and OR lists are sequences of one of more pipelines separated by the <b>&amp;&amp;</b>  and  <b>||</b>  control  operators,
       respectively.  AND and OR lists are executed with left associativity.  An AND list has the form

              <u>command1</u> <b>&amp;&amp;</b> <u>command2</u>

       <u>command2</u> is executed if, and only if, <u>command1</u> returns an exit status of zero.

       An OR list has the form

              <u>command1</u> <b>||</b> <u>command2</u>

       <u>command2</u>  is  executed  if and only if <u>command1</u> returns a non-zero exit status.  The return status of AND
       and OR lists is the exit status of the last command executed in the list.''')

OPERATORS = {';' : OPSEMICOLON, '&' : OPBACKGROUND, '&&' : OPANDOR, '||' : OPANDOR}

REDIRECTION = textwrap.dedent('''       Before a command is executed, its input and output may be <u>redirected</u> using a special notation interpreted
       by  the  shell.   Redirection  may  also  be used to open and close files for the current shell execution
       environment.  The following redirection operators may precede or appear anywhere within a <u>simple</u>  <u>command</u>
       or may follow a <u>command</u>.  Redirections are processed in the order they appear, from left to right.''')

REDIRECTING_INPUT = textwrap.dedent('''   <b>Redirecting</b> <b>Input</b>
       Redirection  of  input  causes  the  file  whose name results from the expansion of <u>word</u> to be opened for
       reading on file descriptor <u>n</u>, or the standard input (file descriptor 0) if <u>n</u> is not specified.

       The general format for redirecting input is:

              [<u>n</u>]<b>&lt;</b><u>word</u>''')

REDIRECTING_OUTPUT = textwrap.dedent('''   <b>Redirecting</b> <b>Output</b>
       Redirection of output causes the file whose name results from the expansion of  <u>word</u>  to  be  opened  for
       writing  on  file descriptor <u>n</u>, or the standard output (file descriptor 1) if <u>n</u> is not specified.  If the
       file does not exist it is created; if it does exist it is truncated to zero size.

       The general format for redirecting output is:

              [<u>n</u>]<b>&gt;</b><u>word</u>

       If the redirection operator is <b>&gt;</b>, and the <b>noclobber</b> option to the  <b>set</b>  builtin  has  been  enabled,  the
       redirection  will  fail if the file whose name results from the expansion of <u>word</u> exists and is a regular
       file.  If the redirection operator is <b>&gt;|</b>, or the redirection operator is <b>&gt;</b> and the  <b>noclobber</b>  option  to
       the  <b>set</b>  builtin  command  is  not  enabled, the redirection is attempted even if the file named by <u>word</u>
       exists.''')

APPENDING_REDIRECTED_OUTPUT = textwrap.dedent('''   <b>Appending</b> <b>Redirected</b> <b>Output</b>
       Redirection of output in this fashion causes the file whose name results from the expansion of <u>word</u> to be
       opened  for  appending  on  file  descriptor  <u>n</u>,  or  the standard output (file descriptor 1) if <u>n</u> is not
       specified.  If the file does not exist it is created.

       The general format for appending output is:

              [<u>n</u>]<b>&gt;&gt;</b><u>word</u>''')

REDIRECTING_OUTPUT_ERROR = textwrap.dedent('''   <b>Redirecting</b> <b>Standard</b> <b>Output</b> <b>and</b> <b>Standard</b> <b>Error</b>
       This construct allows both the standard output (file descriptor 1) and the standard  error  output  (file
       descriptor 2) to be redirected to the file whose name is the expansion of <u>word</u>.

       There are two formats for redirecting standard output and standard error:

              <b>&amp;&gt;</b><u>word</u>
       and
              <b>&gt;&amp;</b><u>word</u>

       Of the two forms, the first is preferred.  This is semantically equivalent to

              <b>&gt;</b><u>word</u> 2<b>&gt;&amp;</b>1''')

APPENDING_OUTPUT_ERROR = textwrap.dedent('''   <b>Appending</b> <b>Standard</b> <b>Output</b> <b>and</b> <b>Standard</b> <b>Error</b>
       This  construct  allows  both the standard output (file descriptor 1) and the standard error output (file
       descriptor 2) to be appended to the file whose name is the expansion of <u>word</u>.

       The format for appending standard output and standard error is:

              <b>&amp;&gt;&gt;</b><u>word</u>

       This is semantically equivalent to

              <b>&gt;&gt;</b><u>word</u> 2<b>&gt;&amp;</b>1''')

HERE_DOCUMENTS = textwrap.dedent('''   <b>Here</b> <b>Documents</b>
       This type of redirection instructs the shell  to  read  input  from  the  current  source  until  a  line
       containing  only <u>delimiter</u> (with no trailing blanks) is seen.  All of the lines read up to that point are
       then used as the standard input for a command.

       The format of here-documents is:

              <b>&lt;&lt;</b>[<b>-</b>]<u>word</u>
                      <u>here-document</u>
              <u>delimiter</u>

       No parameter expansion, command substitution, arithmetic expansion, or pathname expansion is performed on
       <u>word</u>.   If  any  characters in <u>word</u> are quoted, the <u>delimiter</u> is the result of quote removal on <u>word</u>, and
       the lines in the here-document are not expanded.  If <u>word</u> is unquoted, all lines of the here-document are
       subjected  to  parameter  expansion, command substitution, and arithmetic expansion.  In the latter case,
       the character sequence <b>\&lt;newline&gt;</b> is ignored, and <b>\</b> must be used to quote the characters <b>\</b>, <b>$</b>, and <b>`</b>.

       If the redirection operator is <b>&lt;&lt;-</b>, then all leading tab characters are stripped from input lines and the
       line  containing  <u>delimiter</u>.  This allows here-documents within shell scripts to be indented in a natural
       fashion.

   <b>Here</b> <b>Strings</b>
       A variant of here documents, the format is:

              <b>&lt;&lt;&lt;</b><u>word</u>

       The <u>word</u> is expanded and supplied to the command on its standard input.''')

REDIRECTION_KIND = {'<' : REDIRECTING_INPUT,
                   '>' : REDIRECTING_OUTPUT,
                   '>>' : APPENDING_REDIRECTED_OUTPUT,
                   '&>' : REDIRECTING_OUTPUT_ERROR,
                   '>&' : REDIRECTING_OUTPUT_ERROR,
                   '&>>' : APPENDING_OUTPUT_ERROR,
                   '<<' : HERE_DOCUMENTS,
                   '<<<' : HERE_DOCUMENTS}

ASSIGNMENT = textwrap.dedent('''       A <u>variable</u> may be assigned to by a statement of the form

              <u>name</u>=[<u>value</u>]

       If <u>value</u> is not given, the variable is assigned the null string.  All  <u>values</u>  undergo  tilde  expansion,
       parameter  and  variable  expansion,  command  substitution, arithmetic expansion, and quote removal (see
       <b>EXPANSION</b> below).  If the variable has  its  <b>integer</b>  attribute  set,  then  <u>value</u>  is  evaluated  as  an
       arithmetic  expression even if the $((...)) expansion is not used (see <b>Arithmetic</b> <b>Expansion</b> below).  Word
       splitting is not performed, with the exception of <b>"$@"</b>  as  explained  below  under  <b>Special</b>  <b>Parameters</b>.
       Pathname  expansion  is  not performed.  Assignment statements may also appear as arguments to the <b>alias</b>,
       <b>declare</b>, <b>typeset</b>, <b>export</b>, <b>readonly</b>, and <b>local</b> builtin commands.

       In the context where an assignment statement is assigning a value to a shell variable or array index, the
       +=  operator  can  be used to append to or add to the variable's previous value.  When += is applied to a
       variable for which the <u>integer</u> attribute has been set, <u>value</u> is evaluated as an arithmetic expression and
       added  to the variable's current value, which is also evaluated.  When += is applied to an array variable
       using compound assignment (see <b>Arrays</b> below), the variable's value is not unset (as it is when using  =),
       and  new  values  are  appended to the array beginning at one greater than the array's maximum index (for
       indexed arrays) or added as additional key-value pairs in  an  associative  array.   When  applied  to  a
       string-valued variable, <u>value</u> is expanded and appended to the variable's value.''')

_group = textwrap.dedent('''       { <u>list</u>; }
              <u>list</u> is simply executed in the current shell environment.  <u>list</u> must be terminated with a  newline
              or  semicolon.   This  is known as a <u>group</u> <u>command</u>.  The return status is the exit status of <u>list</u>.
              Note that unlike the metacharacters <b>(</b> and <b>)</b>, <b>{</b> and <b>}</b> are <u>reserved</u> <u>words</u> and  must  occur  where  a
              reserved  word  is permitted to be recognized.  Since they do not cause a word break, they must be
              separated from <u>list</u> by whitespace or another shell metacharacter.''')

_subshell = textwrap.dedent('''       (<u>list</u>) <u>list</u> is executed in a subshell environment (see <b>COMMAND</b> <b>EXECUTION</b>  <b>ENVIRONMENT</b>  below).   Variable
              assignments and builtin commands that affect the shell's environment do not remain in effect after
              the command completes.  The return status is the exit status of <u>list</u>.''')

_negate = '''If the reserved word <b>!</b> precedes a pipeline, the exit status of that pipeline is the logical negation of the
exit status as described above.'''

_if = textwrap.dedent('''       <b>if</b> <u>list</u>; <b>then</b> <u>list;</u> [ <b>elif</b> <u>list</u>; <b>then</b> <u>list</u>; ] ... [ <b>else</b> <u>list</u>; ] <b>fi</b>
              The  <b>if</b> <u>list</u> is executed.  If its exit status is zero, the <b>then</b> <u>list</u> is executed.  Otherwise, each
              <b>elif</b> <u>list</u> is executed in turn, and if its exit status is zero,  the  corresponding  <b>then</b>  <u>list</u>  is
              executed  and  the command completes.  Otherwise, the <b>else</b> <u>list</u> is executed, if present.  The exit
              status is the exit status of the last command executed, or zero if no condition tested true.''')

_for = textwrap.dedent('''       <b>for</b> <u>name</u> [ [ <b>in</b> [ <u>word</u> <u>...</u> ] ] ; ] <b>do</b> <u>list</u> ; <b>done</b>
              The  list of words following <b>in</b> is expanded, generating a list of items.  The variable <u>name</u> is set
              to each element of this list in turn, and <u>list</u> is executed each time.  If the <b>in</b> <u>word</u> is  omitted,
              the  <b>for</b>  command  executes  <u>list</u>  once  for each positional parameter that is set (see <b>PARAMETERS</b>
              below).  The return status is the exit status of the last command that executes.  If the expansion
              of  the  items  following  <b>in</b>  results  in an empty list, no commands are executed, and the return
              status is 0.''')

_whileuntil = textwrap.dedent('''       <b>while</b> <u>list-1</u>; <b>do</b> <u>list-2</u>; <b>done</b>
       <b>until</b> <u>list-1</u>; <b>do</b> <u>list-2</u>; <b>done</b>
              The <b>while</b> command continuously executes the list <u>list-2</u> as long as the last command  in  the  list
              <u>list-1</u>  returns  an  exit  status  of  zero.  The <b>until</b> command is identical to the <b>while</b> command,
              except that the test is negated; <u>list-2</u> is executed as long as the last command in <u>list-1</u>  returns
              a non-zero exit status.  The exit status of the <b>while</b> and <b>until</b> commands is the exit status of the
              last command executed in <u>list-2</u>, or zero if none was executed.''')

_select = textwrap.dedent('''       <b>select</b> <u>name</u> [ <b>in</b> <u>word</u> ] ; <b>do</b> <u>list</u> ; <b>done</b>
              The list of words following <b>in</b> is expanded, generating a list of items.  The set of expanded words
              is printed on the standard error, each preceded by a number.  If  the  <b>in</b>  <u>word</u>  is  omitted,  the
              positional  parameters are printed (see <b>PARAMETERS</b> below).  The <b>PS3</b> prompt is then displayed and a
              line read from the standard input.  If the line consists of a number corresponding to one  of  the
              displayed  words, then the value of <u>name</u> is set to that word.  If the line is empty, the words and
              prompt are displayed again.  If EOF is read, the command completes.  Any other value  read  causes
              <u>name</u> to be set to null.  The line read is saved in the variable <b>REPLY</b>.  The <u>list</u> is executed after
              each selection until a <b>break</b> command is executed.  The exit status of <b>select</b> is the exit status of
              the last command executed in <u>list</u>, or zero if no commands were executed.''')

RESERVEDWORDS = {
    '!' : _negate,
    '{' : _group,
    '}' : _group,
    '(' : _subshell,
    ')' : _subshell,
    ';' : OPSEMICOLON,
}

def _addwords(key, text, *words):
    for word in words:
        COMPOUNDRESERVEDWORDS.setdefault(key, {})[word] = text

COMPOUNDRESERVEDWORDS = {}
_addwords('if', _if, 'if', 'then', 'elif', 'else', 'fi', ';')
_addwords('for', _for, 'for', 'in', 'do', 'done', ';')
_addwords('while', _whileuntil, 'while', 'do', 'done', ';')
_addwords('until', _whileuntil, 'until', 'do', 'done')
_addwords('select', _select, 'select', 'in', 'do', 'done')

_function = textwrap.dedent('''       A shell function is an object that is called like a simple command and executes a compound command with a
       new set of positional parameters.  Shell functions are declared as follows:

       <u>name</u> () <u>compound-command</u> [<u>redirection</u>]
       <b>function</b> <u>name</u> [()] <u>compound-command</u> [<u>redirection</u>]
              This  defines  a  function  named  <u>name</u>.  The reserved word <b>function</b> is optional.  If the <b>function</b>
              reserved word is supplied, the parentheses are optional.  The <u>body</u> of the function is the compound
              command  <u>compound-command</u>  (see  <b>Compound</b>  <b>Commands</b>  above).   That  command  is usually a <u>list</u> of
              commands between {  and  },  but  may  be  any  command  listed  under  <b>Compound</b>  <b>Commands</b>  above.
              <u>compound-command</u>  is  executed  whenever  <u>name</u>  is specified as the name of a simple command.  Any
              redirections (see <b>REDIRECTION</b> below) specified when a function is defined are performed  when  the
              function  is  executed.   The  exit  status of a function definition is zero unless a syntax error
              occurs or a readonly function with the same name already exists.  When executed, the  exit  status
              of a function is the exit status of the last command executed in the body.  (See <b>FUNCTIONS</b> below.)''')

_functioncall = 'call shell function %r'
_functionarg = 'argument for shell function %r'

COMMENT = textwrap.dedent('''<b>COMMENTS</b>
      In a non-interactive shell, or an interactive shell in which the <b>interactive_comments</b> option to the <b>shopt</b>
      builtin is enabled (see <b>SHELL</b> <b>BUILTIN</b> <b>COMMANDS</b> below), a word beginning with <b>#</b> causes that word  and  all
      remaining  characters  on that line to be ignored.  An interactive shell without the <b>interactive_comments</b>
      option enabled does not allow comments.  The <b>interactive_comments</b> option is on by default in  interactive
      shells.''')

parameters = {
        '*' : 'star',
        '@' : 'at',
        '#' : 'pound',
        '?' : 'question',
        '-' : 'hyphen',
        '$' : 'dollar',
        '!' : 'exclamation',
        '0' : 'zero',
        '_' : 'underscore',
}



File: explainshell/__init__.py




File: explainshell/errors.py

class ProgramDoesNotExist(Exception):
    pass

class EmptyManpage(Exception):
    pass



File: explainshell/manager.py

import sys, os, argparse, logging, glob

from explainshell import options, store, fixer, manpage, errors, util, config
from explainshell.algo import classifier

logger = logging.getLogger('explainshell.manager')

class managerctx(object):
    def __init__(self, classifier, store, manpage):
        self.classifier = classifier
        self.store = store
        self.manpage = manpage
        self.name = manpage.name

        self.classifiermanpage = None
        self.optionsraw = None
        self.optionsextracted = None
        self.aliases = None

class manager(object):
    '''the manager uses all parts of the system to read, classify, parse, extract
    and write a man page to the database'''
    def __init__(self, dbhost, dbname, paths, overwrite=False, drop=False):
        self.paths = paths
        self.overwrite = overwrite

        self.store = store.store(dbname, dbhost)

        self.classifier = classifier.classifier(self.store, 'bayes')
        self.classifier.train()

        if drop:
            self.store.drop(True)

    def ctx(self, m):
        return managerctx(self.classifier, self.store, m)

    def _read(self, ctx, frunner):
        frunner.pre_get_raw_manpage()
        ctx.manpage.read()
        ctx.manpage.parse()
        assert len(ctx.manpage.paragraphs) > 1

        ctx.manpage = store.manpage(ctx.manpage.shortpath, ctx.manpage.name,
                ctx.manpage.synopsis, ctx.manpage.paragraphs, list(ctx.manpage.aliases))
        frunner.post_parse_manpage()

    def _classify(self, ctx, frunner):
        ctx.classifiermanpage = store.classifiermanpage(ctx.name, ctx.manpage.paragraphs)
        frunner.pre_classify()
        _ = list(ctx.classifier.classify(ctx.classifiermanpage))
        frunner.post_classify()

    def _extract(self, ctx, frunner):
        options.extract(ctx.manpage)
        frunner.post_option_extraction()
        if not ctx.manpage.options:
            logger.warn("couldn't find any options for manpage %s", ctx.manpage.name)

    def _write(self, ctx, frunner):
        frunner.pre_add_manpage()
        return ctx.store.addmanpage(ctx.manpage)

    def _update(self, ctx, frunner):
        frunner.pre_add_manpage()
        return ctx.store.updatemanpage(ctx.manpage)

    def process(self, ctx):
        frunner = fixer.runner(ctx)

        self._read(ctx, frunner)
        self._classify(ctx, frunner)
        self._extract(ctx, frunner)

        m = self._write(ctx, frunner)
        return m

    def edit(self, m, paragraphs=None):
        ctx = self.ctx(m)
        frunner = fixer.runner(ctx)

        if paragraphs:
            m.paragraphs = paragraphs
            frunner.disable('paragraphjoiner')
            frunner.post_option_extraction()
        else:
            self._extract(ctx, frunner)
        m = self._update(ctx, frunner)
        return m

    def run(self):
        added = []
        exists = []
        for path in self.paths:
            try:
                m = manpage.manpage(path)
                logger.info('handling manpage %s (from %s)', m.name, path)
                try:
                    mps = self.store.findmanpage(m.shortpath[:-3])
                    mps = [mp for mp in mps if m.shortpath == mp.source]
                    if mps:
                        assert len(mps) == 1
                        mp = mps[0]
                        if not self.overwrite or mp.updated:
                            logger.info('manpage %r already in the data store, not overwriting it', m.name)
                            exists.append(m)
                            continue
                except errors.ProgramDoesNotExist:
                    pass

                # the manpage is not in the data store; process and add it
                ctx = self.ctx(m)
                m = self.process(ctx)
                if m:
                    added.append(m)
            except errors.EmptyManpage, e:
                logger.error('manpage %r is empty!', e.args[0])
            except ValueError:
                logger.fatal('uncaught exception when handling manpage %s', path)
            except KeyboardInterrupt:
                raise
            except:
                logger.fatal('uncaught exception when handling manpage %s', path)
                raise
        if not added:
            logger.warn('no manpages added')
        else:
            self.findmulticommands()

        return added, exists

    def findmulticommands(self):
        manpages = {}
        potential = []
        for _id, m in self.store.names():
            if '-' in m:
                potential.append((m.split('-'), _id))
            else:
                manpages[m] = _id

        mappings = set([x[0] for x in self.store.mappings()])
        mappingstoadd = []
        multicommands = {}

        for p, _id in potential:
            if ' '.join(p) in mappings:
                continue
            if p[0] in manpages:
                mappingstoadd.append((' '.join(p), _id))
                multicommands[p[0]] = manpages[p[0]]

        for src, dst in mappingstoadd:
            self.store.addmapping(src, dst, 1)
            logger.info('inserting mapping (multicommand) %s -> %s', src, dst)

        for multicommand, _id in multicommands.iteritems():
            self.store.setmulticommand(_id)
            logger.info('making %r a multicommand', multicommand)

        return mappingstoadd, multicommands

def main(files, dbname, dbhost, overwrite, drop, verify):
    if verify:
        s = store.store(dbname, dbhost)
        ok = s.verify()
        return 0 if ok else 1

    if drop:
        if raw_input('really drop db (y/n)? ').strip().lower() != 'y':
            drop = False
        else:
            overwrite = True # if we drop, no need to take overwrite into account

    gzs = set()

    for path in files:
        if os.path.isdir(path):
            gzs.update([os.path.abspath(f) for f in glob.glob(os.path.join(path, '*.gz'))])
        else:
            gzs.add(os.path.abspath(path))

    m = manager(dbhost, dbname, gzs, overwrite, drop)
    added, exists = m.run()
    for mp in added:
        print 'successfully added %s' % mp.source
    if exists:
        print 'these manpages already existed and werent overwritten: \n\n%s' % '\n'.join([m.path for m in exists])

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='process man pages and save them in the store')
    parser.add_argument('--log', type=str, default='ERROR', help='use log as the logger log level')
    parser.add_argument('--overwrite', action='store_true', default=False, help='overwrite man pages that already exist in the store')
    parser.add_argument('--drop', action='store_true', default=False, help='delete all existing man pages')
    parser.add_argument('--db', default='explainshell', help='mongo db name')
    parser.add_argument('--host', default=config.MONGO_URI, help='mongo host')
    parser.add_argument('--verify', action='store_true', default=False, help='verify db integrity')
    parser.add_argument('files', nargs='*')

    args = parser.parse_args()
    logging.basicConfig(level=getattr(logging, args.log.upper()))
    sys.exit(main(args.files, args.db, args.host, args.overwrite, args.drop, args.verify))



File: explainshell/web/__init__.py

from flask import Flask
app = Flask(__name__)

from explainshell.web import views
from explainshell import store, config

if config.DEBUG:
    from explainshell.web import debugviews

app.config.from_object(config)



File: explainshell/web/debugviews.py

import logging

from flask import render_template, request, abort, redirect, url_for, json

from explainshell import manager, config, store
from explainshell.web import app, helpers

logger = logging.getLogger(__name__)

@app.route('/debug')
def debug():
    s = store.store('explainshell', config.MONGO_URI)
    d = {'manpages' : []}
    for mp in s:
        synopsis = ''
        if mp.synopsis:
            synopsis = mp.synopsis[:20]
        dd = {'name' : mp.name, 'synopsis' : synopsis}
        l = []
        for o in mp.options:
            l.append(str(o))
        dd['options'] = ', '.join(l)
        d['manpages'].append(dd)
    d['manpages'].sort(key=lambda d: d['name'].lower())
    return render_template('debug.html', d=d)

def _convertvalue(value):
    if isinstance(value, list):
        return [s.strip() for s in value]
    elif value.lower() == 'true':
        return True
    elif value:
        return value.strip()
    return False

@app.route('/debug/tag/<source>', methods=['GET', 'POST'])
def tag(source):
    mngr = manager.manager(config.MONGO_URI, 'explainshell', [], False, False)
    s = mngr.store
    m = s.findmanpage(source)[0]
    assert m

    if 'paragraphs' in request.form:
        paragraphs = json.loads(request.form['paragraphs'])
        mparagraphs = []
        for d in paragraphs:
            idx = d['idx']
            text = d['text']
            section = d['section']
            short = [s.strip() for s in d['short']]
            long = [s.strip() for s in d['long']]
            expectsarg = _convertvalue(d['expectsarg'])
            nestedcommand = _convertvalue(d['nestedcommand'])
            if isinstance(nestedcommand, str):
                nestedcommand = [nestedcommand]
            elif nestedcommand is True:
                logger.error('nestedcommand %r must be a string or list', nestedcommand)
                abort(503)
            argument = d['argument']
            if not argument:
                argument = None
            p = store.paragraph(idx, text, section, d['is_option'])
            if d['is_option'] and (short or long or argument):
                p = store.option(p, short, long, expectsarg, argument, nestedcommand)
            mparagraphs.append(p)

        if request.form.get('nestedcommand', '').lower() == 'true':
            m.nestedcommand = True
        else:
            m.nestedcommand = False
        m = mngr.edit(m, mparagraphs)
        if m:
            return redirect(url_for('explain', cmd=m.name))
        else:
            abort(503)
    else:
        helpers.convertparagraphs(m)
        for p in m.paragraphs:
            if isinstance(p, store.option):
                if isinstance(p.expectsarg, list):
                    p.expectsarg = ', '.join(p.expectsarg)
                if isinstance(p.nestedcommand, list):
                    p.nestedcommand = ', '.join(p.nestedcommand)

        return render_template('tagger.html', m=m)



File: explainshell/web/helpers.py

from explainshell import util

def convertparagraphs(manpage):
    for p in manpage.paragraphs:
        p.text = p.text.decode('utf-8')
    return manpage

def suggestions(matches, command):
    '''enrich command matches with links to other man pages with the
    same name'''
    for m in matches:
        if 'name' in m and 'suggestions' in m:
            before = command[:m['start']]
            after = command[m['end']:]
            newsuggestions = []
            for othermp in sorted(m['suggestions'], key=lambda mp: mp.section):
                mid = '%s.%s' % (othermp.name, othermp.section)
                newsuggestions.append({'cmd' : ''.join([before, mid, after]),
                                  'text' : othermp.namesection})
            m['suggestions'] = newsuggestions



File: explainshell/web/views.py

import logging, itertools, urllib
import markupsafe

from flask import render_template, request, redirect

import bashlex.errors

from explainshell import matcher, errors, util, store, config
from explainshell.web import app, helpers

logger = logging.getLogger(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/about')
def about():
    return render_template('about.html')

@app.route('/explain')
def explain():
    if 'cmd' not in request.args or not request.args['cmd'].strip():
        return redirect('/')
    command = request.args['cmd'].strip()
    command = command[:1000] # trim commands longer than 1000 characters
    if '\n' in command:
        return render_template('errors/error.html', title='parsing error!',
                               message='no newlines please')

    s = store.store('explainshell', config.MONGO_URI)
    try:
        matches, helptext = explaincommand(command, s)
        return render_template('explain.html',
                               matches=matches,
                               helptext=helptext,
                               getargs=command)

    except errors.ProgramDoesNotExist, e:
        return render_template('errors/missingmanpage.html', title='missing man page', e=e)
    except bashlex.errors.ParsingError, e:
        logger.warn('%r parsing error: %s', command, e.message)
        return render_template('errors/parsingerror.html', title='parsing error!', e=e)
    except NotImplementedError, e:
        logger.warn('not implemented error trying to explain %r', command)
        msg = ("the parser doesn't support %r constructs in the command you tried. you may "
               "<a href='https://github.com/idank/explainshell/issues'>report a "
               "bug</a> to have this added, if one doesn't already exist.") % e.args[0]

        return render_template('errors/error.html', title='error!', message=msg)
    except:
        logger.error('uncaught exception trying to explain %r', command, exc_info=True)
        msg = 'something went wrong... this was logged and will be checked'
        return render_template('errors/error.html', title='error!', message=msg)

@app.route('/explain/<program>', defaults={'section' : None})
@app.route('/explain/<section>/<program>')
def explainold(section, program):
    logger.info('/explain section=%r program=%r', section, program)

    s = store.store('explainshell', config.MONGO_URI)
    if section is not None:
        program = '%s.%s' % (program, section)

    # keep links to old urls alive
    if 'args' in request.args:
        args = request.args['args']
        command = '%s %s' % (program, args)
        return redirect('/explain?cmd=%s' % urllib.quote_plus(command), 301)
    else:
        try:
            mp, suggestions = explainprogram(program, s)
            return render_template('options.html', mp=mp, suggestions=suggestions)
        except errors.ProgramDoesNotExist, e:
            return render_template('errors/missingmanpage.html', title='missing man page', e=e)

def explainprogram(program, store):
    mps = store.findmanpage(program)
    mp = mps.pop(0)
    program = mp.namesection

    synopsis = mp.synopsis
    if synopsis:
        synopsis = synopsis.decode('utf-8')

    mp = {'source' : mp.source[:-3],
          'section' : mp.section,
          'program' : program,
          'synopsis' : synopsis,
          'options' : [o.text.decode('utf-8') for o in mp.options]}

    suggestions = []
    for othermp in mps:
        d = {'text' : othermp.namesection,
             'link' : '%s/%s' % (othermp.section, othermp.name)}
        suggestions.append(d)
    logger.info('suggestions: %s', suggestions)
    return mp, suggestions

def _makematch(start, end, match, commandclass, helpclass):
    return {'match' : match, 'start' : start, 'end' : end, 'spaces' : '',
            'commandclass' : commandclass, 'helpclass' : helpclass}

def explaincommand(command, store):
    matcher_ = matcher.matcher(command, store)
    groups = matcher_.match()
    expansions = matcher_.expansions

    shellgroup = groups[0]
    commandgroups = groups[1:]
    matches = []

    # save a mapping between the help text to its assigned id,
    # we're going to reuse ids that have the same text
    texttoid = {}

    # remember where each assigned id has started in the source,
    # we're going to use it later on to sort the help text by start
    # position
    idstartpos = {}

    l = []
    for m in shellgroup.results:
        commandclass = shellgroup.name
        helpclass = 'help-%d' % len(texttoid)
        text = m.text
        if text:
            text = text.decode('utf-8')
            helpclass = texttoid.setdefault(text, helpclass)
        else:
            # unknowns in the shell group are possible when our parser left
            # an unparsed remainder, see matcher._markunparsedunknown
            commandclass += ' unknown'
            helpclass = ''
        if helpclass:
            idstartpos.setdefault(helpclass, m.start)

        d = _makematch(m.start, m.end, m.match, commandclass, helpclass)
        formatmatch(d, m, expansions)

        l.append(d)
    matches.append(l)

    for commandgroup in commandgroups:
        l = []
        for m in commandgroup.results:
            commandclass = commandgroup.name
            helpclass = 'help-%d' % len(texttoid)
            text = m.text
            if text:
                text = text.decode('utf-8')
                helpclass = texttoid.setdefault(text, helpclass)
            else:
                commandclass += ' unknown'
                helpclass = ''
            if helpclass:
                idstartpos.setdefault(helpclass, m.start)

            d = _makematch(m.start, m.end, m.match, commandclass, helpclass)
            formatmatch(d, m, expansions)

            l.append(d)

        d = l[0]
        d['commandclass'] += ' simplecommandstart'
        if commandgroup.manpage:
            d['name'] = commandgroup.manpage.name
            d['section'] = commandgroup.manpage.section
            if '.' not in d['match']:
                d['match'] = '%s(%s)' % (d['match'], d['section'])
            d['suggestions'] = commandgroup.suggestions
            d['source'] = commandgroup.manpage.source[:-5]
        matches.append(l)

    matches = list(itertools.chain.from_iterable(matches))
    helpers.suggestions(matches, command)

    # _checkoverlaps(matcher_.s, matches)
    matches.sort(key=lambda d: d['start'])

    it = util.peekable(iter(matches))
    while it.hasnext():
        m = it.next()
        spaces = 0
        if it.hasnext():
            spaces = it.peek()['start'] - m['end']
        m['spaces'] = ' ' * spaces

    helptext = sorted(texttoid.iteritems(), key=lambda (k, v): idstartpos[v])

    return matches, helptext

def formatmatch(d, m, expansions):
    '''populate the match field in d by escaping m.match and generating
    links to any command/process substitutions'''

    # save us some work later: do any expansions overlap
    # the current match?
    hassubsinmatch = False

    for start, end, kind in expansions:
        if m.start <= start and end <= m.end:
            hassubsinmatch = True
            break

    # if not, just escape the current match
    if not hassubsinmatch:
        d['match'] = markupsafe.escape(m.match)
        return

    # used in es.js
    d['commandclass'] += ' hasexpansion'

    # go over the expansions, wrapping them with a link; leave everything else
    # untouched
    expandedmatch = ''
    i = 0
    for start, end, kind in expansions:
        if start >= m.end:
            break
        relativestart = start - m.start
        relativeend = end - m.start

        if i < relativestart:
            for j in range(i, relativestart):
                if m.match[j].isspace():
                    expandedmatch += markupsafe.Markup('&nbsp;')
                else:
                    expandedmatch += markupsafe.escape(m.match[j])
            i = relativestart + 1
        if m.start <= start and end <= m.end:
            s = m.match[relativestart:relativeend]

            if kind == 'substitution':
                content = markupsafe.Markup(_substitutionmarkup(s))
            else:
                content = s

            expandedmatch += markupsafe.Markup(
                    '<span class="expansion-{0}">{1}</span>').format(kind, content)
            i = relativeend

    if i < len(m.match):
        expandedmatch += markupsafe.escape(m.match[i:])

    assert expandedmatch
    d['match'] = expandedmatch

def _substitutionmarkup(cmd):
    '''
    >>> _substitutionmarkup('foo')
    '<a href="/explain?cmd=foo" title="Zoom in to nested command">foo</a>'
    >>> _substitutionmarkup('cat <&3')
    '<a href="/explain?cmd=cat+%3C%263" title="Zoom in to nested command">cat <&3</a>'
    '''
    encoded = urllib.urlencode({'cmd': cmd})
    return ('<a href="/explain?{query}" title="Zoom in to nested command">{cmd}'
            '</a>').format(cmd=cmd, query=encoded)

def _checkoverlaps(s, matches):
    explained = [None]*len(s)
    for d in matches:
        for i in range(d['start'], d['end']):
            if explained[i]:
                raise RuntimeError("explained overlap for group %s at %d with %s" % (d, i, explained[i]))
            explained[i] = d



File: explainshell/algo/classifier.py

import itertools, collections, logging

import nltk
import nltk.metrics
import nltk.classify
import nltk.classify.maxent

from explainshell import algo, config

logger = logging.getLogger(__name__)

def get_features(paragraph):
    features = {}
    ptext = paragraph.cleantext()
    assert ptext

    features['starts_with_hyphen'] = algo.features.starts_with_hyphen(ptext)
    features['is_indented'] = algo.features.is_indented(ptext)
    features['par_length'] = algo.features.par_length(ptext)
    for w in ('=', '--', '[', '|', ','):
        features['first_line_contains_%s' % w] = algo.features.first_line_contains(ptext, w)
    features['first_line_length'] = algo.features.first_line_length(ptext)
    features['first_line_word_count'] = algo.features.first_line_word_count(ptext)
    features['is_good_section'] = algo.features.is_good_section(paragraph)
    features['word_count'] = algo.features.word_count(ptext)

    return features

class classifier(object):
    '''classify the paragraphs of a man page as having command line options
    or not'''
    def __init__(self, store, algo, **classifier_args):
        self.store = store
        self.algo = algo
        self.classifier_args = classifier_args
        self.classifier = None

    def train(self):
        if self.classifier:
            return

        manpages = self.store.trainingset()

        # flatten the manpages so we get a list of (manpage-name, paragraph)
        def flatten_manpages(manpage):
            l = []
            for para in manpage.paragraphs:
                l.append(para)
            return l
        paragraphs = itertools.chain(*[flatten_manpages(m) for m in manpages])
        training = list(paragraphs)

        negids = [p for p in training if not p.is_option]
        posids = [p for p in training if p.is_option]

        negfeats = [(get_features(p), False) for p in negids]
        posfeats = [(get_features(p), True) for p in posids]

        negcutoff = len(negfeats)*3/4
        poscutoff = len(posfeats)*3/4

        trainfeats = negfeats[:negcutoff] + posfeats[:poscutoff]
        self.testfeats = negfeats[negcutoff:] + posfeats[poscutoff:]

        logger.info('train on %d instances', len(trainfeats))

        if self.algo == 'maxent':
            c = nltk.classify.maxent.MaxentClassifier
        elif self.algo == 'bayes':
            c = nltk.classify.NaiveBayesClassifier
        else:
            raise ValueError('unknown classifier')

        self.classifier = c.train(trainfeats, **self.classifier_args)

    def evaluate(self):
        self.train()
        refsets = collections.defaultdict(set)
        testsets = collections.defaultdict(set)

        for i, (feats, label) in enumerate(self.testfeats):
            refsets[label].add(i)
            guess = self.classifier.prob_classify(feats)
            observed = guess.max()
            testsets[observed].add(i)
            #if label != observed:
            #    print 'label:', label, 'observed:', observed, feats

        print 'pos precision:', nltk.metrics.precision(refsets[True], testsets[True])
        print 'pos recall:', nltk.metrics.recall(refsets[True], testsets[True])
        print 'neg precision:', nltk.metrics.precision(refsets[False], testsets[False])
        print 'neg recall:', nltk.metrics.recall(refsets[False], testsets[False])

        print self.classifier.show_most_informative_features(10)

    def classify(self, manpage):
        self.train()
        for item in manpage.paragraphs:

            features = get_features(item)
            guess = self.classifier.prob_classify(features)
            option = guess.max()
            certainty = guess.prob(option)

            if option:
                if certainty < config.CLASSIFIER_CUTOFF:
                    pass
                else:
                    logger.info('classified %s (%f) as an option paragraph', item, certainty)
                    item.is_option = True
                    yield certainty, item



File: explainshell/algo/__init__.py

import explainshell.algo.features
import explainshell.util



File: explainshell/algo/features.py

import re

def extract_first_line(paragraph):
    '''
    >>> extract_first_line('a b  cd')
    'a b'
    >>> extract_first_line('a b cd')
    'a b cd'
    >>> extract_first_line('  a b cd')
    'a b cd'
    >>> extract_first_line('  a b   cd')
    'a b'
    '''
    lines = paragraph.splitlines()
    first = lines[0].strip()
    spaces = list(re.finditer(r'(\s+)', first))
    # handle options that have their description in the first line by trying
    # to treat it as two lines (looking at spaces between option and the rest
    # of the text)
    if spaces:
        longest = max(spaces, key=lambda m: m.span()[1] - m.span()[0])
        if longest and longest.start() > 1 and longest.end() - longest.start() > 1:
            first = first[:longest.start()]
    return first

def starts_with_hyphen(paragraph):
    return paragraph.lstrip()[0] == '-'

def is_indented(paragraph):
    return paragraph != paragraph.lstrip()

def par_length(paragraph):
    return round(len(paragraph.strip()), -1) / 2

def first_line_contains(paragraph, what):
    l = paragraph.splitlines()[0]
    return what in l

def first_line_length(paragraph):
    first = extract_first_line(paragraph)
    return round(len(first), -1) / 2

def first_line_word_count(paragraph):
    first = extract_first_line(paragraph)
    splitted = [s for s in first.split() if len(s) > 1]

    return round(len(splitted), -1)

def is_good_section(paragraph):
    if not paragraph.section:
        return False
    s = paragraph.section.lower()
    if 'options' in s:
        return True
    if s in ('description', 'function letters'):
        return True
    return False

def word_count(text):
    return round(len(re.findall(r'\w+', text)), -1)

def has_bold(html):
    return '<b>' in html



# [explainshell.com](http://www.explainshell.com) - match command-line arguments to their help text

explainshell is a tool (with a web interface) capable of parsing man pages, extracting options and
explaining a given command-line by matching each argument to the relevant help text in the man page.

## How?

explainshell is built from the following components:

1. man page reader which converts a given man page from raw format to html (manpage.py)
2. classifier which goes through every paragraph in the man page and classifies
   it as contains options or not (algo/classifier.py)
3. an options extractor that scans classified paragraphs and looks for options (options.py)
4. a storage backend that saves processed man pages to mongodb (store.py)
5. a matcher that walks the command's AST (parsed by [bashlex](https://github.com/idank/bashlex)) and contextually matches each node
   to the relevant help text (matcher.py)

When querying explainshell, it:

1. parses the query into an AST
2. visits interesting nodes in the AST, such as:
   - command nodes - these nodes represent a simple command
   - shell related nodes - these nodes represent something the shell
     interprets such as '|', '&&'
3. for every command node we check if we know how to explain the current program,
   and then go through the rest of the tokens, trying to match each one to the
   list of known options
4. returns a list of matches that are rendered with Flask

## Missing man pages

Right now explainshell.com contains the entire [archive of Ubuntu](http://manpages.ubuntu.com/). It's not
possible to directly add a missing man page to the live site (it might be in the future).

## Running explainshell locally

Setup a working environment that lets you run the web interface locally using docker:

```ShellSession

# download db dump
$ curl -L -o /tmp/dump.gz https://github.com/idank/explainshell/releases/download/db-dump/dump.gz

# Clone Repository
$ git clone https://github.com/idank/explainshell.git

# start containers, load man pages from dump
$ docker-compose build
$ docker-compose up

$ docker-compose exec -T db mongorestore --archive --gzip < /tmp/dump.gz

# run tests
$ docker-compose exec -T web make tests
..SSSSSSSSS.....................................................................
----------------------------------------------------------------------
Ran 80 tests in 0.041s

OK (SKIP=9)
# open http://localhost:5000 to view the ui
```

### Processing a man page

Use the manager to parse and save a gzipped man page in raw format:

```ShellSession
$ docker-compose exec -T web bash -c "PYTHONPATH=. python explainshell/manager.py --log info /usr/share/man/man1/echo.1.gz"
INFO:explainshell.store:creating store, db = 'explainshell_tests', host = 'mongodb://localhost'
INFO:explainshell.algo.classifier:train on 994 instances
INFO:explainshell.manager:handling manpage echo (from /tmp/es/manpages/1/echo.1.gz)
INFO:explainshell.store:looking up manpage in mapping with src 'echo'
INFO:explainshell.manpage:executing '/tmp/es/tools/w3mman2html.cgi local=%2Ftmp%2Fes%2Fmanpages%2F1%2Fecho.1.gz'
INFO:explainshell.algo.classifier:classified <paragraph 3, DESCRIPTION: '-n     do not output the trailing newlin'> (0.991381) as an option paragraph
INFO:explainshell.algo.classifier:classified <paragraph 4, DESCRIPTION: '-e     enable interpretation of backslash escape'> (0.996904) as an option paragraph
INFO:explainshell.algo.classifier:classified <paragraph 5, DESCRIPTION: '-E     disable interpretation of backslash escapes (default'> (0.998640) as an option paragraph
INFO:explainshell.algo.classifier:classified <paragraph 6, DESCRIPTION: '--help display this help and exi'> (0.999215) as an option paragraph
INFO:explainshell.algo.classifier:classified <paragraph 7, DESCRIPTION: '--version'> (0.999993) as an option paragraph
INFO:explainshell.store:inserting mapping (alias) echo -> echo (52207a1fa9b52e42fb59df36) with score 10
successfully added echo
```

Note that if you've setup using the docker instructions above, echo will already be in the database.
