<repo_name>bioShaun/MultiQC-oms<file_sep>/multiqc_oms/modules/expCorr/expCorr.py
#!/usr/bin/env python

""" MultiQC module to parse output from HISAT2 """

from __future__ import print_function
from collections import OrderedDict
import os
import logging
import pandas as pd


from multiqc import config
from multiqc.plots import heatmap
from multiqc.modules.base_module import BaseMultiqcModule

# Initialise the logger
log = logging.getLogger(__name__)


class MultiqcModule(BaseMultiqcModule):
    """ dupradar module. """

    def __init__(self):

        # Initialise the parent object
        super(MultiqcModule, self).__init__(name="Sample Correlation", anchor="expCorr",
                                            info=" is generated from normalised gene counts through "
                                            "<a href='https://bioconductor.org/packages/release/bioc/html/edgeR.html' target='_blank'>edgeR</a>.")

        # Find and load correlation matrix
        self.cor_data = dict()
        for n, f in enumerate(self.find_log_files('expCorr', filehandles=False, filecontents=False)):
            if n > 0:
                raise ValueError('more than one correlation matrix found.')
            self.cor_data['correlation'] = f

        if len(self.cor_data) == 0:
            raise UserWarning

        log.info("Found {} reports".format(len(self.cor_data)))

        # Write parsed report data to a file
        self.write_data_file(self.cor_data, 'multiqc_expCorr')
        heatmap_name, heatmap_val = self.parse_corr_matrix()
        # correlation heatmap
        self.cor_heatmap_plot(heatmap_name, heatmap_val)

    def parse_corr_matrix(self):
        """
        """
        f = self.cor_data['correlation']
        cor_file = os.path.join(f['root'], f['fn'])
        cor_df = pd.read_csv(cor_file, index_col=0)
        heatmap_name = list(cor_df.columns)
        heatmap_val = [list(cor_df.loc[i]) for i in cor_df.index]
        return heatmap_name, heatmap_val

    def cor_heatmap_plot(self, heatmap_name, heatmap_val):
        """ Make the HighCharts HTML to plot sample correlation heatmap. """

        # Split the data into SE and PE
        pconfig = {
            'title': 'Pearson correlation',
            'xlab': True,
        }
        self.add_section(
            description='Pearson correlation between log<sub>2</sub> normalised CPM values are calculated and clustered.',
            plot=heatmap.plot(heatmap_val, heatmap_name, pconfig=pconfig)
        )
<file_sep>/multiqc_oms/modules/expCorr/__init__.py
from __future__ import absolute_import

from .expCorr import MultiqcModule
<file_sep>/multiqc_oms/modules/diffNum/diff_bar.py
#!/usr/bin/env python

""" MultiQC submodule to parse output from diff number table"""

import os
from collections import OrderedDict
import logging
import pandas as pd

from multiqc.plots import bargraph

# Initialise the logger
log = logging.getLogger(__name__)


def parse_reports(self):
    """ Find diff number table and parse their data """

    # Set up vars
    self.diff_num = dict()

    # Go through files and parse data using regexes
    for n, f in enumerate(
            self.find_log_files(
                'diffNum/diff_bar', filehandles=False, filecontents=False)):
        if n > 0:
            raise ValueError('more than one diff number table found.')
        self.diff_num['table'] = f

    if len(self.diff_num) == 0:
        raise UserWarning

    self.write_data_file(self.diff_num, 'multiqc_diffNum_bar')
    diff_num_file = os.path.join(
        self.diff_num['table']['root'], self.diff_num['table']['fn'])
    diff_num_df = pd.read_csv(diff_num_file, index_col=0)
    plot_data = OrderedDict()
    for idx_i in diff_num_df.index:
        plot_data[idx_i] = {'up-regulated': diff_num_df.loc[idx_i].up,
                            'down-regulated': diff_num_df.loc[idx_i].down}

    pconfig = {
        'title': 'Differential Expressed Genes',
        'cpswitch_counts_label': 'Number of Genes',
    }

    self.add_section(
        description=('This plot shows the Differential expressed gene number'
                     ' for each compare.'),
        plot=bargraph.plot(plot_data, pconfig=pconfig)
    )

    # Return number of compares found
    return len(plot_data)
<file_sep>/multiqc_oms/modules/diffNum/diffNum.py
#!/usr/bin/env python

""" MultiQC module to parse output from HISAT2 """

from __future__ import print_function
from collections import OrderedDict
import os
import logging
import pandas as pd


from multiqc import config
from multiqc.plots import heatmap
from multiqc.modules.base_module import BaseMultiqcModule

# Initialise the logger
log = logging.getLogger(__name__)


class MultiqcModule(BaseMultiqcModule):
    """ Show Number of diff genes. """

    def __init__(self):

        # Initialise the parent object
        super(MultiqcModule, self).__init__(name="Differential analysis", anchor="diffNum",
                                            info=" is generated from normalised gene counts through "
                                            "<a href='https://bioconductor.org/packages/release/bioc/html/edgeR.html' target='_blank'>edgeR</a>.")

        n = dict()

        diff_num_sections = [
            'diff_bar',
            'diff_matrix',
        ]

        for sm in diff_num_sections:
            try:
                module = __import__(
                    'multiqc_oms.modules.diffNum.{}'.format(sm), fromlist=[''])
                n[sm] = getattr(module, 'parse_reports')(self)
                if n[sm] > 0:
                    log.info("Found {} {} reports".format(n[sm], sm))
            except (ImportError, AttributeError, UserWarning):
                log.warn("Could not find diffNum Section '{}'".format(sm))

        # Exit if we didn't find anything
        if sum(n.values()) == 0:
            raise UserWarning
<file_sep>/setup.py
#!/usr/bin/env python
"""
MultiQC-oms is a plugin for MultiQC, providing additional tools which are
specific to OMS.

"""

from setuptools import setup, find_packages

version = '0.6'

setup(
    name='multiqc_oms',
    version=version,
    author='<NAME>',
    author_email='<EMAIL>',
    description="MultiQC plugin for the OMS",
    long_description=__doc__,
    keywords='bioinformatics',
    license='MIT',
    packages=find_packages(),
    include_package_data=True,
    install_requires=[
        'couchdb',
        'simplejson',
        'pyyaml',
        'requests',
        'multiqc'
    ],
    entry_points={
        'multiqc.modules.v1': [
            'dupradar = multiqc_oms.modules.dupradar:MultiqcModule',
            'expCorr = multiqc_oms.modules.expCorr:MultiqcModule',
            'stringtie = multiqc_oms.modules.stringtie:MultiqcModule',
            'diffNum = multiqc_oms.modules.diffNum:MultiqcModule',
            'enrichNum = multiqc_oms.modules.enrichNum:MultiqcModule',
        ],
        'multiqc.templates.v1': [
            'oms = multiqc_oms.templates.oms',
        ],
        'multiqc.hooks.v1': [
            'before_config = multiqc_oms.multiqc_oms:load_config',
        ],
    },
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Environment :: Web Environment',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: MIT License',
        'Natural Language :: English',
        'Operating System :: MacOS :: MacOS X',
        'Operating System :: POSIX',
        'Operating System :: Unix',
        'Programming Language :: Python',
        'Programming Language :: JavaScript',
        'Topic :: Scientific/Engineering',
        'Topic :: Scientific/Engineering :: Bio-Informatics',
        'Topic :: Scientific/Engineering :: Visualization',
    ],
)
<file_sep>/multiqc_oms/modules/stringtie/__init__.py
from __future__ import absolute_import

from .stringtie import MultiqcModule
<file_sep>/multiqc_oms/templates/oms/__init__.py
#!/usr/bin/env python

"""
=============
oms
=============

This theme is a stand-alone report themed for use by the OMS

"""
import os

template_parent = 'default'

template_dir = os.path.dirname(__file__)
base_fn = 'base.html'
<file_sep>/multiqc_oms/modules/diffNum/__init__.py
from __future__ import absolute_import

from .diffNum import MultiqcModule
<file_sep>/multiqc_oms/modules/dupradar/dupradar.py
#!/usr/bin/env python

""" MultiQC module to parse output from HISAT2 """

from __future__ import print_function
from collections import OrderedDict
import logging
import re
import json

from multiqc import config
from multiqc.plots import linegraph
from multiqc.modules.base_module import BaseMultiqcModule

# Initialise the logger
log = logging.getLogger(__name__)


class MultiqcModule(BaseMultiqcModule):
    """ dupradar module. """

    def __init__(self):

        # Initialise the parent object
        super(MultiqcModule, self).__init__(name="DupRadar", anchor="dupradar",
                                            href='https://bioconductor.org/packages/release/bioc/html/dupRadar.html',
                                            info=" provides duplication rate quality control for RNA-Seq datasets. "
                                            "Highly expressed genes can be expected to have a lot of duplicate reads, "
                                            "but high numbers of duplicates at low read counts can indicate low library "
                                            "complexity with technical duplication. ")

        # Find and load any HISAT2 reports
        self.dupradar_data = dict()
        for f in self.find_log_files('dupradar', filehandles=True):
            self.parse_dupradar_logs(f)

        # Filter to strip out ignored sample names
        self.dupradar_data = self.ignore_samples(self.dupradar_data)

        if len(self.dupradar_data) == 0:
            raise UserWarning

        log.info("Found {} reports".format(len(self.dupradar_data)))

        # Write parsed report data to a file
        self.write_data_file(self.dupradar_data, 'multiqc_dupradar')

        # Basic Stats Table
        # Report table is immutable, so just updating it works
        # self.hisat2_general_stats_table()

        # Alignment Rate Plot
        self.dupradar_plot()

    def parse_dupradar_logs(self, f):
        """
        """

        s_name = f['s_name']
        parsed_data = OrderedDict()

        for l in f['f']:
            if l[0] != '#':
                reads_kbp, dup_rate = l.strip().split()
                parsed_data[float(reads_kbp)] = float(dup_rate)
        self.add_data_source(f, s_name)
        self.dupradar_data[s_name] = parsed_data

    def dupradar_plot(self):
        """ Make the HighCharts HTML to plot gene duplication distributions. """

        # Split the data into SE and PE
        pconfig = {
            'title': 'DupRadar General Linear Model',
            'xLog': True,
            'xlab': 'expression (reads/kbp)',
            'ylab': '% duplicate reads',
            'ymax': 100,
            'ymin': 0,
            'tt_label': '<b>{point.x:.1f} reads/kbp</b>: {point.y:,.2f}% duplicates',
            'xPlotLines': [
                {
                    'color': 'green',
                    'dashStyle': 'LongDash',
                    'value': 0.5,
                    'width': 1,
                    'label': {
                        'style': {'color': 'green'},
                        'text': '0.5 RPKM',
                        'verticalAlign': 'bottom',
                        'y': -65,
                    }
                },
                {
                    'color': 'red',
                    'dashStyle': 'LongDash',
                    'value': 1000,
                    'width': 1,
                    'label': {
                        'style': {'color': 'green'},
                        'text': '1 read/bp',
                        'verticalAlign': 'bottom',
                        'y': -65,
                    }
                },
            ],

        }
        self.add_section(
            description='This plot shows the general linear models - a summary of the gene duplication distributions.',
            plot=linegraph.plot(self.dupradar_data, pconfig)
        )
<file_sep>/multiqc_oms/modules/dupradar/__init__.py
from __future__ import absolute_import

from .dupradar import MultiqcModule
<file_sep>/multiqc_oms/modules/enrichNum/enrichNum.py
#!/usr/bin/env python

""" MultiQC module to parse output from HISAT2 """


from __future__ import print_function
import os
import json
import logging
from multiqc.modules.base_module import BaseMultiqcModule
from multiqc.plots import bargraph, linegraph

# Initialise the logger
log = logging.getLogger(__name__)


class MultiqcModule(BaseMultiqcModule):
    """ Show Number of diff genes. """

    def __init__(self):

        # Initialise the parent object
        super(MultiqcModule, self).__init__(name="Enrichment analysis", anchor="enrichNum",
                                            info=" is applied to differential expressed genes using "
                                            "<a href='https://bioconductor.org/packages/release/bioc/html/goseq.html' target='_blank'>goseq</a> "
                                            "and <a href='http://kobas.cbi.pku.edu.cn/' target='_blank'>KOBAS</a>."
                                            "Top 20 Enriched KEGG Pathways and GO terms were selected to show in the report.")

        self.kegg_dict = dict()
        for n, f in enumerate(
                self.find_log_files(
                    'enrichNum/kegg',
                    filecontents=False,
                    filehandles=False)):
            if n > 0:
                raise ValueError('more than one kegg summary found.')
            kegg_file = os.path.join(f['root'], f['fn'])
            self.kegg_dict = json.load(open(kegg_file))
            self.write_data_file({'kegg': f}, 'multiqc_enrichNum_kegg')

        if len(self.kegg_dict) == 0:
            raise UserWarning
        else:
            self.add_section(
                name='KEGG Pathway',
                anchor='enrichNum-kegg',
                plot=self.enrich_num_chart('enrichNum_kegg_plot',
                                           'Top Enriched KEGG Pathways',
                                           self.kegg_dict))

        self.go_dict = dict()
        for n, f in enumerate(
                self.find_log_files(
                    'enrichNum/go',
                    filecontents=False,
                    filehandles=False)):
            if n > 0:
                raise ValueError('more than one go summary found.')
            go_file = os.path.join(f['root'], f['fn'])
            self.go_dict = json.load(open(go_file))
            self.write_data_file({'go': f}, 'multiqc_enrichNum_go')

        if len(self.go_dict) == 0:
            raise UserWarning
        else:
            self.add_section(
                name='GO',
                anchor='enrichNum-go',
                plot=self.enrich_num_chart('enrichNum_go_plot',
                                           'Top Enriched GO Terms',
                                           self.go_dict))

    def enrich_num_chart(self, plot_id, plot_title, plot_data):

        # Config for the plot
        pconfig = {
            'id': plot_id,
            'title': plot_title,
            'ylab': '-log10(Corrected P-value)',
            'stacking': None,
            'cpswitch': False,
            'tt_percentages': False,
            'tt_decimals': 2,
        }
        return bargraph.plot(plot_data, pconfig=pconfig)
<file_sep>/multiqc_oms/modules/diffNum/diff_matrix.py
#!/usr/bin/env python

""" MultiQC submodule to parse output from diff number table"""

import os
from collections import OrderedDict
import logging
import pandas as pd

from multiqc.plots import heatmap

# Initialise the logger
log = logging.getLogger(__name__)


def parse_reports(self):
    """ Find diff number table and parse their data """

    # Set up vars
    self.diff_num = dict()

    # Go through files and parse data using regexes
    for n, f in enumerate(
            self.find_log_files(
                'diffNum/diff_matrix', filehandles=False, filecontents=False)):
        if n > 0:
            raise ValueError('more than one diff number table found.')
        self.diff_num['table'] = f

    if len(self.diff_num) == 0:
        raise UserWarning

    self.write_data_file(self.diff_num, 'multiqc_diffNum_matrix')
    diff_num_file = os.path.join(
        self.diff_num['table']['root'], self.diff_num['table']['fn'])
    diff_num_df = pd.read_csv(diff_num_file, index_col=0)
    heatmap_name = list(diff_num_df.columns)
    heatmap_val = [list(diff_num_df.loc[i]) for i in diff_num_df.index]

    pconfig = {
        'title': 'Differential Expressed Genes',
        'xlab': True,
    }

    self.add_section(
        description=('This plot shows the Differential expressed gene number'
                     ' for each compare.'),
        plot=heatmap.plot(heatmap_val, heatmap_name, pconfig=pconfig)
    )

    # Return number of compares found
    return len(heatmap_val)
<file_sep>/multiqc_oms/multiqc_oms.py
from multiqc.utils import config


def load_config():
    my_search_patterns = {
        'dupradar': {'fn': '*.markDups_duprateExpDensCurve.txt'},
        'expCorr': {'fn': 'log2CPM_sample_cor.csv'},
        'stringtie': {'fn': '*.tmap'},
        'diffNum/diff_bar': {'fn': 'diff.genes.compare.csv'},
        'diffNum/diff_matrix': {'fn': 'diff.genes.matrix.csv'},
        'enrichNum/kegg': {'fn': 'top.kegg.enrich.json'},
        'enrichNum/go': {'fn': 'top.go.enrich.json'},
    }
    config.update_dict(config.sp, my_search_patterns)
<file_sep>/multiqc_oms/modules/stringtie/stringtie.py
#!/usr/bin/env python

""" MultiQC module to parse output from HISAT2 """

from __future__ import print_function
from collections import OrderedDict
import os
import logging
import pandas as pd

from multiqc import config
from collections import Counter
from multiqc.plots import bargraph
from multiqc.modules.base_module import BaseMultiqcModule

# Initialise the logger
log = logging.getLogger(__name__)


class MultiqcModule(BaseMultiqcModule):
    """ stringtie assembly module. """

    def __init__(self):

        # Initialise the parent object
        super(MultiqcModule, self).__init__(name="StringTie", anchor="stringtie",
                                            href='https://ccb.jhu.edu/software/stringtie/',
                                            info=" is a fast and highly efficient assembler of RNA-Seq alignments into potential transcripts. ")

        # Find and load any HISAT2 reports
        self.stringtie_data = dict()
        for f in self.find_log_files('stringtie', filehandles=False, filecontents=False):
            self.parse_stringtie_logs(f)

        # Filter to strip out ignored sample names
        self.stringtie_data = self.ignore_samples(self.stringtie_data)

        if len(self.stringtie_data) == 0:
            raise UserWarning

        log.info("Found {} reports".format(len(self.stringtie_data)))

        # Write parsed report data to a file
        self.write_data_file(self.stringtie_data, 'multiqc_dupradar')

        # assembly Plot
        self.stringtie_plot()

    def parse_stringtie_logs(self, f):
        """
        """

        s_name = f['s_name']
        f_file = os.path.join(f['root'], f['fn'])
        f_df = pd.read_csv(f_file, sep='\t')
        self.add_data_source(f, s_name)
        self.stringtie_data[s_name] = dict(Counter(f_df.class_code))

    def stringtie_plot(self):
        """ Make the HighCharts HTML to plot stringtie assembly. """

        # Split the data into SE and PE
        pconfig = {
            'title': 'StringTie assemblies',
            'ylab': 'Transcripts number',
            'cpswitch_counts_label': 'Number of Transcripts',
        }
        self.add_section(
            description='This plot shows the different type of transcripts assembled for each sample.',
            plot=bargraph.plot(self.stringtie_data, pconfig=pconfig)
        )
<file_sep>/multiqc_oms/modules/enrichNum/__init__.py
from __future__ import absolute_import

from .enrichNum import MultiqcModule
