text:
from unittest import TestCase, main
from shutil import rmtree
from tempfile import mkdtemp
from os import listdir, remove
from types import GeneratorType
from os.path import join, isfile, abspath, dirname, splitext

from skbio import DNA

from biom import load_table
import logging
from skbio import Sequence

from deblur.workflow import (dereplicate_seqs,
                             remove_chimeras_denovo_from_seqs,
                             remove_artifacts_seqs,
                             create_otu_table,
                             get_files_for_table,
                             trim_seqs,
                             multiple_sequence_alignment,
                             launch_workflow,
                             split_sequence_file_on_sample_ids_to_files,
                             build_index_sortmerna,
                             start_log, sequence_generator,
                             sample_id_from_read_id,
                             remove_artifacts_from_biom_table,
                             _get_fastq_variant,
                             filter_minreads_samples_from_table,
                             fasta_from_biom)
from deblur.deblurring import get_default_error_profile

class workflowTests(TestCase):
    """ Test deblur pipeline and individual methods functionality """

    def setUp(self):
        """ Create working directory and two FASTA input
            files corresponding to two samples (s1 and s2).
            Each input file contains 120 sequences, of which
            100 are 16S amplicons (ART simulator), 10 are
            chimeric sequences (Grinder) and 10 are PhiX
            artifacts (ART). The 100 amplicon sequences
            intend to evenly represent a community of 10
            species.
        """
        # test output can be written to this directory
        self.working_dir = mkdtemp()

        # the data directory for the workflow test files
        self.test_data_dir = join(dirname(abspath(__file__)), 'data')
        self.seqs_fq_fp = join(self.test_data_dir, 'seqs.fq')
        self.seqs_fq_bad_fp = join(self.test_data_dir, 'seqs_bad.fq')
        self.seqs_fq_illumina13_fp = join(self.test_data_dir,
                                          'seqs_illumina1.3.fq')
        self.seqs_s1_fp = join(self.test_data_dir, 'seqs_s1.fasta')
        self.seqs_s2_fp = join(self.test_data_dir, 'seqs_s2.fasta')
        self.seqs_s3_fp = join(self.test_data_dir, 'seqs_s3.fasta')
        self.orig_s1_fp = join(self.test_data_dir, 'simset.s1.fasta')
        self.orig_s2_fp = join(self.test_data_dir, 'simset.s2.fasta')
        self.orig_s3_fp = join(self.test_data_dir, 'simset.s3.fasta')

        self.no_trim_res = join(self.test_data_dir, 'no_trim_results.fasta')
        self.files_to_remove = []

        logfilename = join(self.working_dir, "log.txt")
        start_log(level=logging.DEBUG, filename=logfilename)

    def tearDown(self):
        for f in self.files_to_remove:
            remove(f)
        rmtree(self.working_dir)

    def test_get_fastq_variant(self):
        exp = 'illumina1.8'
        obs = _get_fastq_variant(self.seqs_fq_fp)
        self.assertEqual(obs, exp)

        exp = 'illumina1.3'
        obs = _get_fastq_variant(self.seqs_fq_illumina13_fp)
        self.assertEqual(obs, exp)

        with self.assertRaises(ValueError):
            _get_fastq_variant(self.seqs_fq_bad_fp)

    def test_sequence_generator_fasta(self):
        exp_len = 135
        first_id = "s1_1001203-10"
        first_few_nucs = "TACGTAGGTGGCAAGCGTTA"

        obs = list(sequence_generator(self.seqs_s1_fp))
        self.assertEqual(len(obs), exp_len)
        self.assertEqual(obs[0][0], first_id)
        self.assertTrue(obs[0][1].startswith(first_few_nucs))

    def test_sequence_generator_fastq(self):
        exp_len = 2
        first_id = "foo"
        first_few_nucs = "GCgc"

        obs = list(sequence_generator(self.seqs_fq_fp))
        self.assertEqual(len(obs), exp_len)
        self.assertEqual(obs[0][0], first_id)
        self.assertTrue(obs[0][1].startswith(first_few_nucs))

    def test_sequence_generator_invalid_format(self):
        allres = []
        input_fp = join(self.test_data_dir, 'readme.txt')
        msg = "input file %s does not appear to be FASTA or FASTQ" % input_fp
        with self.assertWarns(UserWarning) as W:
            for res in sequence_generator(input_fp):
                allres.append(res)
        self.assertEqual(len(allres), 0)
        self.assertEqual(str(W.warning), msg)

    def test_trim_seqs_notrim(self):
        seqs = [("seq1", "tagggcaagactccatggtatga"),
                ("seq2", "cggaggcgagatgcgtggta"),
                ("seq3", "tactagcaagattcctggtaaagga"),
                ("seq4", "aggatgcgagatgcgtg"),
                ("seq5", "gagtgcgagatgcgtggtgagg"),
                ("seq6", "ggatgcgagatgcgtggtgatt"),
                ("seq7", "agggcgagattcctagtgga--")]
        obs = trim_seqs(seqs, -1, 0)
        self.assertEqual(list(obs), seqs)

    def test_trim_seqs_notrim_outofbounds(self):
        seqs = [("seq1", "tagggcaagactccatggtatga"),
                ("seq2", "cggaggcgagatgcgtggta"),
                ("seq3", "tactagcaagattcctggtaaagga"),
                ("seq4", "aggatgcgagatgcgtg"),
                ("seq5", "gagtgcgagatgcgtggtgagg"),
                ("seq6", "ggatgcgagatgcgtggtgatt"),
                ("seq7", "agggcgagattcctagtgga--")]
        with self.assertRaises(ValueError):
            list(trim_seqs(seqs, -2, 0))

    def test_trim_seqs(self):
        seqs = [("seq1", "tagggcaagactccatggtatga"),
                ("seq2", "cggaggcgagatgcgtggta"),
                ("seq3", "tactagcaagattcctggtaaagga"),
                ("seq4", "aggatgcgagatgcgtg"),
                ("seq5", "gagtgcgagatgcgtggtgagg"),
                ("seq6", "ggatgcgagatgcgtggtgatt"),
                ("seq7", "agggcgagattcctagtgga--")]
        obs = trim_seqs(seqs, 20, 0)

        self.assertTrue(isinstance(obs, GeneratorType))

        exp = [("seq1", "tagggcaagactccatggta"),
               ("seq2", "cggaggcgagatgcgtggta"),
               ("seq3", "tactagcaagattcctggta"),
               ("seq5", "gagtgcgagatgcgtggtga"),
               ("seq6", "ggatgcgagatgcgtggtga"),
               ("seq7", "agggcgagattcctagtgga")]
        self.assertEqual(list(obs), exp)

    def test_trim_seqs_left(self):
        seqs = [("seq1", "tagggcaagactccatggtatga"),
                ("seq2", "cggaggcgagatgcgtggta"),
                ("seq3", "tactagcaagattcctggtaaagga"),
                ("seq4", "aggatgcgagatgcgtg"),
                ("seq5", "gagtgcgagatgcgtggtgagg"),
                ("seq6", "ggatgcgagatgcgtggtgatt"),
                ("seq7", "agggcgagattcctagtgga--")]
        obs = trim_seqs(seqs, 20, 5)

        self.assertTrue(isinstance(obs, GeneratorType))

        exp = [("seq1", "caagactccatggta"),
               ("seq2", "gcgagatgcgtggta"),
               ("seq3", "gcaagattcctggta"),
               ("seq5", "cgagatgcgtggtga"),
               ("seq6", "cgagatgcgtggtga"),
               ("seq7", "gagattcctagtgga")]
        self.assertEqual(list(obs), exp)

    def test_dereplicate_seqs_remove_singletons(self):
        """ Test dereplicate_seqs() method functionality with
            removing singletons
        """
        seqs = [("seq1", "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
                ("seq2", "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
                ("seq3", "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
                ("seq4", "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCT"),
                ("seq5", "TACCAGCCCCTTAAGTGGTAGGGACGATTATTTGGCCTAAAGCGTCCG"),
                ("seq6", "CTGCAAGGCTAGGGGGCGGGAGAGGCGGGTGGTACTTGAGGGGAGAAT"),
                ("seq7", "CTGCAAGGCTAGGGGGCGGGAGAGGCGGGTGGTACTTGAGGGGAGAAT")]
        seqs_fp = join(self.working_dir, "seqs.fasta")
        with open(seqs_fp, 'w') as seqs_f:
            for seq in seqs:
                seqs_f.write(">%s\n%s\n" % seq)

        output_fp = join(self.working_dir, "seqs_derep.fasta")

        dereplicate_seqs(seqs_fp=seqs_fp,
                         output_fp=output_fp)
        self.assertTrue(isfile(output_fp))

        exp = [("seq1;size=3;",
                "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
               ("seq6;size=2;",
                "CTGCAAGGCTAGGGGGCGGGAGAGGCGGGTGGTACTTGAGGGGAGAAT")]

        act = [item for item in sequence_generator(output_fp)]

        self.assertEqual(act, exp)

    def test_dereplicate_seqs(self):
        """ Test dereplicate_seqs() method functionality,
            keep singletons
        """
        seqs = [("seq1", "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
                ("seq2", "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
                ("seq3", "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
                ("seq4", "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCT"),
                ("seq5", "TACCAGCCCCTTAAGTGGTAGGGACGATTATTTGGCCTAAAGCGTCCG"),
                ("seq6", "CTGCAAGGCTAGGGGGCGGGAGAGGCGGGTGGTACTTGAGGGGAGAAT"),
                ("seq7", "CTGCAAGGCTAGGGGGCGGGAGAGGCGGGTGGTACTTGAGGGGAGAAT")]
        seqs_fp = join(self.working_dir, "seqs.fasta")
        with open(seqs_fp, 'w') as seqs_f:
            for seq in seqs:
                seqs_f.write(">%s\n%s\n" % seq)

        output_fp = join(self.working_dir, "seqs_derep.fasta")

        dereplicate_seqs(seqs_fp=seqs_fp,
                         output_fp=output_fp,
                         min_size=1)
        self.assertTrue(isfile(output_fp))

        exp = [("seq1;size=3;",
                "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
               ("seq6;size=2;",
                "CTGCAAGGCTAGGGGGCGGGAGAGGCGGGTGGTACTTGAGGGGAGAAT"),
               ("seq4;size=1;",
                "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCT"),
               ("seq5;size=1;",
                "TACCAGCCCCTTAAGTGGTAGGGACGATTATTTGGCCTAAAGCGTCCG")]

        act = [item for item in sequence_generator(output_fp)]

        self.assertEqual(act, exp)

    def test_filter_minreads_samples_from_table(self):
        """ Test filter_minreads_samples_from_table() function
        for removal of samples with small number of reads
        using the s4 dataset biom table
        """
        input_biom_file = join(self.test_data_dir, 'final.s4.biom')
        table = load_table(input_biom_file)

        # test basic filtering with 0 reads does not remove ok sample
        new_table = filter_minreads_samples_from_table(table)
        self.assertEqual(new_table.shape[1], 1)

        # test basic filtering with enough reads removes the sample
        # and also inplace=False works
        new_table = filter_minreads_samples_from_table(table,
                                                       minreads=182,
                                                       inplace=False)
        self.assertEqual(new_table.shape[1], 0)
        self.assertEqual(table.shape[1], 1)

        # test basic filtering with enough reads removes the sample
        # and also inplace=True works
        filter_minreads_samples_from_table(table,
                                           minreads=200,
                                           inplace=True)
        self.assertEqual(table.shape[1], 0)

    def test_remove_artifacts_from_biom_table(self):
        """ Test remove_artifacts_from_biom_table() function for
        removing non 16s sequences from a biom table and matching
        fasta file. This test uses a pre-calculated biom table and
        fasta file and tests the output only 16s and only artifacts
        tables
        s4 dataset is similar to s2 but with two added non-16s
        sequences (which are not phix/adapter)
        """
        # create the positive reference databases
        pos_ref_fp = join(self.test_data_dir, '70_otus.fasta')
        pos_ref_db_fp = build_index_sortmerna(
            ref_fp=(pos_ref_fp,),
            working_dir=self.working_dir)

        # remove the artifacts from the s4 biom table
        input_biom_file = join(self.test_data_dir, 'final.s4.biom')
        input_fasta_file = join(self.test_data_dir, 'final.s4.seqs.fa')
        tmp_files = remove_artifacts_from_biom_table(input_biom_file,
                                                     input_fasta_file,
                                                     [pos_ref_fp],
                                                     self.working_dir,
                                                     pos_ref_db_fp)

        origfilename = join(self.test_data_dir, 'simset.s2.fasta')
        trim_length = 150
        orig_seqs = [item[1] for item in sequence_generator(origfilename)]
        orig_seqs = [item[:trim_length].upper() for item in orig_seqs]

        no_artifacts_table_name = join(self.working_dir, 'reference-hit.biom')
        no_artifacts_table = load_table(no_artifacts_table_name)
        obs_seqs = no_artifacts_table.ids(axis='observation')
        self.assertEqual(set(obs_seqs), set(orig_seqs))
        # test the fasta file
        no_artifacts_fasta_name = join(self.working_dir, 'reference-hit.seqs.fa')
        fasta_seqs = [item[1] for item in sequence_generator(no_artifacts_fasta_name)]
        self.assertEqual(set(fasta_seqs), set(orig_seqs))

        # test the non-hit output biom
        artifacts_table_name = join(self.working_dir, 'reference-non-hit.biom')
        artifacts_table = load_table(artifacts_table_name)
        obs_seqs = artifacts_table.ids(axis='observation')
        artifact_seqs = ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaatttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt',
                         'AACAATGGGGGCAAGCGTTAATCATAATGGCTTAAAGAATTCGTAGAATtatatatattatatatatatTAGAGTTAATAAATATTAATTAAAGAATTATAACAATGGGGGCAAGCGTTAATCATAATGGCTTAAAGAATTCGTAGAATT']
        artifact_seqs = [item[:trim_length].upper() for item in artifact_seqs]
        obs_seqs = [item[:trim_length].upper() for item in obs_seqs]
        self.assertEqual(set(obs_seqs), set(artifact_seqs))
        # test the fasta file
        artifacts_fasta_name = join(self.working_dir, 'reference-non-hit.seqs.fa')
        fasta_seqs = [item[1].upper() for item in sequence_generator(artifacts_fasta_name)]
        self.assertEqual(set(fasta_seqs), set(artifact_seqs))

        self.assertEqual(len(obs_seqs), 2)
        self.assertEqual(len(tmp_files), 2)

    def test_remove_artifacts_seqs(self):
        """ Test remove_artifacts_seqs() function for removing
            sequences not matching to a reference database
            using SortMeRNA. This test forces a new index
            construction for the reference sequences.
        """
        seqs = [("seq1", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCC"),
                ("seq2", "CCTAAAACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
                ("seq3", "TCGCTATTATTGAGCCTAAAACGTCCGTAGTCGGCTTTGTAAATCCC"),
                ("seq4", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCC"),
                ("seq5", "CTAAAACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAATAGGGTC"),
                ("seq6", "TTGAGCCTAAAACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAAT"),
                ("phix1", "TCTAAAGGTAAAAAACGTTCTGGCGCTCGCCCTGGTCGTCCGCAGCC"),
                ("phix2", "CTGGCGCTCGCCCTGGTCGTCCGCAGCCGTTGCGAGGTACTAAAGGC"),
                ("phix3", "GCGCATAAATTTGAGCAGATTTGTCGTCACAGGTTGCGCCGCCAAAA")]
        exp_seqs = ["seq1", "seq2", "seq3", "seq4", "seq5", "seq6"]
        seqs_fp = join(self.working_dir, "seqs.fasta")
        with open(seqs_fp, 'w') as seqs_f:
            for seq in seqs:
                seqs_f.write(">%s\n%s\n" % seq)
        ref = [("ref1", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTA"
                        "GTCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref2", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref3", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref4", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref5", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATAGGGT"),
               ("ref6", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT")]
        ref_fp = join(self.working_dir, "ref2.fasta")
        with open(ref_fp, 'w') as ref_f:
            for seq in ref:
                ref_f.write(">%s\n%s\n" % seq)
        self.files_to_remove.append(ref_fp)
        ref_db_fp = build_index_sortmerna(
            ref_fp=(ref_fp,),
            working_dir=self.working_dir)
        output_fp, num_seqs_left, tmp_files = remove_artifacts_seqs(seqs_fp=seqs_fp,
                                                                    ref_fp=(ref_fp,),
                                                                    working_dir=self.working_dir,
                                                                    ref_db_fp=ref_db_fp,
                                                                    negate=False,
                                                                    threads=1)
        obs_seqs = []
        for label, seq in sequence_generator(output_fp):
            obs_seqs.append(label)
        self.assertEqual(obs_seqs, exp_seqs)
        # validate it creates one tmp file
        self.assertEqual(len(tmp_files), 1)

    def test_remove_artifacts_seqs_index_prebuilt(self):
        """ Test remove_artifacts_seqs() function for removing
            sequences not matching to a reference database
            using SortMeRNA. This test passes a built index.
        """
        seqs = [("seq1", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCC"),
                ("seq2", "CCTAAAACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
                ("seq3", "TCGCTATTATTGAGCCTAAAACGTCCGTAGTCGGCTTTGTAAATCCC"),
                ("seq4", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCC"),
                ("seq5", "CTAAAACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAATAGGGTC"),
                ("seq6", "TTGAGCCTAAAACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAAT"),
                ("phix1", "TCTAAAGGTAAAAAACGTTCTGGCGCTCGCCCTGGTCGTCCGCAGCC"),
                ("phix2", "CTGGCGCTCGCCCTGGTCGTCCGCAGCCGTTGCGAGGTACTAAAGGC"),
                ("phix3", "GCGCATAAATTTGAGCAGATTTGTCGTCACAGGTTGCGCCGCCAAAA")]
        exp_seqs = ["seq1", "seq2", "seq3", "seq4", "seq5", "seq6"]
        seqs_fp = join(self.working_dir, "seqs.fasta")
        with open(seqs_fp, 'w') as seqs_f:
            for seq in seqs:
                seqs_f.write(">%s\n%s\n" % seq)
        ref = [("ref1", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTA"
                        "GTCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref2", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref3", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref4", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref5", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATAGGGT"),
               ("ref6", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT")]
        ref_fp = join(self.working_dir, "ref3.fasta")
        with open(ref_fp, 'w') as ref_f:
            for seq in ref:
                ref_f.write(">%s\n%s\n" % seq)
        self.files_to_remove.append(ref_fp)
        # build index
        sortmerna_db = build_index_sortmerna([ref_fp], self.working_dir)
        output_fp = join(self.working_dir, "seqs_filtered.fasta")
        output_fp, num_seqs_left, _ = remove_artifacts_seqs(seqs_fp=seqs_fp,
                                                            ref_fp=(ref_fp,),
                                                            working_dir=self.working_dir,
                                                            ref_db_fp=sortmerna_db,
                                                            negate=False,
                                                            threads=1)

        obs_seqs = []
        for label, seq in sequence_generator(output_fp):
            obs_seqs.append(label)
        self.assertEqual(obs_seqs, exp_seqs)

    def test_remove_artifacts_seqs_negate(self):
        """ Test remove_artifacts_seqs() function for removing
            sequences matching to a reference database
            using SortMeRNA.
        """
        seqs = [("seq1", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCC"),
                ("seq2", "CCTAAAACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
                ("seq3", "TCGCTATTATTGAGCCTAAAACGTCCGTAGTCGGCTTTGTAAATCCC"),
                ("seq4", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCC"),
                ("seq5", "CTAAAACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAATAGGGTC"),
                ("seq6", "TTGAGCCTAAAACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAAT"),
                ("phix1", "TCTAAAGGTAAAAAACGTTCTGGCGCTCGCCCTGGTCGTCCGCAGCC"),
                ("phix2", "CTGGCGCTCGCCCTGGTCGTCCGCAGCCGTTGCGAGGTACTAAAGGC"),
                ("phix3", "GCGCATAAATTTGAGCAGATTTGTCGTCACAGGTTGCGCCGCCAAAA")]
        # seq5 is 80% similar, so should be kept for 0.95 default similarity
        # to artifacts
        exp_seqs = ["seq5", "phix1", "phix2", "phix3"]
        seqs_fp = join(self.working_dir, "seqs.fasta")
        with open(seqs_fp, 'w') as seqs_f:
            for seq in seqs:
                seqs_f.write(">%s\n%s\n" % seq)
        ref = [("ref1", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTA"
                        "GTCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref2", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref3", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref4", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
               ("ref5", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATAGGGT"),
               ("ref6", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                        "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT")]
        ref_fp = join(self.working_dir, "ref4.fasta")
        with open(ref_fp, 'w') as ref_f:
            for seq in ref:
                ref_f.write(">%s\n%s\n" % seq)
        self.files_to_remove.append(ref_fp)
        ref_db_fp = build_index_sortmerna([ref_fp], self.working_dir)
        output_fp = join(self.working_dir, "seqs_filtered.fasta")
        output_fp, num_seqs_left, _ = remove_artifacts_seqs(seqs_fp=seqs_fp,
                                                            ref_fp=(ref_fp,),
                                                            working_dir=self.working_dir,
                                                            ref_db_fp=ref_db_fp,
                                                            negate=True,
                                                            threads=1)
        obs_seqs = []
        for label, seq in sequence_generator(output_fp):
            obs_seqs.append(label)
        self.assertEqual(obs_seqs, exp_seqs)

    def test_remove_chimeras_denovo_from_seqs(self):
        """ Test remove_chimeras_denovo_from_seqs() method functionality.
            Remove chimeric sequences from a FASTA file using the UCHIME
            algorithm, implemented in VSEARCH.
        """
        seqs = [("s1_104;size=2;", "GTGCCAGCCGCCGCGGTAATACCCGCAGCTCAAGTGGTG"
                                   "GTCGCTATTATTGAGCCTAAAACGTCCGTAGTCGGCTTT"
                                   "GTAAATCCCTGGGTAAATCGGGAAGCTTAACTTTCCGAC"
                                   "TTCCGAGGAGACTGTCAAACTTGGGACCGGGAG"),
                ("s1_106;size=2;", "GTGTCAGCCGCCGCGGTAATACCAGCTCTCCGAGTGGTG"
                                   "TGGATGTTTATTGGGCCTAAAGCGTCCGTAGCCGGCTGC"
                                   "GCAAGTCTGTCGGGAAATCCGCACGCCTAACGTGCGGGC"
                                   "GTCCGGCGGAAACTGCGTGGCTTGGGACCGGAA"),
                ("s1_1;size=9;", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAA"
                                 "ACGTCCGTAGTCGGCTTTGTAAATCCCTGGGTAAATCGGGT"
                                 "CGCTTAACGATCCGATTCTGGGGAGACTGCAAAGCTTGGGA"
                                 "CCGGGCGAGGTTAGAGGTACTCTCGGG"),
                ("s1_20;size=9;", "TACCTGCAGCCCAAGTGGTGGTCGATTTTATTGAGTCTAA"
                                  "AACGTTCGTAGCCGGTTTGATAAATCCTTGGGTAAATCGG"
                                  "GAAGCTTAACTTTCCGATTCCGAGGAGACTGTCAAACTTG"
                                  "GGACCGGGAGAGGCTAGAGGTACTTCTGGG"),
                ("s1_40;size=8;", "TACCAGCTCTCCGAGTGGTGTGGATGTTTATTGGGCCTAA"
                                  "AGCATCCGTAGCTGGCTAGGTTAGTCCCCTGTTAAATCCA"
                                  "CCGAATTAATCGTTGGATGCGGGGGATACTGCTTGGCTAG"
                                  "GGGACGAGAGAGGCAGACGGTATTTCCGGG"),
                ("s1_60;size=8;", "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAA"
                                  "AGCGTCCGTAGCCGGCTGCGCAAGTCTGTCGGGAAATCCG"
                                  "CACGCCTAACGTGCGGGTCCGGCGGAAACTGCGTGGCTTG"
                                  "GGACCGGAAGACTCGAGGGGTACGTCAGGG")]
        seqs_non_chimera = ["s1_1;size=9;", "s1_20;size=9;",
                            "s1_40;size=8;", "s1_60;size=8;"]
        seqs_fp = join(self.working_dir, "seqs.fasta")
        with open(seqs_fp, 'w') as seqs_f:
            for seq in seqs:
                seqs_f.write(">%s\n%s\n" % seq)
        output_fp = remove_chimeras_denovo_from_seqs(
            seqs_fp=seqs_fp,
            working_dir=self.working_dir)
        seqs_obs = []
        for label, seq in sequence_generator(output_fp):
            label = label.split()[0]
            seqs_obs.append(label)
        self.assertEqual(seqs_non_chimera, seqs_obs)

    def test_multiple_sequence_alignment(self):
        """Test multiple sequence alignment.
        """
        seqs = [DNA('caccggcggcccggtggtggccattattattgggtctaaag',
                    metadata={'id': 'seq_1'}, lowercase=True),
                DNA('caccggcggcccgagtggtggccattattattgggtcaagg',
                    metadata={'id': 'seq_2'}, lowercase=True),
                DNA('caccggcggcccgagtgatggccattattattgggtctaaag',
                    metadata={'id': 'seq_3'}, lowercase=True),
                DNA('aaccggcggcccaagtggtggccattattattgggtctaaag',
                    metadata={'id': 'seq_4'}, lowercase=True),
                DNA('caccgggcccgagtggtggccattattattgggtctaaag',
                    metadata={'id': 'seq_5'}, lowercase=True)]

        seqs_fp = join(self.working_dir, "seqs.fna")
        with open(seqs_fp, 'w') as o:
            for seq in seqs:
                seq.write(o, format='fasta')
        alignment_file = multiple_sequence_alignment(seqs_fp)
        aligned_seqs = [DNA(item[1], metadata={'id': item[0]}, lowercase=True)
                        for item in sequence_generator(alignment_file)]

        align_exp = [
            DNA('caccggcggcccg-gtggtggccattattattgggtctaaag',
                lowercase=True, metadata={'id': 'seq_1'}),
            DNA('caccggcggcccgagtggtggccattattattgggtcaagg-',
                lowercase=True, metadata={'id': 'seq_2'}),
            DNA('caccggcggcccgagtgatggccattattattgggtctaaag',
                lowercase=True, metadata={'id': 'seq_3'}),
            DNA('aaccggcggcccaagtggtggccattattattgggtctaaag',
                lowercase=True, metadata={'id': 'seq_4'}),
            DNA('caccg--ggcccgagtggtggccattattattgggtctaaag',
                lowercase=True, metadata={'id': 'seq_5'})]
        self.assertEqual(aligned_seqs, align_exp)

    def test_build_index_sortmerna(self):
        """Test functionality of build_index_sortmerna()
        """
        ref1 = [("ref1", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTA"
                 "GTCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
                ("ref2", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                 "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
                ("ref3", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                 "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
                ("ref4", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                 "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT"),
                ("ref5", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                 "TCGGCTTTGTAAATCCCTGGGTAAATAGGGT"),
                ("ref6", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                 "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT")]
        ref2 = [("ref1", "GTCGTAGCTAGCTGCCCACGATCGTAGCTAGCTAGCTACGTAGCTCATCAC"
                 "TCGCCGACCCACGTCCCACTGATGCTGTGGG"),
                ("ref2", "GCGGCGCCCAAAAATGTCGTGTAAAATTTTCTCGTACCCACTTGCTACCCA"
                 "TGGCCGCCATGCTGCTAACGCAATATATATA"),
                ("ref3", "TGTGAAAGCGCGCGAGAGAGTCGTATATATGGGCGCGGCGCGATGCTGCCC"
                 "GTCGATGCTGATCCCCCACGTACGTAGCCCC"),
                ("ref4", "GTGTGCTCGCGTAGCTAGCTTATATATCGGCGCGTAGTGCTAGCCCCAAAA"
                 "GTGTCCCCCCCCTCCTTTTTTATATATGCAA"),
                ("ref5", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                 "TCGGCTTTGTAAATCCCTGGGTAAATAGGGT"),
                ("ref6", "TACCCGCAGCTCAAGTGGTGGTCGCTATTATTGAGCCTAAAACGTCCGTAG"
                 "TCGGCTTTGTAAATCCCTGGGTAAATCGGGT")]
        ref1_fp = join(self.working_dir, "ref1.fasta")
        with open(ref1_fp, 'w') as ref_f:
            for seq in ref1:
                ref_f.write(">%s\n%s\n" % seq)
        ref2_fp = join(self.working_dir, "ref2.fasta")
        with open(ref2_fp, 'w') as ref_f:
            for seq in ref2:
                ref_f.write(">%s\n%s\n" % seq)
        ref_fps = tuple([ref1_fp, ref2_fp])
        ref_db_fp = build_index_sortmerna(
            ref_fp=ref_fps,
            working_dir=self.working_dir)
        self.assertEqual(len(ref_fps), len(ref_db_fp))

    def test_build_index_sortmerna_fail(self):
        """Test functionality of build_index_sortmerna()
        """
        with self.assertRaises(RuntimeError):
            build_index_sortmerna(
                ref_fp='foo',
                working_dir=self.working_dir)

    def run_workflow_try(self, simfilename, origfilename,
                         ref_fp, ref_db_fp, threads=1,
                         trim_length=100):
        """Test launching the complete workflow using simulated sequences
        and compare to original ground truth.

        Parameters
        ----------
        simfilename : str
            name of the simulated reads fasta file
        origfilename : str
            name of the fasta file with the ground truth sequences
        ref_fp : list of str
            list of the reference database files
        def_db_fp : list of str
            list of the indexed database files or None to create them
        threads : int
            number of threads to use (default=1)
        trim_length : int
            length of sequences to trim to (default=100)
        """
        seqs_fp = simfilename
        output_fp = self.working_dir
        mean_error = 0.005
        error_dist = get_default_error_profile()
        indel_prob = 0.01
        indel_max = 3
        min_size = 2
        left_trim_length = 0
        nochimera = launch_workflow(seqs_fp=seqs_fp, working_dir=output_fp,
                                    mean_error=mean_error,
                                    error_dist=error_dist,
                                    indel_prob=indel_prob,
                                    indel_max=indel_max,
                                    trim_length=trim_length,
                                    left_trim_length=left_trim_length,
                                    min_size=min_size,
                                    ref_fp=(ref_fp,),
                                    ref_db_fp=ref_db_fp,
                                    threads_per_sample=threads)

        # get the trimmed ground truth sequences
        orig_seqs = [item[1] for item in sequence_generator(origfilename)]
        orig_seqs = [item[:trim_length].upper() for item in orig_seqs]

        output_filename = 'all.biom'
        output_table_fp = join(output_fp, output_filename)

        create_otu_table(output_table_fp, [(nochimera, seqs_fp)])

        table_obs = load_table(output_table_fp)
        outseqs = table_obs.ids(axis='observation')
        outseqs = list(outseqs)
        outseqs.sort()
        orig_seqs.sort()

        # test we see all ground truth sequences and no other
        self.assertEqual(outseqs, orig_seqs)

    def test_get_files_for_table(self):
        filelist = get_files_for_table(self.test_data_dir)
        file1 = join(self.test_data_dir,
                     'testmerge.fasta.trim.derep.no_artifacts'
                     '.msa.deblur.no_chimeras')
        file2 = join(self.test_data_dir,
                     'testmerge2.fasta.trim.derep.no_artifacts'
                     '.msa.deblur.no_chimeras')
        file3 = join(self.test_data_dir,
                     'testmerge.fastq.trim.derep.no_artifacts'
                     '.msa.deblur.no_chimeras')
        self.assertEqual(len(filelist), 3)
        fnames, sample_names = zip(*filelist)
        self.assertIn(file1, fnames)
        self.assertIn(file2, fnames)
        self.assertIn(file3, fnames)
        self.assertIn('testmerge', sample_names)

    def test_create_otu_table(self):
        # merge the fasta files
        m1 = join(self.test_data_dir,
                  'testmerge.fasta.trim.derep.no_artifacts'
                  '.msa.deblur.no_chimeras')
        m2 = join(self.test_data_dir,
                  'testmerge2.fasta.trim.derep.no_artifacts'
                  '.msa.deblur.no_chimeras')
        outfile = join(self.working_dir, 'testmerge.biom')
        fasta_outfile = join(self.working_dir, 'testmerge.seq.fa')
        create_otu_table(outfile, [(m1, 'testmerge'), (m2, 'testmerge2')],
                         outputfasta_fp=fasta_outfile)

        # test the result
        table = load_table(outfile)
        tableids = table.ids(axis='observation')
        # test a sequence present in both
        self.assertEqual(table.get_value_by_ids(
            'TACGAGGggggCGAGCGTTGTTCGGAATTATTGGGCGTAAAAGGTGCGTAGGCGGTTCG'
            'GTAAGTTTCGTGTGAAATCTTCGGGCTCAACTCGAAGCCTGCACGAAATACTGCCGGGC'
            'TTGAGTGTGGGAGAGGTGAGTGGAATTTCCGGT', 'testmerge'), 5)
        self.assertEqual(table.get_value_by_ids(
            'TACGAGGggggCGAGCGTTGTTCG'
            'GAATTATTGGGCGTAAAAGGTGCGTAGGCGGTTCGGTAAGTTTCGTGTGAAATCTTCGGG'
            'CTCAACTCGAAGCCTGCACGAAATACTGCCGGGCTTGAGTGTGGGAGAGGTGAGTGGAAT'
            'TTCCGGT', 'testmerge2'), 8)
        # and an otu present only in one
        self.assertEqual(table.get_value_by_ids(
            'TACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGAGCGTAGGCGGTTTCTT'
            'AAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGGGAACTTGA'
            'GTGCAGAAGAGGAGAGTGGAATTCCATGT', 'testmerge'), 7)
        self.assertEqual(table.get_value_by_ids(
            'TACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGAGCGTAGGCGGTTTCTTA'
            'AGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGGGAACTTGAGT'
            'GCAGAAGAGGAGAGTGGAATTCCATGT', 'testmerge2'), 0)
        # test the output fasta file
        allseqs = []
        for label, seq in sequence_generator(fasta_outfile):
            self.assertTrue(seq in tableids)
            allseqs.append(seq)
        self.assertEqual(len(allseqs), len(tableids))

        # test minimal read filtering ( minreads>0 )
        minreads = 7
        outfile2 = join(self.working_dir, 'testmerge2.biom')
        create_otu_table(outfile2, [(m1, 'testmerge'), (m2, 'testmerge2')],
                         minreads=minreads)

        table2 = load_table(outfile2)
        table2ids = table2.ids(axis='observation')
        tablesum = table.sum(axis='observation')
        for idx, cid in enumerate(table.ids(axis='observation')):
            if tablesum[idx] >= minreads:
                self.assertIn(cid, table2ids)
            else:
                self.assertNotIn(cid, table2ids)

        self.assertEqual(table2.get_value_by_ids(
            'TACGAGGggggCGAGCGTTGTTCG'
            'GAATTATTGGGCGTAAAAGGTGCGTAGGCGGTTCGGTAAGTTTCGTGTGAAATCTTCGGG'
            'CTCAACTCGAAGCCTGCACGAAATACTGCCGGGCTTGAGTGTGGGAGAGGTGAGTGGAAT'
            'TTCCGGT', 'testmerge2'), 8)
        # and an otu present only in one
        self.assertEqual(table2.get_value_by_ids(
            'TACGTAGGTGGCAAGCGTTATCCGGAATTATTGGGCGTAAAGCGAGCGTAGGCGGTTTCTT'
            'AAGTCTGATGTGAAAGCCCACGGCTCAACCGTGGAGGGTCATTGGAAACTGGGGAACTTGA'
            'GTGCAGAAGAGGAGAGTGGAATTCCATGT', 'testmerge'), 7)

    def test_create_otu_table_same_sampleid(self):
        # merge the fasta files
        m1 = join(self.test_data_dir,
                  'testmerge.fasta.trim.derep.no_artifacts'
                  '.msa.deblur.no_chimeras')
        m2 = join(self.test_data_dir,
                  'testmerge2.fasta.trim.derep.no_artifacts'
                  '.msa.deblur.no_chimeras')
        outfile = join(self.working_dir, 'testmerge.biom')
        with self.assertWarns(UserWarning):
            create_otu_table(outfile, [(m1, 'testmerge'), (m2, 'testmerge')])

    def test_launch_workflow(self):
        """Test launching complete workflow using 3 simulated sequence files.
        seqs1 - 100 reads using art, original sequences are >0.5 identical.
        seqs2 - 200 reads using grinder, original sequences are >0.9 identical,
        0.1 chimeras, 35 phix reads
        seqs3 - simple - 15 reads from seqs1 (10 reads for 1001203,
        5 reads for 694276) for manual test validation
        """
        # index the 70% rep. set database
        ref_fp = join(self.test_data_dir, '70_otus.fasta')
        ref_db_fp = build_index_sortmerna(
            ref_fp=(ref_fp,),
            working_dir=self.working_dir)

        self.run_workflow_try(self.seqs_s1_fp,
                              self.orig_s1_fp, ref_fp, ref_db_fp)
        self.run_workflow_try(self.seqs_s2_fp,
                              self.orig_s2_fp, ref_fp, ref_db_fp)
        self.run_workflow_try(self.seqs_s3_fp,
                              self.orig_s3_fp, ref_fp, ref_db_fp)
        self.run_workflow_try(self.seqs_s3_fp,
                              self.orig_s3_fp, ref_fp, ref_db_fp,
                              threads=3)

    def test_launch_workflow_skip_trim(self):
        # index the 70% rep. set database
        ref_fp = join(self.test_data_dir, '70_otus.fasta')
        ref_db_fp = build_index_sortmerna(
            ref_fp=(ref_fp,),
            working_dir=self.working_dir)

        seqs_fp = self.seqs_s1_fp
        output_fp = self.working_dir
        mean_error = 0.005
        error_dist = get_default_error_profile()
        indel_prob = 0.01
        indel_max = 3
        min_size = 2
        # trim length longer than sequences
        trim_length = -1
        left_trim_length = 0
        threads = 1

        output_fp = launch_workflow(seqs_fp=seqs_fp,
                                    working_dir=output_fp,
                                    mean_error=mean_error,
                                    error_dist=error_dist,
                                    indel_prob=indel_prob,
                                    indel_max=indel_max,
                                    trim_length=trim_length,
                                    left_trim_length=left_trim_length,
                                    min_size=min_size,
                                    ref_fp=(ref_fp,),
                                    ref_db_fp=ref_db_fp,
                                    threads_per_sample=threads)
        exp = Sequence.read(self.no_trim_res, format='fasta')
        res = Sequence.read(output_fp, format='fasta')
        self.assertEqual(exp, res)

    def test_launch_workflow_incorrect_trim(self):
        """
        test if we get the warning when trim length
        is too long
        """
        # index the 70% rep. set database
        ref_fp = join(self.test_data_dir, '70_otus.fasta')
        ref_db_fp = build_index_sortmerna(
            ref_fp=(ref_fp,),
            working_dir=self.working_dir)

        seqs_fp = self.seqs_s1_fp
        output_fp = self.working_dir
        mean_error = 0.005
        error_dist = get_default_error_profile()
        indel_prob = 0.01
        indel_max = 3
        min_size = 2
        # trim length longer than sequences
        trim_length = 151
        left_trim_length = 0
        threads = 1
        with self.assertWarns(UserWarning):
            launch_workflow(seqs_fp=seqs_fp, working_dir=output_fp,
                            mean_error=mean_error,
                            error_dist=error_dist,
                            indel_prob=indel_prob,
                            indel_max=indel_max,
                            trim_length=trim_length,
                            left_trim_length=left_trim_length,
                            min_size=min_size,
                            ref_fp=(ref_fp,),
                            ref_db_fp=ref_db_fp,
                            threads_per_sample=threads)

    def get_seqs_act_split_sequence_on_sample_ids(self, output_dir):
        """Parse output of split_sequence_file_on_sample_ids_to_files()

        Parameters
        ----------
        output_dir: string
            output directory path storing FASTA files

        Returns
        -------
        seqs_act: dict
            dictionary with keys being sample IDs and values list of
            sequences belonging to sample ID
        """
        seqs_act = {}
        for fn in listdir(output_dir):
            input_fp = join(output_dir, fn)
            sample_file = splitext(fn)[0]
            for label, seq in sequence_generator(input_fp):
                sample = label.split('_')[0]
                self.assertEqual(sample_file, sample)
                if sample not in seqs_act:
                    seqs_act[sample] = [(label, seq)]
                else:
                    seqs_act[sample].append((label, seq))
        return seqs_act

    def test_sample_id_from_read_id(self):
        """Test the fasta readid to sample id
        used in split_sequence_file_on_sample_ids_to_files
        """
        self.assertEqual(sample_id_from_read_id("Samp1_0 M04771:27:000000000"
                                                "-ARFWH:1:1101:18081:1897 1:"
                                                "N:0:0 orig_bc=CGTTAAGTCAGC n"
                                                "ew_bc=CGTTAAGTCAGC bc_diffs="
                                                "0"), "Samp1")
        self.assertEqual(sample_id_from_read_id("S1_1_0 M04771:27:000000000"
                                                "-ARFWH:1:1101:18081:1897 1:"
                                                "N:0:0 orig_bc=CGTTAAGTCAGC n"
                                                "ew_bc=CGTTAAGTCAGC bc_diffs="
                                                "0"), "S1_1")

    def test_split_sequence_file_on_sample_ids_to_files(self):
        """Test functionality of split_sequence_file_on_sample_ids_to_files()
        """
        seqs_fasta = {"s1": [
                      ("s1_seq1",
                       "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
                      ("s1_seq2",
                       "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG")],
                      "s2": [
                      ("s2_seq3",
                       "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCG"),
                      ("s2_seq4",
                       "TACCGGCAGCTCAAGTGATGACCGCTATTATTGGGCCTAAAGCGTCCT")],
                      "s3": [
                      ("s3_seq5",
                       "TACCAGCCCCTTAAGTGGTAGGGACGATTATTTGGCCTAAAGCGTCCG"),
                      ("s3_seq6",
                       "CTGCAAGGCTAGGGGGCGGGAGAGGCGGGTGGTACTTGAGGGGAGAAT")],
                      "s4": [
                      ("s4_seq7",
                       "CTGCAAGGCTAGGGGGCGGGAGAGGCGGGTGGTACTTGAGGGGAGAAT")]}
        # Test FASTA file split on sample IDs to multiple FASTA files
        seqs_fp = join(self.working_dir, "seqs.fasta")
        with open(seqs_fp, 'w') as seqs_f:
            for sample in seqs_fasta:
                for seq in seqs_fasta[sample]:
                    seqs_f.write(">%s\n%s\n" % seq)
        output_dir = mkdtemp()
        split_sequence_file_on_sample_ids_to_files(seqs=seqs_fp,
                                                   outdir=output_dir)
        seqs_act = self.get_seqs_act_split_sequence_on_sample_ids(
            output_dir=output_dir)
        self.assertEqual(seqs_fasta, seqs_act)

    def test_fasta_from_biom(self):
        '''Test the fasta file from biom table function fasta_from_biom()
        '''
        input_biom_file = join(self.test_data_dir, 'final.s4.biom')
        table = load_table(input_biom_file)
        output_fasta = join(self.working_dir, 'final.s4.seqs.fa')
        fasta_from_biom(table, output_fasta)
        self.assertTrue(isfile(output_fasta))
        table_seqs = table.ids(axis='observation')
        expected = [(seq, seq) for seq in table_seqs]
        written_seqs = [item for item in sequence_generator(output_fasta)]
        self.assertListEqual(written_seqs, expected)

if __name__ == '__main__':
    main()


text:
"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports._REMOTE_REPO_URL = exports._LAST_UPDATED_FILE = exports.updateCacheRepo = exports._ensureCacheRepo = exports._CACHE_REPO_GIT_DIR = exports._CACHE_REPO_EXPIRY = exports._CACHE_REPO_DIR = exports._cacheRepoAssure = undefined;

var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();

exports.getLibDefs = getLibDefs;
exports.getLocalLibDefs = getLocalLibDefs;
exports.getCacheLibDefs = getCacheLibDefs;
exports.getCacheLibDefVersion = getCacheLibDefVersion;
exports.filterLibDefs = filterLibDefs;

var _semver = require("semver");

var _semver2 = _interopRequireDefault(_semver);

var _git = require("./git.js");

var _fileUtils = require("./fileUtils.js");

var _node = require("./node.js");

var _semver3 = require("./semver.js");

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

var P = Promise;

var CACHE_DIR = _node.path.join(_node.os.homedir(), '.flow-typed');
var CACHE_REPO_DIR = _node.path.join(CACHE_DIR, 'repo');
var GIT_REPO_DIR = _node.path.join(__dirname, '..', '..', '..');

var REMOTE_REPO_URL = 'http://github.com/flowtype/flow-typed.git';
var LAST_UPDATED_FILE = _node.path.join(CACHE_DIR, 'lastUpdated');

function cloneCacheRepo(verbose) {
  return regeneratorRuntime.async(function cloneCacheRepo$(_context) {
    while (1) {
      switch (_context.prev = _context.next) {
        case 0:
          _context.next = 2;
          return regeneratorRuntime.awrap((0, _fileUtils.mkdirp)(CACHE_REPO_DIR));

        case 2:
          _context.prev = 2;
          _context.next = 5;
          return regeneratorRuntime.awrap((0, _git.cloneInto)(REMOTE_REPO_URL, CACHE_REPO_DIR));

        case 5:
          _context.next = 11;
          break;

        case 7:
          _context.prev = 7;
          _context.t0 = _context["catch"](2);

          writeVerbose(verbose, 'ERROR: Unable to clone the local cache repo.');
          throw _context.t0;

        case 11:
          _context.next = 13;
          return regeneratorRuntime.awrap(_node.fs.writeFile(LAST_UPDATED_FILE, String(Date.now())));

        case 13:
        case "end":
          return _context.stop();
      }
    }
  }, null, this, [[2, 7]]);
}

var CACHE_REPO_GIT_DIR = _node.path.join(CACHE_REPO_DIR, '.git');
function rebaseCacheRepo(verbose) {
  return regeneratorRuntime.async(function rebaseCacheRepo$(_context2) {
    while (1) {
      switch (_context2.prev = _context2.next) {
        case 0:
          _context2.next = 2;
          return regeneratorRuntime.awrap(_node.fs.exists(CACHE_REPO_DIR));

        case 2:
          _context2.t0 = _context2.sent;

          if (!_context2.t0) {
            _context2.next = 7;
            break;
          }

          _context2.next = 6;
          return regeneratorRuntime.awrap(_node.fs.exists(CACHE_REPO_GIT_DIR));

        case 6:
          _context2.t0 = _context2.sent;

        case 7:
          if (!_context2.t0) {
            _context2.next = 22;
            break;
          }

          _context2.prev = 8;
          _context2.next = 11;
          return regeneratorRuntime.awrap((0, _git.rebaseRepoMaster)(CACHE_REPO_DIR));

        case 11:
          _context2.next = 17;
          break;

        case 13:
          _context2.prev = 13;
          _context2.t1 = _context2["catch"](8);

          writeVerbose(verbose, 'ERROR: Unable to rebase the local cache repo. ' + _context2.t1.message);
          return _context2.abrupt("return", false);

        case 17:
          _context2.next = 19;
          return regeneratorRuntime.awrap(_node.fs.writeFile(LAST_UPDATED_FILE, String(Date.now())));

        case 19:
          return _context2.abrupt("return", true);

        case 22:
          _context2.next = 24;
          return regeneratorRuntime.awrap(cloneCacheRepo(verbose));

        case 24:
          return _context2.abrupt("return", true);

        case 25:
        case "end":
          return _context2.stop();
      }
    }
  }, null, this, [[8, 13]]);
}

/**
 * Utility wrapper for ensureCacheRepo with an update expiry of 0 hours.
 */
function updateCacheRepo(verbose) {
  return regeneratorRuntime.async(function updateCacheRepo$(_context3) {
    while (1) {
      switch (_context3.prev = _context3.next) {
        case 0:
          _context3.next = 2;
          return regeneratorRuntime.awrap(ensureCacheRepo(verbose, 0));

        case 2:
          return _context3.abrupt("return", _context3.sent);

        case 3:
        case "end":
          return _context3.stop();
      }
    }
  }, null, this);
}

/**
 * Ensure that the CACHE_REPO_DIR exists and is recently rebased.
 * (else: create/rebase it)
 */
var CACHE_REPO_EXPIRY = 1000 * 60; // 1 minute
var _cacheRepoAssure = exports._cacheRepoAssure = {
  lastAssured: 0,
  pendingAssure: Promise.resolve()
};
function ensureCacheRepo(verbose) {
  var cacheRepoExpiry = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : CACHE_REPO_EXPIRY;
  var prevAssure;
  return regeneratorRuntime.async(function ensureCacheRepo$(_context5) {
    while (1) {
      switch (_context5.prev = _context5.next) {
        case 0:
          if (!(_cacheRepoAssure.lastAssured + 5 * 1000 * 60 >= Date.now())) {
            _context5.next = 2;
            break;
          }

          return _context5.abrupt("return", _cacheRepoAssure.pendingAssure);

        case 2:

          _cacheRepoAssure.lastAssured = Date.now();
          prevAssure = _cacheRepoAssure.pendingAssure;
          return _context5.abrupt("return", _cacheRepoAssure.pendingAssure = prevAssure.then(function () {
            return function _callee() {
              var repoDirExists, repoGitDirExists, lastUpdated, lastUpdatedRaw, lastUpdatedNum, rebaseSuccessful;
              return regeneratorRuntime.async(function _callee$(_context4) {
                while (1) {
                  switch (_context4.prev = _context4.next) {
                    case 0:
                      repoDirExists = _node.fs.exists(CACHE_REPO_DIR);
                      repoGitDirExists = _node.fs.exists(CACHE_REPO_GIT_DIR);
                      _context4.next = 4;
                      return regeneratorRuntime.awrap(repoDirExists);

                    case 4:
                      _context4.t0 = !_context4.sent;

                      if (_context4.t0) {
                        _context4.next = 9;
                        break;
                      }

                      _context4.next = 8;
                      return regeneratorRuntime.awrap(repoGitDirExists);

                    case 8:
                      _context4.t0 = !_context4.sent;

                    case 9:
                      if (!_context4.t0) {
                        _context4.next = 16;
                        break;
                      }

                      writeVerbose(verbose, '• flow-typed cache not found, fetching from GitHub...', false);
                      _context4.next = 13;
                      return regeneratorRuntime.awrap(cloneCacheRepo(verbose));

                    case 13:
                      writeVerbose(verbose, 'done.');
                      _context4.next = 31;
                      break;

                    case 16:
                      lastUpdated = 0;
                      _context4.next = 19;
                      return regeneratorRuntime.awrap(_node.fs.exists(LAST_UPDATED_FILE));

                    case 19:
                      if (!_context4.sent) {
                        _context4.next = 25;
                        break;
                      }

                      _context4.next = 22;
                      return regeneratorRuntime.awrap(_node.fs.readFile(LAST_UPDATED_FILE));

                    case 22:
                      lastUpdatedRaw = _context4.sent;
                      lastUpdatedNum = parseInt(lastUpdatedRaw, 10);

                      if (String(lastUpdatedNum) === String(lastUpdatedRaw)) {
                        lastUpdated = lastUpdatedNum;
                      }

                    case 25:
                      if (!(lastUpdated + cacheRepoExpiry < Date.now())) {
                        _context4.next = 31;
                        break;
                      }

                      writeVerbose(verbose, '• rebasing flow-typed cache...', false);
                      _context4.next = 29;
                      return regeneratorRuntime.awrap(rebaseCacheRepo(verbose));

                    case 29:
                      rebaseSuccessful = _context4.sent;

                      if (rebaseSuccessful) {
                        writeVerbose(verbose, 'done.');
                      } else {
                        writeVerbose(verbose, '\nNOTE: Unable to rebase local cache! If you don\'t currently ' + 'have internet connectivity, no worries -- we\'ll update the ' + 'local cache the next time you do.\n');
                      }

                    case 31:
                    case "end":
                      return _context4.stop();
                  }
                }
              }, null, this);
            }();
          }));

        case 5:
        case "end":
          return _context5.stop();
      }
    }
  }, null, this);
}
// Exported for tests -- since we really want this part well-tested.
exports._CACHE_REPO_DIR = CACHE_REPO_DIR;
exports._CACHE_REPO_EXPIRY = CACHE_REPO_EXPIRY;
exports._CACHE_REPO_GIT_DIR = CACHE_REPO_GIT_DIR;
exports._ensureCacheRepo = ensureCacheRepo;
exports.updateCacheRepo = updateCacheRepo;
exports._LAST_UPDATED_FILE = LAST_UPDATED_FILE;
exports._REMOTE_REPO_URL = REMOTE_REPO_URL;

function addLibDefs(pkgDirPath, libDefs, validationErrs) {
  var parsedDirItem;
  return regeneratorRuntime.async(function addLibDefs$(_context6) {
    while (1) {
      switch (_context6.prev = _context6.next) {
        case 0:
          parsedDirItem = parseRepoDirItem(pkgDirPath, validationErrs);
          _context6.next = 3;
          return regeneratorRuntime.awrap(parseLibDefsFromPkgDir(parsedDirItem, pkgDirPath, validationErrs));

        case 3:
          _context6.t0 = function (libDef) {
            return libDefs.push(libDef);
          };

          _context6.sent.forEach(_context6.t0);

        case 5:
        case "end":
          return _context6.stop();
      }
    }
  }, null, this);
}

/**
 * Given a 'definitions/npm' dir, return a list of LibDefs that it contains.
 */
function getLibDefs(defsDir, validationErrs) {
  var _this = this;

  var libDefs, defsDirItems;
  return regeneratorRuntime.async(function getLibDefs$(_context10) {
    while (1) {
      switch (_context10.prev = _context10.next) {
        case 0:
          libDefs = [];
          _context10.next = 3;
          return regeneratorRuntime.awrap(_node.fs.readdir(defsDir));

        case 3:
          defsDirItems = _context10.sent;
          _context10.next = 6;
          return regeneratorRuntime.awrap(P.all(defsDirItems.map(function _callee4(item) {
            var itemPath, itemStat, error;
            return regeneratorRuntime.async(function _callee4$(_context9) {
              while (1) {
                switch (_context9.prev = _context9.next) {
                  case 0:
                    if (!(item === '.cli-metadata.json')) {
                      _context9.next = 2;
                      break;
                    }

                    return _context9.abrupt("return");

                  case 2:
                    itemPath = _node.path.join(defsDir, item);
                    _context9.next = 5;
                    return regeneratorRuntime.awrap(_node.fs.stat(itemPath));

                  case 5:
                    itemStat = _context9.sent;

                    if (!itemStat.isDirectory()) {
                      _context9.next = 16;
                      break;
                    }

                    if (!(item.charAt(0) === '@')) {
                      _context9.next = 12;
                      break;
                    }

                    _context9.next = 10;
                    return regeneratorRuntime.awrap(function _callee3() {
                      var scope, defsDirItems;
                      return regeneratorRuntime.async(function _callee3$(_context8) {
                        while (1) {
                          switch (_context8.prev = _context8.next) {
                            case 0:
                              // directory is of the form '@<scope>', so go one level deeper
                              scope = item;
                              _context8.next = 3;
                              return regeneratorRuntime.awrap(_node.fs.readdir(itemPath));

                            case 3:
                              defsDirItems = _context8.sent;
                              _context8.next = 6;
                              return regeneratorRuntime.awrap(P.all(defsDirItems.map(function _callee2(item) {
                                var itemPath, itemStat, error;
                                return regeneratorRuntime.async(function _callee2$(_context7) {
                                  while (1) {
                                    switch (_context7.prev = _context7.next) {
                                      case 0:
                                        itemPath = _node.path.join(defsDir, scope, item);
                                        _context7.next = 3;
                                        return regeneratorRuntime.awrap(_node.fs.stat(itemPath));

                                      case 3:
                                        itemStat = _context7.sent;

                                        if (!itemStat.isDirectory()) {
                                          _context7.next = 9;
                                          break;
                                        }

                                        _context7.next = 7;
                                        return regeneratorRuntime.awrap(addLibDefs(itemPath, libDefs, validationErrs));

                                      case 7:
                                        _context7.next = 11;
                                        break;

                                      case 9:
                                        error = "Expected only directories in the 'definitions/npm/@<scope>' directory!";

                                        validationError(itemPath, error, validationErrs);

                                      case 11:
                                      case "end":
                                        return _context7.stop();
                                    }
                                  }
                                }, null, _this);
                              })));

                            case 6:
                            case "end":
                              return _context8.stop();
                          }
                        }
                      }, null, _this);
                    }());

                  case 10:
                    _context9.next = 14;
                    break;

                  case 12:
                    _context9.next = 14;
                    return regeneratorRuntime.awrap(addLibDefs(itemPath, libDefs, validationErrs));

                  case 14:
                    _context9.next = 18;
                    break;

                  case 16:
                    error = "Expected only directories in the 'definitions/npm' directory!";

                    validationError(itemPath, error, validationErrs);

                  case 18:
                  case "end":
                    return _context9.stop();
                }
              }
            }, null, _this);
          })));

        case 6:
          return _context10.abrupt("return", libDefs);

        case 7:
        case "end":
          return _context10.stop();
      }
    }
  }, null, this);
};

/**
 * Given the path to a flow directory within a package directory
 * (i.e. definitions/npm/pkgDir/flow_v0.24.0/), parse the flow directory name
 * into a Version.
 */
var FLOW_VER = 'v([0-9]+)\.([0-9]+|x)\.([0-9]+|x)';
var FLOW_DIR_NAME_RE = new RegExp("^flow_(all|((-)?" + FLOW_VER + "(-)?(" + FLOW_VER + ")?))$");

function parsePkgFlowDirVersion(pkgFlowDirPath, validationErrs) {
  var pkgFlowDirName = _node.path.basename(pkgFlowDirPath);

  function invalidDirectoryName() {
    var repoPath = _node.path.resolve(pkgFlowDirPath, '..', '..', '..');
    var pkgFlowDirContext = _node.path.relative(repoPath, pkgFlowDirPath);
    var error = "Malformed flow-version directory name! Expected the name to be " + "formatted as 'flow_all' or " + "'flow_-?v<VERSION>' or " + "'flow_v<VERSION>-?' or " + "'flow_v<VERSION>-v<VERSION>' (for a range).";
    validationError(pkgFlowDirContext, error, validationErrs);
  }

  var matches = pkgFlowDirName.match(FLOW_DIR_NAME_RE);
  if (matches == null) {
    invalidDirectoryName();
    return (0, _semver3.emptyVersion)();
  }

  var upperBound = void 0;

  var _matches = _slicedToArray(matches, 12);

  var _1 = _matches[0];
  var isAll = _matches[1];
  var _2 = _matches[2];
  var range = _matches[3];
  var major = _matches[4];
  var minor = _matches[5];
  var patch = _matches[6];
  var upRange = _matches[7];
  var _3 = _matches[8];
  var upMajor = _matches[9];
  var upMinor = _matches[10];
  var upPatch = _matches[11];

  if (isAll === 'all') {
    range = undefined;
    major = 'x';
    minor = 'x';
    patch = 'x';
  } else if (range && upRange) {
    // flow_-v0.x.x-
    invalidDirectoryName();
    return (0, _semver3.emptyVersion)();
  } else if (range && upMajor) {
    // flow_-v0.x.xv0.x.x
    invalidDirectoryName();
    return (0, _semver3.emptyVersion)();
  } else if (range) {
    // flow_-0.x.x
    range = "<=";
    major = validateVersionNumPart(major, "major", pkgFlowDirPath, validationErrs);
    minor = validateVersionPart(minor, "minor", pkgFlowDirPath, validationErrs);
    patch = validateVersionPart(patch, "patch", pkgFlowDirPath, validationErrs);
  } else if (upRange) {
    // flow_0.x.x-
    // flow_0.x.x-0.x.x (if upMajor)
    range = ">=";
    major = validateVersionNumPart(major, "major", pkgFlowDirPath, validationErrs);
    minor = validateVersionPart(minor, "minor", pkgFlowDirPath, validationErrs);
    patch = validateVersionPart(patch, "patch", pkgFlowDirPath, validationErrs);
    if (upMajor) {
      upRange = "<=";
      upMajor = validateVersionNumPart(upMajor, "major", pkgFlowDirPath, validationErrs);
      upMinor = validateVersionPart(upMinor, "minor", pkgFlowDirPath, validationErrs);
      upPatch = validateVersionPart(upPatch, "patch", pkgFlowDirPath, validationErrs);
      upperBound = {
        range: upRange,
        major: upMajor,
        minor: upMinor,
        patch: upPatch
      };
    }
  } else {
    // shouldn't happen
    invalidDirectoryName();
    return (0, _semver3.emptyVersion)();
  }

  var ver = { range: range, major: major, minor: minor, patch: patch, upperBound: upperBound };
  if (!(0, _semver3.isSatVersion)(ver)) {
    validationError(pkgFlowDirPath, 'Flow version is unsatisfiable!', validationErrs);
  }
  return ver;
}

/**
 * Given a parsed package name and version and a path to the package directory
 * on disk, scan the directory and generate a list of LibDefs for each
 * flow-versioned definition file.
 */
function parseLibDefsFromPkgDir(_ref, pkgDirPath, validationErrs) {
  var pkgName = _ref.pkgName;
  var pkgVersion = _ref.pkgVersion;

  var _this2 = this;

  var repoPath, pkgVersionStr, pkgDirItems, commonTestFiles, flowDirs, libDefs;
  return regeneratorRuntime.async(function parseLibDefsFromPkgDir$(_context12) {
    while (1) {
      switch (_context12.prev = _context12.next) {
        case 0:
          repoPath = _node.path.relative(pkgDirPath, '..');
          pkgVersionStr = (0, _semver3.versionToString)(pkgVersion);
          _context12.next = 4;
          return regeneratorRuntime.awrap(_node.fs.readdir(pkgDirPath));

        case 4:
          pkgDirItems = _context12.sent;
          commonTestFiles = [];
          flowDirs = [];

          pkgDirItems.forEach(function (pkgDirItem) {
            var pkgDirItemPath = _node.path.join(pkgDirPath, pkgDirItem);
            var pkgDirItemContext = _node.path.relative(repoPath, pkgDirItemPath);

            var pkgDirItemStat = _node.fs.statSync(pkgDirItemPath);
            if (pkgDirItemStat.isFile()) {
              if (_node.path.extname(pkgDirItem) === '.swp') {
                return;
              }

              var isValidTestFile = validateTestFile(pkgDirItemPath, pkgDirItemContext, validationErrs);

              if (isValidTestFile) {
                commonTestFiles.push(pkgDirItemPath);
              }
            } else if (pkgDirItemStat.isDirectory()) {
              flowDirs.push([pkgDirItemPath, parsePkgFlowDirVersion(pkgDirItemPath, validationErrs)]);
            } else {
              var error = 'Unexpected directory item';
              validationError(pkgDirItemContext, error, validationErrs);
            }
          });

          if (!(0, _semver3.disjointVersionsAll)(flowDirs.map(function (_ref2) {
            var _ref3 = _slicedToArray(_ref2, 2);

            var _ = _ref3[0];
            var ver = _ref3[1];
            return ver;
          }))) {
            validationError(pkgDirPath, 'Flow versions not disjoint!', validationErrs);
          }

          if (flowDirs.length === 0) {
            validationError(pkgDirPath, 'No libdef files found!', validationErrs);
          }

          libDefs = [];
          _context12.next = 13;
          return regeneratorRuntime.awrap(P.all(flowDirs.map(function _callee5(_ref4) {
            var _ref5 = _slicedToArray(_ref4, 2);

            var flowDirPath = _ref5[0];
            var flowVersion = _ref5[1];
            var testFilePaths, basePkgName, libDefFileName, libDefFilePath, error;
            return regeneratorRuntime.async(function _callee5$(_context11) {
              while (1) {
                switch (_context11.prev = _context11.next) {
                  case 0:
                    testFilePaths = [].concat(commonTestFiles);
                    basePkgName = pkgName.charAt(0) === '@' ? pkgName.split(_node.path.sep).pop() : pkgName;
                    libDefFileName = basePkgName + "_" + pkgVersionStr + ".js";
                    libDefFilePath = void 0;
                    _context11.next = 6;
                    return regeneratorRuntime.awrap(_node.fs.readdir(flowDirPath));

                  case 6:
                    _context11.t0 = function (flowDirItem) {
                      var flowDirItemPath = _node.path.join(flowDirPath, flowDirItem);
                      var flowDirItemContext = _node.path.relative(repoPath, flowDirItemPath);
                      var flowDirItemStat = _node.fs.statSync(flowDirItemPath);
                      if (flowDirItemStat.isFile()) {
                        // If we couldn't discern the package name, we've already recorded an
                        // error for that -- so try to avoid spurious downstream errors.
                        if (pkgName === 'ERROR') {
                          return;
                        }

                        if (_node.path.extname(flowDirItem) === '.swp') {
                          return;
                        }

                        if (flowDirItem === libDefFileName) {
                          libDefFilePath = _node.path.join(flowDirPath, flowDirItem);
                          return;
                        }

                        var isValidTestFile = validateTestFile(flowDirItemPath, flowDirItemContext, validationErrs);

                        if (isValidTestFile) {
                          testFilePaths.push(flowDirItemPath);
                        }
                      } else {
                        var error = 'Unexpected directory item';
                        validationError(flowDirItemContext, error, validationErrs);
                      }
                    };

                    _context11.sent.forEach(_context11.t0);

                    if (!(libDefFilePath == null)) {
                      _context11.next = 12;
                      break;
                    }

                    libDefFilePath = _node.path.join(flowDirPath, libDefFileName);
                    if (pkgName !== 'ERROR') {
                      error = 'No libdef file found!';

                      validationError(flowDirPath, error, validationErrs);
                    }
                    return _context11.abrupt("return");

                  case 12:

                    libDefs.push({
                      pkgName: pkgName,
                      pkgVersionStr: pkgVersionStr,
                      flowVersion: flowVersion,
                      flowVersionStr: (0, _semver3.versionToString)(flowVersion),
                      path: libDefFilePath,
                      testFilePaths: testFilePaths
                    });

                  case 13:
                  case "end":
                    return _context11.stop();
                }
              }
            }, null, _this2);
          })));

        case 13:
          return _context12.abrupt("return", libDefs);

        case 14:
        case "end":
          return _context12.stop();
      }
    }
  }, null, this);
}

/**
 * Given the path to a directory item in the 'definitions' directory, parse the
 * directory's name into a package name and version.
 */
var REPO_DIR_ITEM_NAME_RE = /^(.*)_v([0-9]+)\.([0-9]+|x)\.([0-9]+|x)$/;
function parseRepoDirItem(dirItemPath, validationErrs) {
  var dirItem = _node.path.basename(dirItemPath);
  var itemMatches = dirItem.match(REPO_DIR_ITEM_NAME_RE);
  if (itemMatches == null) {
    var error = "'" + dirItem + "' is a malformed definitions/npm/ directory name! " + "Expected the name to be formatted as <PKGNAME>_v<MAJOR>.<MINOR>.<PATCH>";
    validationError(dirItem, error, validationErrs);
    var _pkgName = 'ERROR';
    var pkgVersion = (0, _semver3.emptyVersion)();
    return { pkgName: _pkgName, pkgVersion: pkgVersion };
  }

  var _itemMatches = _slicedToArray(itemMatches, 5);

  var _ = _itemMatches[0];
  var pkgName = _itemMatches[1];
  var major = _itemMatches[2];
  var minor = _itemMatches[3];
  var patch = _itemMatches[4];

  var item = _node.path.dirname(dirItemPath).split(_node.path.sep).pop();
  if (item.charAt(0) === '@') {
    pkgName = "" + item + _node.path.sep + pkgName;
  }
  major = validateVersionNumPart(major, "major", dirItemPath, validationErrs);
  minor = validateVersionPart(minor, "minor", dirItemPath, validationErrs);
  patch = validateVersionPart(patch, "patch", dirItemPath, validationErrs);

  return { pkgName: pkgName, pkgVersion: { major: major, minor: minor, patch: patch } };
}

/**
 * Given a path to an assumed test file, ensure that it is named as expected.
 */
var TEST_FILE_NAME_RE = /^test_.*\.js$/;
function validateTestFile(testFilePath, context, validationErrs) {
  var testFileName = _node.path.basename(testFilePath);
  if (!TEST_FILE_NAME_RE.test(testFileName)) {
    var error = "Malformed test file name! Test files must be formatted as test_(.*).js";
    validationError(context, error, validationErrs);
    return false;
  }
  return true;
}

/**
 * Given a number-only part of a version string (i.e. the `major` part), parse
 * the string into a number.
 */
function validateVersionNumPart(part, partName, context, validationErrs) {
  var num = parseInt(part, 10);
  if (String(num) !== part) {
    var error = "'" + context + "': Invalid " + partName + " number: '" + part + "'. Expected a number.";
    validationError(context, error, validationErrs);
  }
  return num;
}

/**
 * Given a number-or-wildcard part of a version string (i.e. a `minor` or
 * `patch` part), parse the string into either a number or 'x'.
 */
function validateVersionPart(part, partName, context, validationErrs) {
  if (part === "x") {
    return part;
  }
  return validateVersionNumPart(part, partName, context, validationErrs);
}

/**
 * Helper function to extract the error-list from the validationErrs map, append
 * to it, then set it back in the map.
 */
function validationError(errKey, errMsg, validationErrs) {
  if (validationErrs) {
    var errors = validationErrs.get(errKey) || [];
    errors.push(errMsg);
    validationErrs.set(errKey, errors);
  } else {
    throw new Error(errKey + ": " + errMsg);
  }
}

/**
 * Given a path to a 'definitions' dir, assert that the currently-running
 * version of the CLI is compatible with the repo.
 */
function verifyCLIVersion(defsDirPath) {
  var metadataFilePath, metadata, minCLIVersion, thisCLIVersion;
  return regeneratorRuntime.async(function verifyCLIVersion$(_context13) {
    while (1) {
      switch (_context13.prev = _context13.next) {
        case 0:
          metadataFilePath = _node.path.join(defsDirPath, '.cli-metadata.json');
          _context13.t0 = JSON;
          _context13.next = 4;
          return regeneratorRuntime.awrap(_node.fs.readFile(metadataFilePath));

        case 4:
          _context13.t1 = _context13.sent;
          _context13.t2 = String(_context13.t1);
          metadata = _context13.t0.parse.call(_context13.t0, _context13.t2);

          if (metadata.compatibleCLIRange) {
            _context13.next = 9;
            break;
          }

          throw new Error("Unable to find the 'compatibleCLIRange' property in " + (metadataFilePath + ". You might need to update to a newer version of ") + "the Flow CLI.");

        case 9:
          minCLIVersion = metadata.compatibleCLIRange;
          thisCLIVersion = require('../../package.json').version;

          if (_semver2.default.satisfies(thisCLIVersion, minCLIVersion)) {
            _context13.next = 13;
            break;
          }

          throw new Error("Please upgrade your CLI version! This CLI is version " + (thisCLIVersion + ", but the latest flow-typed definitions are only ") + ("compatible with flow-typed@" + minCLIVersion));

        case 13:
        case "end":
          return _context13.stop();
      }
    }
  }, null, this);
}

/**
 * Helper function to write verbose output only when an output stream was
 * provided.
 */

function writeVerbose(stream, msg) {
  var writeNewline = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : true;

  if (stream != null) {
    stream.write(msg + (writeNewline ? '\n' : ''));
  }
}

/**
 * Get a list of LibDefs from the local repo.
 *
 * Note that this is mainly only useful while working on the flow-typed repo
 * itself. It is useless when running the npm-install CLI.
 */

var GIT_REPO_DEFS_DIR = _node.path.join(GIT_REPO_DIR, 'definitions', 'npm');
function getLocalLibDefs(validationErrs) {
  return regeneratorRuntime.async(function getLocalLibDefs$(_context14) {
    while (1) {
      switch (_context14.prev = _context14.next) {
        case 0:
          _context14.next = 2;
          return regeneratorRuntime.awrap(verifyCLIVersion(_node.path.join(GIT_REPO_DIR, 'definitions')));

        case 2:
          return _context14.abrupt("return", getLibDefs(GIT_REPO_DEFS_DIR, validationErrs));

        case 3:
        case "end":
          return _context14.stop();
      }
    }
  }, null, this);
};

/**
 * Get a list of LibDefs from the flow-typed cache repo checkout.
 *
 * If the repo checkout does not exist or is out of date, it will be
 * created/updated automatically first.
 */
var CACHE_REPO_DEFS_DIR = _node.path.join(CACHE_REPO_DIR, 'definitions', 'npm');
function getCacheLibDefs() {
  var verbose = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : process.stdout;
  var validationErrs = arguments[1];
  return regeneratorRuntime.async(function getCacheLibDefs$(_context15) {
    while (1) {
      switch (_context15.prev = _context15.next) {
        case 0:
          _context15.next = 2;
          return regeneratorRuntime.awrap(ensureCacheRepo(verbose));

        case 2:
          _context15.next = 4;
          return regeneratorRuntime.awrap(verifyCLIVersion(_node.path.join(CACHE_REPO_DIR, 'definitions')));

        case 4:
          return _context15.abrupt("return", getLibDefs(CACHE_REPO_DEFS_DIR, validationErrs));

        case 5:
        case "end":
          return _context15.stop();
      }
    }
  }, null, this);
};

function getCacheLibDefVersion(libDef) {
  var latestCommitHash;
  return regeneratorRuntime.async(function getCacheLibDefVersion$(_context16) {
    while (1) {
      switch (_context16.prev = _context16.next) {
        case 0:
          _context16.next = 2;
          return regeneratorRuntime.awrap(ensureCacheRepo());

        case 2:
          _context16.next = 4;
          return regeneratorRuntime.awrap(verifyCLIVersion(_node.path.join(CACHE_REPO_DIR, 'definitions')));

        case 4:
          _context16.next = 6;
          return regeneratorRuntime.awrap((0, _git.findLatestFileCommitHash)(CACHE_REPO_DIR, _node.path.relative(CACHE_REPO_DIR, libDef.path)));

        case 6:
          latestCommitHash = _context16.sent;
          return _context16.abrupt("return", latestCommitHash.substr(0, 10) + "/" + (libDef.pkgName + "_" + libDef.pkgVersionStr + "/") + ("flow_" + libDef.flowVersionStr));

        case 8:
        case "end":
          return _context16.stop();
      }
    }
  }, null, this);
};

function packageNameMatch(a, b) {
  return a.toLowerCase() === b.toLowerCase();
}

function libdefMatchesPackageVersion(pkgSemver, defVersionRaw) {
  // The libdef version should be treated as a semver prefixed by a carat
  // (i.e: "foo_v2.2.x" is the same range as "^2.2.x")
  // UNLESS it is prefixed by the equals character (i.e. "foo_=v2.2.x")
  var defVersion = defVersionRaw;
  if (defVersionRaw[0] !== '=' && defVersionRaw[0] !== '^') {
    defVersion = '^' + defVersionRaw;
  }

  if (_semver2.default.valid(pkgSemver)) {
    // test the single package version against the libdef range
    return _semver2.default.satisfies(pkgSemver, defVersion);
  }

  if (_semver2.default.valid(defVersion)) {
    // test the single defVersion agains the package range
    return _semver2.default.satisfies(defVersion, pkgSemver);
  }

  var pkgRange = new _semver2.default.Range(pkgSemver);
  var defRange = new _semver2.default.Range(defVersion);

  if (defRange.set[0].length !== 2) {
    throw Error("Invalid libDef version, It appears to be a non-contiguous range.");
  }

  var defLowerB = defRange.set[0][0].semver.version;
  var defUpperB = defRange.set[0][1].semver.version;

  if (_semver2.default.gtr(defLowerB, pkgSemver) || _semver2.default.ltr(defUpperB, pkgSemver)) {
    return false;
  }

  var pkgLowerB = pkgRange.set[0][0].semver.version;
  return defRange.test(pkgLowerB);
}

/**
 * Filter a given list of LibDefs down using a specified filter.
 */
function filterLibDefs(defs, filter) {
  return defs.filter(function (def) {
    var filterMatch = false;
    switch (filter.type) {
      case 'exact':
        filterMatch = packageNameMatch(def.pkgName, filter.pkgName) && libdefMatchesPackageVersion(filter.pkgVersionStr, def.pkgVersionStr);
        break;

      case 'exact-name':
        filterMatch = packageNameMatch(def.pkgName, filter.term);
        break;

      case 'fuzzy':
        filterMatch = def.pkgName.toLowerCase().indexOf(filter.term.toLowerCase()) !== -1;
        break;

      default:
        throw new Error("'" + filter.type + "' is an unexpected filter type! This should never " + "happen!");
    }

    if (!filterMatch) {
      return false;
    }

    var filterFlowVerStr = filter.flowVersionStr;
    if (filterFlowVerStr) {
      var defUpperFlow = def.flowVersion.upperBound;
      if (defUpperFlow) {
        var defLowerFlow = (0, _semver3.copyVersion)(def.flowVersion);
        defLowerFlow.upperBound = undefined;
        return _semver2.default.satisfies(filterFlowVerStr, (0, _semver3.versionToString)(defLowerFlow)) && _semver2.default.satisfies(filterFlowVerStr, (0, _semver3.versionToString)(defUpperFlow));
      } else {
        return _semver2.default.satisfies(filterFlowVerStr, def.flowVersionStr);
      }
    }

    return true;
  }).sort(function (a, b) {
    var aZeroed = a.pkgVersionStr.replace(/x/g, '0');
    var bZeroed = b.pkgVersionStr.replace(/x/g, '0');
    return _semver2.default.gt(aZeroed, bZeroed) ? -1 : 1;
  });
};

redpajama_set_name: RedPajamaGithub


text:
using System.Collections.Generic;
using TentacleSoftware.XmlRpc.Core;

namespace TentacleSoftware.XmlRpc.Sample
{
    public class Blog
    {
        [XmlRpcMember("blogid")]
        public string BlogId { get; set; }

        [XmlRpcMember("url")]
        public string Url { get; set; }

        [XmlRpcMember("blogName")]
        public string BlogName { get; set; }

        [XmlRpcIgnore]
        public bool IsAdmin { get; set; }

        [XmlRpcMember("xmlrpc")]
        public string XmlRpc { get; set; }

        [XmlRpcMember("users")]
        public List<User> Users { get; set; }
    }
}


text:
(function() {

  QUnit.module('fabric.Shadow');

  var REFERENCE_SHADOW_OBJECT = {
    'color':   'rgb(0,255,0)',
    'blur':    10,
    'offsetX': 20,
    'offsetY': 5
  };

  test('constructor', function() {
    ok(fabric.Shadow);

    var shadow = new fabric.Shadow();
    ok(shadow instanceof fabric.Shadow, 'should inherit from fabric.Shadow');
  });

  test('initializing with object', function() {
    ok(fabric.Shadow);

    var shadow = new fabric.Shadow(REFERENCE_SHADOW_OBJECT);
    equal(shadow.color, 'rgb(0,255,0)');
    equal(shadow.offsetX, 20);
    equal(shadow.offsetY, 5);
    equal(shadow.blur, 10);
  });

  test('initializing with string', function() {
    ok(fabric.Shadow);

    // old text-shadow definition - color offsetX offsetY blur
    var shadow1 = new fabric.Shadow('rgba(0,0,255,0.5) 10px 20px 5px');

    equal(shadow1.color, 'rgba(0,0,255,0.5)');
    equal(shadow1.offsetX, 10);
    equal(shadow1.offsetY, 20);
    equal(shadow1.blur, 5);

    var shadow2 = new fabric.Shadow('rgb(0,0,255) 10px 20px ');

    equal(shadow2.color, 'rgb(0,0,255)');
    equal(shadow2.offsetX, 10);
    equal(shadow2.offsetY, 20);
    equal(shadow2.blur, 0);

    var shadow3 = new fabric.Shadow('#00FF00 30 10 ');

    equal(shadow3.color, '#00FF00');
    equal(shadow3.offsetX, 30);
    equal(shadow3.offsetY, 10);
    equal(shadow3.blur, 0);

    var shadow4 = new fabric.Shadow(' #FF0000 10px');

    equal(shadow4.color, '#FF0000');
    equal(shadow4.offsetX, 10);
    equal(shadow4.offsetY, 0);
    equal(shadow4.blur, 0);

    var shadow5 = new fabric.Shadow('#000000');

    equal(shadow5.color, '#000000');
    equal(shadow5.offsetX, 0);
    equal(shadow5.offsetY, 0);
    equal(shadow5.blur, 0);

    // new text-shadow definition - offsetX offsetY blur color
    var shadow6 = new fabric.Shadow('10px 20px 5px rgba(0,0,255,0.5)');

    equal(shadow6.color, 'rgba(0,0,255,0.5)');
    equal(shadow6.offsetX, 10);
    equal(shadow6.offsetY, 20);
    equal(shadow6.blur, 5);

    var shadow7 = new fabric.Shadow('10 20 5px #00FF00');

    equal(shadow7.color, '#00FF00');
    equal(shadow7.offsetX, 10);
    equal(shadow7.offsetY, 20);
    equal(shadow7.blur, 5);

    var shadow8 = new fabric.Shadow('10px 20px rgb(0,0,255)');

    equal(shadow8.color, 'rgb(0,0,255)');
    equal(shadow8.offsetX, 10);
    equal(shadow8.offsetY, 20);
    equal(shadow8.blur, 0);

    var shadow9 = new fabric.Shadow(' 10px #FF0000 ');

    equal(shadow9.color, '#FF0000');
    equal(shadow9.offsetX, 10);
    equal(shadow9.offsetY, 0);
    equal(shadow9.blur, 0);

    var shadow10 = new fabric.Shadow('  #FF0000 ');

    equal(shadow10.color, '#FF0000');
    equal(shadow10.offsetX, 0);
    equal(shadow10.offsetY, 0);
    equal(shadow10.blur, 0);

    var shadow11 = new fabric.Shadow('');

    equal(shadow11.color, 'rgb(0,0,0)');
    equal(shadow11.offsetX, 0);
    equal(shadow11.offsetY, 0);
    equal(shadow11.blur, 0);
  });

  test('properties', function() {
    var shadow = new fabric.Shadow();

    equal(shadow.blur, 0);
    equal(shadow.color, 'rgb(0,0,0)');
    equal(shadow.offsetX, 0);
    equal(shadow.offsetY, 0);
  });

  test('toString', function() {
    var shadow = new fabric.Shadow();
    ok(typeof shadow.toString == 'function');

    equal(shadow.toString(), '0px 0px 0px rgb(0,0,0)');
  });

  test('toObject', function() {
    var shadow = new fabric.Shadow();
    ok(typeof shadow.toObject == 'function');

    var object = shadow.toObject();
    equal(JSON.stringify(object), '{"color":"rgb(0,0,0)","blur":0,"offsetX":0,"offsetY":0}');
  });

  test('toObject without default value', function() {
    var shadow = new fabric.Shadow();
    shadow.includeDefaultValues = false;

    equal(JSON.stringify(shadow.toObject()), '{}');

    shadow.color = 'red';
    equal(JSON.stringify(shadow.toObject()), '{"color":"red"}');

    shadow.offsetX = 15;
    equal(JSON.stringify(shadow.toObject()), '{"color":"red","offsetX":15}');
  });

  test('toSVG', function() {
    // reset uid
    fabric.Object.__uid = 0;

    var shadow = new fabric.Shadow({color: '#FF0000', offsetX: 10, offsetY: -10, blur: 2});
    var object = new fabric.Object({fill: '#FF0000'});

    equal(shadow.toSVG(object), '<filter id="SVGID_0" y="-40%" height="180%" x="-40%" width="180%" >\n\t<feGaussianBlur in="SourceGraphic" stdDeviation="1" result="blurOut"></feGaussianBlur>\n\t<feColorMatrix result="matrixOut" in="blurOut" type="matrix" values="1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0.30 0" /></feColorMatrix >\n\t<feOffset dx="10" dy="-10"></feOffset>\n\t<feMerge>\n\t\t<feMergeNode></feMergeNode>\n\t\t<feMergeNode in="SourceGraphic"></feMergeNode>\n\t</feMerge>\n</filter>\n');

    shadow.color = '#000000';
    equal(shadow.toSVG(object), '<filter id="SVGID_0" y="-40%" height="180%" x="-40%" width="180%" >\n\t<feGaussianBlur in="SourceAlpha" stdDeviation="1" result="blurOut"></feGaussianBlur>\n\t<feColorMatrix result="matrixOut" in="blurOut" type="matrix" values="1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0.30 0" /></feColorMatrix >\n\t<feOffset dx="10" dy="-10"></feOffset>\n\t<feMerge>\n\t\t<feMergeNode></feMergeNode>\n\t\t<feMergeNode in="SourceGraphic"></feMergeNode>\n\t</feMerge>\n</filter>\n');
  });

})();


text:
package im.actor.runtime.actors;

public interface Cancellable {
    void cancel();
}