text:
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations.Schema;
using System.Data;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CodeOnlyStoredProcedure;
using CodeOnlyStoredProcedure.DataTransformation;
using CodeOnlyStoredProcedure.RowFactory;
using FluentAssertions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;

#if NET40
namespace CodeOnlyTests.Net40.RowFactory
#else
namespace CodeOnlyTests.RowFactory
#endif
{
    [TestClass]
    public class ComplexTypeRowFactoryTests
    {
        [TestClass]
        public class ParseRows
        {
            [TestMethod]
            public void CancelsWhenTokenCanceledBeforeExecuting()
            {
                var reader  = new Mock<IDataReader>();

                var cts = new CancellationTokenSource();
                cts.Cancel();

                var toTest = new ComplexTypeRowFactory<SingleResultSet>();
                toTest.Invoking(f => f.ParseRows(reader.Object, Enumerable.Empty<IDataTransformer>(), cts.Token))
                      .ShouldThrow<OperationCanceledException>("the operation was cancelled");

                reader.Verify(d => d.Read(), Times.Never);
            }

            [TestMethod]
            public void CancelsWhenTokenCanceled()
            {
                var sema    = new SemaphoreSlim(0, 1);
                var values = new Dictionary<string, object>
                {
                    { "MyRenamedColumn", "Hello, World!" }
                };

                var reader = CreateDataReader(values);
                var execCount = 0;

                Mock.Get(reader)
                    .Setup(d => d.Read())
                    .Callback(() =>
                    {
                        sema.Release();
                        Thread.Sleep(100);
                        ++execCount;
                    })
                    .Returns(() => execCount == 1);

                var cts = new CancellationTokenSource();

                var toTest = new ComplexTypeRowFactory<RenamedColumn>();
                var task = Task.Factory.StartNew(() => toTest.Invoking(f => f.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), cts.Token))
                                                             .ShouldThrow<OperationCanceledException>("the operation was cancelled"),
                                                 cts.Token);

                sema.Wait(TimeSpan.FromMilliseconds(250));
                cts.Cancel();

                task.Wait(TimeSpan.FromMilliseconds(250)).Should().BeTrue();
            }

            [TestMethod]
            public void ReturnsSingleResultSetOneRow()
            {
                var values = new Dictionary<string, object>
                {
                    { "String",  "Hello, World!"           },
                    { "Double",  42.0                      },
                    { "Decimal", 100M                      },
                    { "Int",     99                        },
                    { "Long",    1028130L                  },
                    { "Date",    new DateTime(1982, 1, 31) },
                    { "FooBar",  (int)FooBar.Bar           }
                };

                var reader = CreateDataReader(values);

                var toTest = new ComplexTypeRowFactory<SingleResultSet>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                res.Should().ContainSingle().Which
                   .ShouldBeEquivalentTo(
                   new SingleResultSet
                   {
                       String  = "Hello, World!",
                       Double  = 42.0,
                       Decimal = 100M,
                       Int     = 99,
                       Long    = 1028130L,
                       Date    = new DateTime(1982, 1, 31),
                       FooBar  = FooBar.Bar
                   });
            }

            [TestMethod]
            public void ReturnsSingleResultSetOneRowWithStringEnumValue()
            {
                var values = new Dictionary<string, object>
                {
                    { "String",  "Hello, World!"           },
                    { "Double",  42.0                      },
                    { "Decimal", 100M                      },
                    { "Int",     99                        },
                    { "Long",    1028130L                  },
                    { "Date",    new DateTime(1982, 1, 31) },
                    { "FooBar",  "Bar"                     }
                };

                var reader = CreateDataReader(values);

                var toTest = new ComplexTypeRowFactory<SingleResultSet>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                res.Should().ContainSingle().Which
                   .ShouldBeEquivalentTo(
                   new SingleResultSet
                   {
                       String  = "Hello, World!",
                       Double  = 42.0,
                       Decimal = 100M,
                       Int     = 99,
                       Long    = 1028130L,
                       Date    = new DateTime(1982, 1, 31),
                       FooBar  = FooBar.Bar
                   });
            }

            [TestMethod]
            public void TestExecuteHandlesRenamedColumns()
            {
                var values = new Dictionary<string, object>
                {
                    { "MyRenamedColumn", "Hello, World!" }
                };

                var reader = CreateDataReader(values);

                var toTest = new ComplexTypeRowFactory<RenamedColumn>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                res.Should().ContainSingle().Which
                   .ShouldBeEquivalentTo(
                   new RenamedColumn
                   {
                       Column = "Hello, World!"
                   });
            }

            [TestMethod]
            public void ConvertsDbNullToNullValues()
            {
                var values = new Dictionary<string, object>
                {
                    { "Name",           null },
                    { "NullableDouble", null },
                    { "NullableInt",    null }
                };

                var reader = CreateDataReader(values);
                Mock.Get(reader).Setup(r => r.GetFieldType(0)).Returns(typeof(string));
                Mock.Get(reader).Setup(r => r.GetFieldType(1)).Returns(typeof(double));
                Mock.Get(reader).Setup(r => r.GetFieldType(2)).Returns(typeof(int));
                Mock.Get(reader).Setup(r => r.IsDBNull(It.IsAny<int>())).Returns(true);

                var toTest = new ComplexTypeRowFactory<NullableColumns>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                // all values are null by default
                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new NullableColumns());
            }

            [TestMethod]
            public void ThrowsIfMappedColumnDoesNotExistInResultSet()
            {
                var values = new Dictionary<string, object>
                {
                    { "OtherColumnName", null }
                };

                var reader = CreateDataReader(values);
                Mock.Get(reader).Setup(r => r.GetFieldType(0)).Returns(typeof(string));

                var toTest = new ComplexTypeRowFactory<SingleColumn>();

                toTest.Invoking(f => f.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None))
                      .ShouldThrow<StoredProcedureResultsException>("one of the mapped columns isn't returned")
                      .WithMessage("No column with name Column was found in the result set for type SingleColumn.\nThis property will be ignored if it is decorated with a NotMappedAttribute.\nYou can also map the property to a different column in the result set with the ColumnAttribute.\nIf the stored procedure can sometimes return the column, decorate the column with the OptionalAttribute.", "we should provide friendly error messages");
            }

            [TestMethod]
            public void TransformsValueWhenPropertyDecoratedWithTransformer()
            {
                var values = new Dictionary<string, object>
                {
                    { "Name", "Hello, world!" }
                };

                var reader = CreateDataReader(values);
                var toTest = new ComplexTypeRowFactory<WithStaticValue>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                res.Should().ContainSingle().Which.Name.Should().Be("Foobar", "the database value should be passed through the DataTransformerAttribute");
            }

            [TestMethod]
            public void TransformsDBNullValueWhenPropertyDecoratedWithTransformer()
            {
                var values = new Dictionary<string, object>
                {
                    { "Name", "Hello, world!" }
                };

                var reader = CreateDataReader(values);
                Mock.Get(reader).Setup(r => r.GetFieldType(0)).Returns(typeof(string));

                var toTest = new ComplexTypeRowFactory<WithStaticValue>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                res.Should().ContainSingle().Which.Name.Should().Be("Foobar", "the database value should be passed through the DataTransformerAttribute");
            }

            [TestMethod]
            public void TransformsRenamedColumnWhenPropertyDecoratedWithTransformer()
            {
                var values = new Dictionary<string, object>
                {
                    { "MyRenamedColumn", "Hello, world!" }
                };

                var reader = CreateDataReader(values);
                var toTest = new ComplexTypeRowFactory<RenamedColumnWithStaticValue>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                res.Should().ContainSingle().Which.Name.Should().Be("Foobar", "the database value should be passed through the DataTransformerAttribute");
            }

            [TestMethod]
            public void ChainsTransformPropertyDecoratedWithTransformerAttributesInOrder()
            {
                var values = new Dictionary<string, object>
                {
                    { "Name", "Hello, world!" }
                };

                var reader = CreateDataReader(values);
                var toTest = new ComplexTypeRowFactory<WithStaticValueToUpper>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                res.Should().ContainSingle().Which.Name.Should().Be("IS UPPER?", "the database value should be passed through the DataTransformerAttributes in the order they specify");
            }

            [TestMethod]
            public void IDataTransformerTransformsData()
            {
                var values = new Dictionary<string, object>
                {
                    { "Column", "Hello, world!" }
                };

                var reader = CreateDataReader(values);
                var toTest = new ComplexTypeRowFactory<SingleColumn>();
                var res    = toTest.ParseRows(reader,
                                              new IDataTransformer[] { new StaticTransformer { Result = "Foobar" } },
                                              CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new SingleColumn { Column = "Foobar" }, "the column should be transformed");
            }

            [TestMethod]
            public void IDataTransformerTransformsDBNullValue()
            {
                var values = new Dictionary<string, object>
                {
                    { "Column", null }
                };

                var reader = CreateDataReader(values);

                Mock.Get(reader).Setup(r => r.GetFieldType(0)).Returns(typeof(string));

                var toTest = new ComplexTypeRowFactory<SingleColumn>();
                var res    = toTest.ParseRows(reader,
                                              new IDataTransformer[] { new StaticTransformer { Result = "Foobar" } },
                                              CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new SingleColumn { Column = "Foobar" }, "the column should be transformed");
            }

            [TestMethod]
            public void IDataTransformerTransformsDataWithRenamedColumn()
            {
                var values = new Dictionary<string, object>
                {
                    { "MyRenamedColumn", "Hello, world!" }
                };

                var reader = CreateDataReader(values);
                var toTest = new ComplexTypeRowFactory<RenamedColumn>();
                var res    = toTest.ParseRows(reader,
                                              new IDataTransformer[] { new StaticTransformer { Result = "Foobar" } },
                                              CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new RenamedColumn { Column = "Foobar" }, "the column should be transformed");
            }

            [TestMethod]
            public void IDataTransformerNotCalledWhenCanNotTransformValue()
            {
                var values = new Dictionary<string, object>
                {
                    { "Column", "Hello, world!" }
                };

                var reader = CreateDataReader(values, true);
                var toTest = new ComplexTypeRowFactory<SingleColumn>();
                var res    = toTest.ParseRows(reader,
                                              new IDataTransformer[] { new NeverTransformer() },
                                              CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new SingleColumn { Column = "Hello, world!" }, "the column should not be transformed");
            }

            [TestMethod]
            public void IDataTransformerNotCalledWhenCanNotTransformValueWithDBNullValue()
            {
                var values = new Dictionary<string, object>
                {
                    { "Column", null }
                };

                var reader = CreateDataReader(values, true);
                Mock.Get(reader).Setup(r => r.GetFieldType(0)).Returns(typeof(string));
                var toTest = new ComplexTypeRowFactory<SingleColumn>();
                var res    = toTest.ParseRows(reader,
                                              new IDataTransformer[] { new NeverTransformer() },
                                              CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new SingleColumn(), "the column should not be transformed");
            }

            [TestMethod]
            public void IDataTransformerNotCalledWhenCanNotTransformValueWithRenamedColumn()
            {
                var values = new Dictionary<string, object>
                {
                    { "MyRenamedColumn", "Hello, world!" }
                };

                var reader = CreateDataReader(values, true);
                var toTest = new ComplexTypeRowFactory<RenamedColumn>();
                var res    = toTest.ParseRows(reader,
                                              new IDataTransformer[] { new NeverTransformer() },
                                              CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new RenamedColumn { Column = "Hello, world!" }, "the column should not be transformed");
            }

            [TestMethod]
            public void DoesNotPassNullableToTransformationAttributes()
            {
                var values = new Dictionary<string, object>
                {
                    { "Value", 1.0 },
                    { "FooBar", 4L }
                };

                var reader = CreateDataReader(values, true);

                var toTest = new ComplexTypeRowFactory<NullableChecker>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new NullableChecker { Value = 1.0, FooBar = FooBar.Foo });
            }

            [TestMethod]
            public void DoesNotPassNullableToTransformationAttributesWhenPassedNull()
            {
                var values = new Dictionary<string, object>
                {
                    { "Value", null },
                    { "FooBar", null }
                };

                var reader = CreateDataReader(values, true);
                Mock.Get(reader).Setup(r => r.GetFieldType(0)).Returns(typeof(int));
                Mock.Get(reader).Setup(r => r.GetFieldType(1)).Returns(typeof(double));
                var xformer = new Mock<IDataTransformer>();

                var toTest = new ComplexTypeRowFactory<NullableChecker>();
                var res    = toTest.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new NullableChecker());
            }

            [TestMethod]
            public void DoesNotPassNullableToTransformers()
            {
                var values = new Dictionary<string, object>
                {
                    { "Value", 42.0 },
                    { "FooBar", 6 }
                };

                var reader  = CreateDataReader(values, true);
                Mock.Get(reader).Setup(r => r.GetFieldType(0)).Returns(typeof(int));
                Mock.Get(reader).Setup(r => r.GetFieldType(1)).Returns(typeof(double));
                var xformer = new Mock<IDataTransformer>();

                xformer.Setup(x => x.CanTransform(It.IsAny<object>(), typeof(double), true, It.Is<IEnumerable<Attribute>>(attrs => attrs != null)))
                       .Returns(true)
                       .Verifiable();
                xformer.Setup(x => x.Transform(It.IsAny<object>(), typeof(double), true, It.Is<IEnumerable<Attribute>>(attrs => attrs != null)))
                       .Returns<object, Type, bool, IEnumerable<Attribute>>((o, t, b, e) =>
                       {
                           Assert.IsTrue(b, "isNullable must be true for a Nullable<T> property");
                           if (t.IsGenericType)
                               Assert.AreNotEqual(typeof(Nullable<>), t.GetGenericTypeDefinition(), "A Nullable<T> type can not be passed to a DataTransformerAttributeBase");

                           return o;
                       })
                       .Verifiable();

                var toTest = new ComplexTypeRowFactory<NullableChecker>();
                var res    = toTest.ParseRows(reader, new[] { xformer.Object }, CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new NullableChecker { Value = 42, FooBar = FooBar.Bar });
            }

            [TestMethod]
            public void DoesNotPassNullableToTransformersWhenPassedNull()
            {
                var values = new Dictionary<string, object>
                {
                    { "Value", null },
                    { "FooBar", null }
                };

                var reader  = CreateDataReader(values, true);
                Mock.Get(reader).Setup(r => r.GetFieldType(0)).Returns(typeof(int));
                Mock.Get(reader).Setup(r => r.GetFieldType(1)).Returns(typeof(double));
                var xformer = new Mock<IDataTransformer>();

                xformer.Setup(x => x.CanTransform(It.IsAny<object>(), typeof(double), true, It.Is<IEnumerable<Attribute>>(attrs => attrs != null)))
                       .Returns(true)
                       .Verifiable();
                xformer.Setup(x => x.Transform(It.IsAny<object>(), typeof(double), true, It.Is<IEnumerable<Attribute>>(attrs => attrs != null)))
                       .Returns<object, Type, bool, IEnumerable<Attribute>>((o, t, b, e) =>
                       {
                           b.Should().BeTrue("isNullable must be true for a Nullable<T> property");

                           if (t.IsGenericType)
                               Assert.AreNotEqual(typeof(Nullable<>), t.GetGenericTypeDefinition(), "A Nullable<T> type can not be passed to a DataTransformerAttributeBase");

                           return o;
                       })
                       .Verifiable();

                var toTest = new ComplexTypeRowFactory<NullableChecker>();
                var res    = toTest.ParseRows(reader, new[] { xformer.Object }, CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new NullableChecker());
            }

            [TestMethod]
            public void TypedTransformerTransformsStrings()
            {
                var values = new Dictionary<string, object>
                {
                    { "Column", "Foo" }
                };

                var reader  = CreateDataReader(values, false);
                var xformer = new Mock<IDataTransformer<string>>();

                xformer.Setup(x => x.Transform("Foo", It.Is<IEnumerable<Attribute>>(attrs => attrs != null)))
                       .Returns("Bar");

                var toTest = new ComplexTypeRowFactory<SingleColumn>();
                var res    = toTest.ParseRows(reader, new[] { xformer.Object }, CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new SingleColumn { Column = "Bar" });
                xformer.Verify(x => x.CanTransform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
                xformer.Verify(x => x.Transform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
            }

            [TestMethod]
            public void TypedTransformerNotUsedForColumnWithDifferentType()
            {
                var values = new Dictionary<string, object>
                {
                    { "Column", "Foo" }
                };

                var reader  = CreateDataReader(values, false);
                var xformer = new Mock<IDataTransformer<int>>();

                var toTest = new ComplexTypeRowFactory<SingleColumn>();
                var res    = toTest.ParseRows(reader, new[] { xformer.Object }, CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new SingleColumn { Column = "Foo" });
                xformer.Verify(x => x.CanTransform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
                xformer.Verify(x => x.Transform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
                xformer.Verify(x => x.Transform(It.IsAny<int>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
            }

            [TestMethod]
            public void TypedTransformerTransformsValueTypes()
            {
                var values = new Dictionary<string, object>
                {
                    { "Value", 13 }
                };

                var reader  = CreateDataReader(values, false);
                var xformer = new Mock<IDataTransformer<int>>();
                xformer.Setup(x => x.Transform(13, It.IsAny<IEnumerable<Attribute>>()))
                       .Returns(42);

                var toTest = new ComplexTypeRowFactory<ConvertToInt>();
                var res    = toTest.ParseRows(reader, new[] { xformer.Object }, CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new ConvertToInt { Value = 42 });
                xformer.Verify(x => x.CanTransform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
                xformer.Verify(x => x.Transform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
            }

            [TestMethod]
            public void TypedTransformerTransformsConvertedValueTypes()
            {
                var values = new Dictionary<string, object>
                {
                    { "Value", 13M }
                };

                var reader  = CreateDataReader(values, false);
                var xformer = new Mock<IDataTransformer<double>>();
                xformer.Setup(x => x.Transform(13, It.IsAny<IEnumerable<Attribute>>()))
                       .Returns(42);

                var toTest = new ComplexTypeRowFactory<ConvertToDouble>();
                var res    = toTest.ParseRows(reader, new[] { xformer.Object }, CancellationToken.None);

                res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new ConvertToDouble { Value = 42 });
                xformer.Verify(x => x.CanTransform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
                xformer.Verify(x => x.Transform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
            }

            [TestMethod]
            public void MultipleTypedTransformersDoNotCauseValuesToBeRetrievedBoxed()
            {
                var values = new Dictionary<string, object>
                {
                    { "Value",  42M  },
                    { "FooBar", "Not a Match" }
                };

                var reader = CreateDataReader(values);
                var xformer1 = new Transformer<string>(Tuple.Create("Not a Match", "Bar"));
                var xformer2 = new Transformer<double>(Tuple.Create(42.0, 84.0));

                var toTest = new ComplexTypeRowFactory<MultipleChecker>();
                var res    = toTest.ParseRows(reader, new IDataTransformer[] { xformer1, xformer2 }, CancellationToken.None);

                res.Should().ContainSingle().Which
                   .ShouldBeEquivalentTo(
                   new MultipleChecker
                   {
                       Value = 84.0,
                       FooBar = FooBar.Bar
                   });
            }

            [TestMethod]
            public void AllTypedTransformersDoNotCauseValuesToBeRetrievedBoxed()
            {
                var values = new Dictionary<string, object>
                {
                    { "String",  "Hello, World!"           },
                    { "Double",  42.0                      },
                    { "Decimal", 100M                      },
                    { "Int",     99                        },
                    { "Long",    1028130L                  },
                    { "Date",    new DateTime(1982, 1, 31) },
                    { "FooBar",  "Bar"                     }
                };

                var reader = CreateDataReader(values);
                var xformer1 = new Transformer<string>(Tuple.Create("Hello, World!", "Hello, World!Hello, World!"));
                var xformer2 = new Transformer<int>(Tuple.Create(99, 33));
                var xformer3 = new Transformer<double>(Tuple.Create(42.0, 45.0));

                var toTest = new ComplexTypeRowFactory<SingleResultSet>();
                var res    = toTest.ParseRows(reader, new IDataTransformer[] { xformer1, xformer2, xformer3 }, CancellationToken.None);

                res.Should().ContainSingle().Which
                   .ShouldBeEquivalentTo(
                   new SingleResultSet
                   {
                       String = "Hello, World!Hello, World!",
                       Double = 45.0,
                       Decimal = 100M,
                       Int = 33,
                       Long = 1028130L,
                       Date = new DateTime(1982, 1, 31),
                       FooBar = FooBar.Bar
                   });
            }

            [TestMethod]
            public void ReturnsNewRowWithoutOptionalProperty()
            {
                var data = new Dictionary<string, object>
                {
                    { "Name", "Foo" },
                    { "Price", 13.3 },
                    { "Rename", "Bar" }
                };

                var reader = CreateDataReader(data);
                var toTest = new ComplexTypeRowFactory<Row>().ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Single().Should().NotBeNull("one row should have been returned")
                      .And.BeOfType<Row>()
                      .And.Subject
                      .Should().Match<Row>(r => r.Name == "Foo", "Name column returned Foo")
                      .And.Match<Row>(r => r.Price == 13.3, "Price column returned 13.3")
                      .And.Match<Row>(r => r.Other == "Bar", "Other column returned Bar");
            }

            [TestMethod]
            public void SetsOptionalProperties()
            {
                var data = new Dictionary<string, object>
                {
                    { "Name", "Foo" },
                    { "Price", 13.3 },
                    { "Rename", "Bar" },
                    { "Optional", true }
                };

                var reader = CreateDataReader(data);
                var toTest = new ComplexTypeRowFactory<Row>().ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Single().Should().NotBeNull("one row should have been returned")
                      .And.BeOfType<Row>()
                      .And.Subject
                      .Should().Match<Row>(r => r.Name == "Foo", "Name column returned Foo")
                      .And.Match<Row>(r => r.Price == 13.3, "Price column returned 13.3")
                      .And.Match<Row>(r => r.Other == "Bar", "Other column returned Bar")
                      .And.Match<Row>(r => r.Optional, "Optional column returned true");
            }

            [TestMethod]
            public void WithoutPropertyInResultsSpecifiesThemInException()
            {
                var data = new Dictionary<string, object>
                {
                    { "Name", "Foo" }
                };

                var reader = CreateDataReader(data);
                var toTest = new ComplexTypeRowFactory<Row>();

                toTest.Invoking(t => t.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None))
                      .ShouldThrow<StoredProcedureResultsException>("not all columns are returned")
                      .WithMessage("No columns with name Price or Rename were found in the result set for type Row.\nThis property will be ignored if it is decorated with a NotMappedAttribute.\nYou can also map the property to a different column in the result set with the ColumnAttribute.\nIf the stored procedure can sometimes return the column, decorate the column with the OptionalAttribute.", "we should provide friendly error messages");
            }

            [TestMethod]
            public void ExceptionsIncludeUsefulInformation()
            {
                var data = new Dictionary<string, object>
                {
                    { "Name", "Foo" },
                    { "Price", "Blah" },
                    { "Rename", "Bar" }
                };

                var reader = CreateDataReader(data);
                var toTest = new ComplexTypeRowFactory<Row>();

                toTest.Invoking(t => t.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None))
                      .ShouldThrow<StoredProcedureColumnException>("because the property result type does not match")
                      .WithMessage("Error setting [Double] Price. Stored Procedure returns [String].");
            }

            [TestMethod]
            public void ExceptionIncludesTypeForValueReturned()
            {
                var data = new Dictionary<string, object>
                {
                    { "Name", "Foo" },
                    { "Price", 42M },
                    { "Rename", "Bar" }
                };

                var reader = CreateDataReader(data);
                var toTest = new ComplexTypeRowFactory<Row>();

                toTest.Invoking(t => t.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None))
                      .ShouldThrow<StoredProcedureColumnException>("because the property result type does not match")
                      .WithMessage("Error setting [Double] Price. Stored Procedure returns [Decimal].");
            }

            [TestMethod]
            public void WithPropertyTransformerWillCallTransformer()
            {
                var data = new Dictionary<string, object>
                {
                    { "Name", "Foo" },
                    { "Price", 13.3 },
                    { "Rename", "Bar" },
                    { "Transformed", 42.0 }
                };

                var reader = CreateDataReader(data);
                var toTest = new ComplexTypeRowFactory<Row>().ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Single().Should().NotBeNull("one row should have been returned")
                      .And.BeOfType<Row>()
                      .And.Subject
                      .Should().Match<Row>(r => r.Name == "Foo", "Name column returned Foo")
                      .And.Match<Row>(r => r.Price == 13.3, "Price column returned 13.3")
                      .And.Match<Row>(r => r.Other == "Bar", "Other column returned Bar")
                      .And.Match<Row>(r => r.Transformed == 42M, "Transformed column returned 42.0 double, which should have been converted to 42 decimal");
            }

            [TestMethod]
            public void MappedInterface_ReturnsImplementation()
            {
                using (GlobalSettings.UseTestInstance())
                {
                    StoredProcedure.MapResultType<Interface, InterfaceImpl>();
                    var data = new Dictionary<string, object>
                    {
                        { "Id", "42" }
                    };

                    var reader = CreateDataReader(data);
                    var toTest = new ComplexTypeRowFactory<Interface>().ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                    toTest.Single().Should().NotBeNull("one row should have been returned")
                          .And.BeOfType<InterfaceImpl>()
                          .And.Match<Interface>(i => i.Id == "42");
                }
            }

            [TestMethod]
            public void StrongTypedDataTransformers_RetrievesValuesUnboxed()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "Value", 42 }
                    });

                var toTest = new ComplexTypeRowFactory<WithStrongTypedDataTransformer>().ParseRows(
                    reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Single().Should().NotBeNull("one row should have been returned")
                      .And.Match<WithStrongTypedDataTransformer>(i => i.Value == 84, "the transformer doubles the result from the database");
            }

            [TestMethod]
            public void EnumPropertiesWithWrongUnderlyingTypeWillThrowIfNotMarkedWithConvert()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "FooBar", 4L }
                    });

                var toTest = new ComplexTypeRowFactory<EnumValue>();

                toTest.Invoking(t => t.ParseRows(reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None))
                      .ShouldThrow<StoredProcedureColumnException>("the stored procedure returns a different type than the underlying type of the enum.");
            }

            [TestMethod]
            public void EnumPropertiesWillConvertNumericTypes()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "FooBar", 4L }
                    });

                var toTest = new ComplexTypeRowFactory<EnumValueTypesConverted>().ParseRows(
                    reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Single().Should().NotBeNull("one row should have been returned")
                      .And.Match<EnumValueTypesConverted>(i => i.FooBar == FooBar.Foo, "The value should be converted to the underlying type.");
            }

            [TestMethod]
            public void EnumPropertiesCallTypedConverterTransformNumericResult()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "FooBar", 4L }
                    });

                var xformer = new Mock<IDataTransformer<FooBar>>();
                xformer.Setup(x => x.Transform(FooBar.Foo, It.IsAny<IEnumerable<Attribute>>()))
                       .Returns(FooBar.Bar);

                var toTest = new ComplexTypeRowFactory<EnumValueTypesConverted>().ParseRows(
                    reader, new[] { xformer.Object }, CancellationToken.None);

                toTest.Single().Should().NotBeNull("one row should have been returned")
                      .And.Match<EnumValueTypesConverted>(i => i.FooBar == FooBar.Bar, "The value should be altered by the Transformer.");

                xformer.Verify(x => x.CanTransform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
                xformer.Verify(x => x.Transform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
            }

            [TestMethod]
            public void EnumPropertiesCallTypedConverterTransformStringResult()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "FooBar", "Foo" }
                    });

                var xformer = new Mock<IDataTransformer<FooBar>>();
                xformer.Setup(x => x.Transform(FooBar.Foo, It.IsAny<IEnumerable<Attribute>>()))
                       .Returns(FooBar.Bar);

                var toTest = new ComplexTypeRowFactory<EnumValueTypesConverted>().ParseRows(
                    reader, new[] { xformer.Object }, CancellationToken.None);

                toTest.Single().Should().NotBeNull("one row should have been returned")
                      .And.Match<EnumValueTypesConverted>(i => i.FooBar == FooBar.Bar, "The value should be altered by the Transformer.");

                xformer.Verify(x => x.CanTransform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
                xformer.Verify(x => x.Transform(It.IsAny<object>(), It.IsAny<Type>(), It.IsAny<bool>(), It.IsAny<IEnumerable<Attribute>>()), Times.Never());
            }

            [TestMethod]
            public void NumericTypeWillConvertToTrueBoolIfMarkedWithConvert()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 1 }
                    });

                var toTest = new ComplexTypeRowFactory<ConvertToBool>().ParseRows(
                    reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToBool>(i => i.IsEnabled, "the property should be converted to bool");
            }

            [TestMethod]
            public void NumericTypeWillConvertToFalseBoolIfMarkedWithConvert()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 0 }
                    });

                var toTest = new ComplexTypeRowFactory<ConvertToBool>().ParseRows(
                    reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToBool>(i => !i.IsEnabled, "the property should be converted to bool");
            }

            [TestMethod]
            public void NumericTypeWillConvertToTrueBoolIfMarkedWithConvert_WhenTransformerPassed()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 1 }
                    }, true);

                var toTest = new ComplexTypeRowFactory<ConvertToBool>().ParseRows(
                    reader, new[] { Mock.Of<IDataTransformer>() }, CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToBool>(i => i.IsEnabled, "the property should be converted to bool");
            }

            [TestMethod]
            public void NumericTypeWillConvertToFalseBoolIfMarkedWithConvert_WhenTransformerPassed()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 0L }
                    }, true);

                var toTest = new ComplexTypeRowFactory<ConvertToBool>().ParseRows(
                    reader, new[] { Mock.Of<IDataTransformer>() }, CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToBool>(i => !i.IsEnabled, "the property should be converted to bool");
            }

            [TestMethod]
            public void NumericTypeWillConvertToTrueBoolIfMarkedWithConvert_WhenTypedTransformerPassed()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 0 }
                    }, true);

                var xformer = new Mock<IDataTransformer<bool>>();
                xformer.Setup(x => x.Transform(false, It.IsAny<IEnumerable<Attribute>>()))
                       .Returns(true);

                var toTest = new ComplexTypeRowFactory<ConvertToBool>().ParseRows(
                    reader, new[] { xformer.Object }, CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToBool>(i => i.IsEnabled == true, "the property should be converted to bool");
            }

            [TestMethod]
            public void NumericTypeWillConvertToFalseBoolIfMarkedWithConvert_WhenTypedTransformerPassed()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 0L }
                    }, true);

                var toTest = new ComplexTypeRowFactory<ConvertToBool>().ParseRows(
                    reader, new[] { Mock.Of<IDataTransformer<bool>>() }, CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToBool>(i => !i.IsEnabled, "the property should be converted to bool");
            }

            [TestMethod]
            public void NumericTypeWillConvertToTrueNullableBoolIfMarkedWithConvert()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 1M }
                    });

                var toTest = new ComplexTypeRowFactory<ConvertToNullableBool>().ParseRows(
                    reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToNullableBool>(i => i.IsEnabled.Value, "the property should be converted to bool");
            }

            [TestMethod]
            public void NullNumericTypeWillConvertToNullNullableBoolIfMarkedWithConvert()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", null }
                    }, true);
                Mock.Get(reader).Setup(rdr => rdr.GetFieldType(0)).Returns(typeof(int));

                var toTest = new ComplexTypeRowFactory<ConvertToNullableBool>().ParseRows(
                    reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToNullableBool>(i => !i.IsEnabled.HasValue, "the property should not have a value");
            }

            [TestMethod]
            public void NumericTypeWillConvertToFalseNullableBoolIfMarkedWithConvert()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 0 }
                    });

                var toTest = new ComplexTypeRowFactory<ConvertToNullableBool>().ParseRows(
                    reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToNullableBool>(i => !i.IsEnabled.Value, "the property should be converted to bool");
            }

            [TestMethod]
            public void NumericTypeWillConvertToTrueNullableBoolIfMarkedWithConvert_WhenTransformerPassed()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 1.0 }
                    }, true);

                var toTest = new ComplexTypeRowFactory<ConvertToNullableBool>().ParseRows(
                    reader, new[] { Mock.Of<IDataTransformer>() }, CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToNullableBool>(i => i.IsEnabled.Value, "the property should be converted to bool");
            }

            [TestMethod]
            public void NumericTypeWillConvertToFalseNullableBoolIfMarkedWithConvert_WhenTransformerPassed()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", 0 }
                    }, true);

                var toTest = new ComplexTypeRowFactory<ConvertToNullableBool>().ParseRows(
                    reader, new[] { Mock.Of<IDataTransformer>() }, CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToNullableBool>(i => !i.IsEnabled.Value, "the property should be converted to bool");
            }

            [TestMethod]
            public void NullNumericTypeWillConvertToNullNullableBoolIfMarkedWithConvert_WhenTransformerPassed()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "IsEnabled", null }
                    }, true);
                Mock.Get(reader).Setup(rdr => rdr.GetFieldType(0)).Returns(typeof(short));

                var toTest = new ComplexTypeRowFactory<ConvertToNullableBool>().ParseRows(
                    reader, new[] { Mock.Of<IDataTransformer>() }, CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToNullableBool>(i => !i.IsEnabled.HasValue, "the property should not have a value");
            }

            [TestMethod]
            public void TrueBoolWillConvertToNumericTypeIfMarkedWithConvert()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "Value", true }
                    });

                var toTest = new ComplexTypeRowFactory<ConvertToInt>().ParseRows(
                    reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToInt>(i => i.Value == 1, "the property should be converted from bool");
            }

            [TestMethod]
            public void FalseBoolWillConvertToNumericTypeIfMarkedWithConvert()
            {
                var reader = CreateDataReader(new Dictionary<string, object>
                    {
                        { "Value", false }
                    });

                var toTest = new ComplexTypeRowFactory<ConvertToInt>().ParseRows(
                    reader, Enumerable.Empty<IDataTransformer>(), CancellationToken.None);

                toTest.Should().ContainSingle("because one row is returned").Which
                    .Should().Match<ConvertToInt>(i => i.Value == 0, "the property should be converted from bool");
            }

            [TestMethod]
            public void GlobalTransformer_UsedToTransformData()
            {
                using (GlobalSettings.UseTestInstance())
                {
                    StoredProcedure.AddGlobalTransformer(new StaticTransformer { Result = "Foobar" });

                    var values = new Dictionary<string, object>
                    {
                        { "Column", "Hello, world!" }
                    };

                    var reader = CreateDataReader(values);
                    var toTest = new ComplexTypeRowFactory<SingleColumn>();
                    var res    = toTest.ParseRows(reader, new IDataTransformer[0], CancellationToken.None);

                    res.Should().ContainSingle().Which.ShouldBeEquivalentTo(new SingleColumn { Column = "Foobar" }, "the column should be transformed");
                }
            }

            [TestMethod]
            public void GlobalTypedTransformer_UsedToTransformOnlyRequiredData()
            {
                using (GlobalSettings.UseTestInstance())
                {
                    StoredProcedure.AddGlobalTransformer(new Transformer<string>(Tuple.Create("Hello, World!", "Hello, World!Hello, World!")));
                    StoredProcedure.AddGlobalTransformer(new Transformer<int>(Tuple.Create(99, 33)));
                    StoredProcedure.AddGlobalTransformer(new Transformer<double>(Tuple.Create(42.0, 45.0)));

                    var values = new Dictionary<string, object>
                    {
                        { "String",  "Hello, World!"           },
                        { "Double",  42.0                      },
                        { "Decimal", 100M                      },
                        { "Int",     99                        },
                        { "Long",    1028130L                  },
                        { "Date",    new DateTime(1982, 1, 31) },
                        { "FooBar",  "Bar"                     }
                    };

                    var reader = CreateDataReader(values);
                    var toTest = new ComplexTypeRowFactory<SingleResultSet>();
                    var res    = toTest.ParseRows(reader, new IDataTransformer[0], CancellationToken.None);

                    res.Should().ContainSingle().Which
                       .ShouldBeEquivalentTo(
                       new SingleResultSet
                       {
                           String = "Hello, World!Hello, World!",
                           Double = 45.0,
                           Decimal = 100M,
                           Int = 33,
                           Long = 1028130L,
                           Date = new DateTime(1982, 1, 31),
                           FooBar = FooBar.Bar
                       });
                }
            }

            [TestMethod]
            public void TimeSpanAndDateTimeOffset_ColumnsReturnedFromGetValue()
            {
                var values = new Dictionary<string, object>
                {
                    { "DateTimeOffset", new DateTimeOffset(new DateTime(1984, 1, 1), TimeSpan.FromHours(1)) },
                    { "TimeSpan",       TimeSpan.FromMinutes(42) }
                };

                var reader = CreateDataReader(values, true);
                var toTest = new ComplexTypeRowFactory<ForbiddenDates>();
                var res    = toTest.ParseRows(reader, new IDataTransformer[0], CancellationToken.None);

                res.Should().ContainSingle().Which
                   .ShouldBeEquivalentTo(
                   new ForbiddenDates
                   {
                       DateTimeOffset = new DateTimeOffset(new DateTime(1984, 1, 1), TimeSpan.FromHours(1)),
                       TimeSpan       = TimeSpan.FromMinutes(42)
                   });
            }

            private static IDataReader CreateDataReader(Dictionary<string, object> values, bool setupGetValue = false)
            {
                var keys = values.Keys.OrderBy(s => s).ToList();
                var vals = values.OrderBy(kv => kv.Key).Select(kv => kv.Value).ToArray();

                var reader  = new Mock<IDataReader>();

                reader.SetupGet(r => r.FieldCount)
                      .Returns(keys.Count);
                reader.Setup(r => r.GetFieldType(It.IsAny<int>()))
                      .Returns((int i) => vals[i].GetType());
                reader.Setup(r => r.IsDBNull(It.IsAny<int>()))
                      .Returns((int i) => vals[i] == null);

                var first = true;
                reader.Setup(r => r.Read())
                      .Returns(() =>
                      {
                          if (first)
                          {
                              first = false;
                              return true;
                          }

                          return false;
                      });

                if (setupGetValue)
                {
                    reader.Setup(r => r.GetValue(It.IsAny<int>()))
                          .Returns((int i) => vals[i]);
                }

                reader.Setup(r => r.GetName(It.IsAny<int>()))
                      .Returns((int i) => keys[i]);
                reader.Setup(r => r.GetString(It.IsAny<int>()))
                      .Returns((int i) => (string)vals[i]);
                reader.Setup(r => r.GetInt16(It.IsAny<int>()))
                      .Returns((int i) => (short)vals[i]);
                reader.Setup(r => r.GetInt32(It.IsAny<int>()))
                      .Returns((int i) => (int)vals[i]);
                reader.Setup(r => r.GetInt64(It.IsAny<int>()))
                      .Returns((int i) => (long)vals[i]);
                reader.Setup(r => r.GetFloat(It.IsAny<int>()))
                      .Returns((int i) => (float)vals[i]);
                reader.Setup(r => r.GetDecimal(It.IsAny<int>()))
                      .Returns((int i) => (decimal)vals[i]);
                reader.Setup(r => r.GetDouble(It.IsAny<int>()))
                      .Returns((int i) => (double)vals[i]);
                reader.Setup(r => r.GetDateTime(It.IsAny<int>()))
                      .Returns((int i) => (DateTime)vals[i]);
                reader.Setup(r => r.GetChar(It.IsAny<int>()))
                      .Returns((int i) => (char)vals[i]);
                reader.Setup(r => r.GetByte(It.IsAny<int>()))
                      .Returns((int i) => (byte)vals[i]);
                reader.Setup(r => r.GetBoolean(It.IsAny<int>()))
                      .Returns((int i) => (bool)vals[i]);

                return reader.Object;
            }
        }

        [TestClass]
        public class MatchesColumns
        {
            [TestMethod]
            public void SingleColumnMatches_SingleResultColumn()
            {
                int leftoverColumns;
                var toTest = new ComplexTypeRowFactory<SingleColumn>();
                var result = toTest.MatchesColumns(new[] { "Column" }, out leftoverColumns);

                result.Should().BeTrue("because all required property columns were returned");
                leftoverColumns.Should().Be(0, "because only one column exists, and it is used");
            }

            [TestMethod]
            public void SingleColumnMatches_MultipleResultColumn()
            {
                int leftoverColumns;
                var toTest = new ComplexTypeRowFactory<SingleColumn>();
                var result = toTest.MatchesColumns(new[] { "Column", "Foo", "Bar" }, out leftoverColumns);

                result.Should().BeTrue("because all required property columns were returned");
                leftoverColumns.Should().Be(2, "because two of the result columns were not used");
            }

            [TestMethod]
            public void SingleColumn_ReturnsFalse_IfDoesNotMatch()
            {
                int leftoverColumns;
                var toTest = new ComplexTypeRowFactory<SingleColumn>();
                var result = toTest.MatchesColumns(new[] { "Foo", "Bar" }, out leftoverColumns);

                result.Should().BeFalse("because the required column is not in the results");
                leftoverColumns.Should().Be(2, "because 2 of the columns do not match columns on the result");
            }

            [TestMethod]
            public void RenamedColumn_ReturnsTrue_WhenDbColumnNameExists()
            {
                int leftoverColumns;
                var toTest = new ComplexTypeRowFactory<RenamedColumn>();
                var result = toTest.MatchesColumns(new[] { "MyRenamedColumn" }, out leftoverColumns);

                result.Should().BeTrue("because the property's renamed column was returned");
                leftoverColumns.Should().Be(0, "because only one column exists, and it is used");
            }

            [TestMethod]
            public void OptionalColumn_ReturnsTrue_IfOptionalColumnNotPresent()
            {
                int leftoverColumns;
                var toTest = new ComplexTypeRowFactory<WithOptional>();
                var result = toTest.MatchesColumns(new[] { "Column" }, out leftoverColumns);

                result.Should().BeTrue("because all required property columns were returned");
                leftoverColumns.Should().Be(0, "because only one column exists, and it is used");
            }

            [TestMethod]
            public void OptionalColumn_ReturnsFalse_IfRequiredColumnNotPresent()
            {
                int leftoverColumns;
                var toTest = new ComplexTypeRowFactory<WithOptional>();
                var result = toTest.MatchesColumns(new[] { "Optional" }, out leftoverColumns);

                result.Should().BeFalse("because not all required property columns were returned");
                leftoverColumns.Should().Be(0, "because only one column returned, and it is used");
            }

            [TestMethod]
            public void OptionalColumn_ReturnsTrue_IfOptionalColumnIsPresent()
            {
                int leftoverColumns;
                var toTest = new ComplexTypeRowFactory<WithOptional>();
                var result = toTest.MatchesColumns(new[] { "Column", "Optional" }, out leftoverColumns);

                result.Should().BeTrue("because all required property columns were returned");
                leftoverColumns.Should().Be(0, "because only all columns returned are used");
            }

            [TestMethod]
            public void ChildrenHierarchicalPropertyNames_NotRequired()
            {
                int leftoverColumns;
                var toTest = new ComplexTypeRowFactory<WithChildren>();
                var result = toTest.MatchesColumns(new[] { "Id" }, out leftoverColumns);

                result.Should().BeTrue("because enumerable properties can't be set, and should be ignored");
                leftoverColumns.Should().Be(0, "because only one column exists, and it is used");
            }
        }

        #region Test Helper Classes
        private class SingleResultSet
        {
            public String String { get; set; }
            public Double Double { get; set; }
            public Decimal Decimal { get; set; }
            public Int32 Int { get; set; }
            public Int64 Long { get; set; }
            public DateTime Date { get; set; }
            public FooBar FooBar { get; set; }
        }

        private class SingleColumn
        {
            public string Column { get; set; }
        }

        private class RenamedColumn
        {
            [Column("MyRenamedColumn")]
            public string Column { get; set; }
        }

        private class WithOptional
        {
            public string Column { get; set; }
            [OptionalResult]
            public string Optional { get; set; }
        }

        private class NullableColumns
        {
            public string Name { get; set; }
            public int? NullableInt { get; set; }
            public double? NullableDouble { get; set; }
        }

        private class WithStaticValue
        {
            [StaticValue(Result = "Foobar")]
            public string Name { get; set; }
        }

        private class RenamedColumnWithStaticValue
        {
            [Column("MyRenamedColumn")]
            [StaticValue(Result = "Foobar")]
            public string Name { get; set; }
        }

        private class WithStaticValueToUpper
        {
            [StaticValue(Result = "is upper?")]
            [ToUpper(1)]
            public string Name { get; set; }
        }

        private class WithInvalidTransformer
        {
            [ToUpper]
            public double Value { get; set; }
        }

        private class NullableChecker
        {
            [NullableTransformation]
            public double? Value { get; set; }
            [NullableTransformation, ConvertNumeric]
            public FooBar? FooBar { get; set; }
        }

        private class MultipleChecker
        {
            [ConvertNumeric]
            public double Value { get; set; }
            [ConvertNumeric]
            public FooBar FooBar { get; set; }
        }

        private interface Interface
        {
            string Id { get; set; }
        }

        private class InterfaceImpl : Interface
        {
            public string Id { get; set; }
        }

        private class Row
        {
            public string Name { get; set; }
            public double Price { get; set; }
            [Column("Rename")]
            public string Other { get; set; }
            [OptionalResult]
            public bool Optional { get; set; }
            [OptionalResult, ConvertNumeric]
            public decimal Transformed { get; set; }
        }

        private class WithStrongTypedDataTransformer
        {
            [StrongTypedDataTransformer]
            public uint Value { get; set; }
        }

        private class EnumValue
        {
            public FooBar FooBar { get; set; }
        }

        private class EnumValueTypesConverted
        {
            [ConvertNumeric]
            public FooBar FooBar { get; set; }
        }

        private class ConvertToBool
        {
            [ConvertNumeric]
            public bool IsEnabled { get; set; }
        }

        private class ConvertToNullableBool
        {
            [ConvertNumeric]
            public bool? IsEnabled { get; set; }
        }

        private class ConvertToInt
        {
            [ConvertNumeric]
            public int Value { get; set; }
        }

        private class ConvertToDouble
        {
            [ConvertNumeric]
            public double Value { get; set; }
        }

        private class WithChildren
        {
            public int Id { get; set; }
            public IEnumerable<SingleColumn> Children { get; set; }
        }

        private class ForbiddenDates
        {
            public DateTimeOffset DateTimeOffset { get; set; }
            public TimeSpan       TimeSpan       { get; set; }
        }

        private class StaticValueAttribute : DataTransformerAttributeBase
        {
            public object Result { get; set; }

            public override object Transform(object value, Type targetType, bool isNullable)
            {
                return Result;
            }
        }

        private class ToUpperAttribute : DataTransformerAttributeBase
        {
            public ToUpperAttribute(int order = 0)
                : base(order)
            {

            }

            public override object Transform(object value, Type targetType, bool isNullable)
            {
                return ((string)value).ToUpper();
            }
        }

        private class StaticTransformer : IDataTransformer
        {
            public string Result { get; set; }

            public bool CanTransform(object value, Type targetType, bool isNullable, IEnumerable<Attribute> propertyAttributes)
            {
                return true;
            }

            public object Transform(object value, Type targetType, bool isNullable, IEnumerable<Attribute> propertyAttributes)
            {
                return Result;
            }
        }

        private class NeverTransformer : IDataTransformer
        {
            public bool CanTransform(object value, Type targetType, bool isNullable, IEnumerable<Attribute> propertyAttributes)
            {
                return false;
            }

            public object Transform(object value, Type targetType, bool isNullable, IEnumerable<Attribute> propertyAttributes)
            {
                Assert.Fail("Transform should not be called when an IDataTransformer returns false from CanTransform");
                return null;
            }
        }

        private class NullableTransformationAttribute : DataTransformerAttributeBase
        {
            public override object Transform(object value, Type targetType, bool isNullable)
            {
                isNullable.Should().BeTrue("isNullable must be true for a Nullable<T> property");
                if (targetType.IsGenericType)
                    targetType.GetGenericTypeDefinition().Should().NotBe(typeof(Nullable<>), "a Nullable<T> type should not be passed to a DataTransformerAttributeBase");

                return value;
            }
        }

        private class StrongTypedDataTransformer : DataTransformerAttributeBase, IDataTransformerAttribute<uint>
        {
            public uint Transform(uint value)
            {
                return value * 2;
            }

            public override object Transform(object value, Type targetType, bool isNullable)
            {
                Assert.Fail("Boxed Transform method should never be called.");
                return Transform((uint)value);
            }
        }

        private class Transformer<T> : IDataTransformer<T>
        {
            private readonly Dictionary<T, T> transforms;

            public Transformer(params Tuple<T, T>[] transforms)
            {
                this.transforms = transforms.ToDictionary(t => t.Item1, t => t.Item2);
            }

            public T Transform(T value, IEnumerable<Attribute> propertyAttributes)
            {
                T result;
                if (transforms.TryGetValue(value, out result))
                    return result;

                return value;
            }

            public bool CanTransform(object value, Type targetType, bool isNullable, IEnumerable<Attribute> propertyAttributes)
            {
                throw new NotSupportedException();
            }

            public object Transform(object value, Type targetType, bool isNullable, IEnumerable<Attribute> propertyAttributes)
            {
                throw new NotSupportedException();
            }
        }

        public enum FooBar
        {
            Foo = 4,
            Bar = 6
        }
        #endregion
    }
}


text:
		<title>testing Core FX</title>

testing Core FX

		<form name="testForm">
		<input type="button" onClick="dojo.fadeOut({ node: 'foo', duration: 1000 }).play()" value="fade out"></input>
		<input type="button" onClick="dojo.fadeIn({ node: 'foo', duration: 1000 }).play()" value="fade in"></input>
		</form>

			Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean
			semper sagittis velit. Cras in mi. Duis porta mauris ut ligula.
			Proin porta rutrum lacus. Etiam consequat scelerisque quam. Nulla
			facilisi.  Maecenas luctus venenatis nulla. In sit amet dui non mi
			semper iaculis.  Sed molestie tortor at ipsum. Morbi dictum rutrum
			magna. Sed vitae risus.

			Aliquam vitae enim. Duis scelerisque metus auctor est venenatis
			imperdiet. Fusce dignissim porta augue. Nulla vestibulum. Integer
			lorem nunc, ullamcorper a, commodo ac, malesuada sed, dolor. Aenean
			id mi in massa bibendum suscipit. Integer eros. Nullam suscipit
			mauris. In pellentesque. Mauris ipsum est, pharetra semper,
			pharetra in, viverra quis, tellus. Etiam purus. Quisque egestas,
			tortor ac cursus lacinia, felis leo adipiscing nisi, et rhoncus
			elit dolor eget eros. Fusce ut quam. Suspendisse eleifend leo vitae
			ligula. Nulla facilisi. Nulla rutrum, erat vitae lacinia dictum,
			pede purus imperdiet lacus, ut semper velit ante id metus. Praesent
			massa dolor, porttitor sed, pulvinar in, consequat ut, leo. Nullam
			nec est. Aenean id risus blandit tortor pharetra congue.
			Suspendisse pulvinar.

		Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean semper
		sagittis velit. Cras in mi. Duis porta mauris ut ligula.  Proin porta
		rutrum lacus. Etiam consequat scelerisque quam. Nulla facilisi.
		Maecenas luctus venenatis nulla. In sit amet dui non mi semper iaculis.
		Sed molestie tortor at ipsum. Morbi dictum rutrum magna. Sed vitae
		risus.

		Aliquam vitae enim. Duis scelerisque metus auctor est venenatis
		imperdiet. Fusce dignissim porta augue. Nulla vestibulum. Integer lorem
		nunc, ullamcorper a, commodo ac, malesuada sed, dolor. Aenean id mi in
		massa bibendum suscipit. Integer eros. Nullam suscipit mauris. In
		pellentesque. Mauris ipsum est, pharetra semper, pharetra in, viverra
		quis, tellus. Etiam purus. Quisque egestas, tortor ac cursus lacinia,
		felis leo adipiscing nisi, et rhoncus elit dolor eget eros. Fusce ut
		quam. Suspendisse eleifend leo vitae ligula. Nulla facilisi. Nulla
		rutrum, erat vitae lacinia dictum, pede purus imperdiet lacus, ut
		semper velit ante id metus. Praesent massa dolor, porttitor sed,
		pulvinar in, consequat ut, leo. Nullam nec est. Aenean id risus blandit
		tortor pharetra congue.  Suspendisse pulvinar.

		Aliquam vitae enim. Duis scelerisque metus auctor est venenatis
		imperdiet. Fusce dignissim porta augue. Nulla vestibulum. Integer lorem
		nunc, ullamcorper a, commodo ac, malesuada sed, dolor. Aenean id mi in
		massa bibendum suscipit. Integer eros. Nullam suscipit mauris. In
		pellentesque. Mauris ipsum est, pharetra semper, pharetra in, viverra
		quis, tellus. Etiam purus. Quisque egestas, tortor ac cursus lacinia,
		felis leo adipiscing nisi, et rhoncus elit dolor eget eros. Fusce ut
		quam. Suspendisse eleifend leo vitae ligula. Nulla facilisi. Nulla
		rutrum, erat vitae lacinia dictum, pede purus imperdiet lacus, ut
		semper velit ante id metus. Praesent massa dolor, porttitor sed,
		pulvinar in, consequat ut, leo. Nullam nec est. Aenean id risus blandit
		tortor pharetra congue.  Suspendisse pulvinar.

redpajama_set_name: RedPajamaGithub


text:
package com.jetbrains.commandInterface.commandLine.psi.impl;

import com.intellij.psi.PsiElement;
import com.jetbrains.commandInterface.command.Argument;
import com.jetbrains.commandInterface.command.Help;
import com.jetbrains.commandInterface.command.Option;
import com.jetbrains.commandInterface.commandLine.CommandLinePart;
import com.jetbrains.commandInterface.commandLine.ValidationResult;
import com.jetbrains.commandInterface.commandLine.psi.CommandLineArgument;
import com.jetbrains.commandInterface.commandLine.psi.CommandLineFile;
import com.jetbrains.commandInterface.commandLine.psi.CommandLineOption;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

/**
 * Class to be used by autogenerated PSI elements to delegate logic
 *
 * @author Ilya.Kazakevich
 */
@SuppressWarnings("StaticMethodOnlyUsedInOneClass") // This class was created to store functions for grammar kit
final class CommandLinePsiImplUtils {

  private CommandLinePsiImplUtils() {
  }

  /**
   * Checks if option is long (--long-option vs -s)
   *
   * @param o option
   * @return true if long
   */
  static boolean isLong(@NotNull final CommandLineOption o) {
    return o.getLongOptionNameToken() != null;
  }

  /**
   * Finds real option based on psi opton
   *
   * @param option psi option
   * @return real option (if any)
   */
  @Nullable
  static Option findRealOption(@NotNull final CommandLineOption option) {
    final ValidationResult validationResult = getValidationResult(option);
    if (validationResult == null) {
      return null;
    }
    return validationResult.getOption(option);
  }

  /**
   * Tries to find appropriate help for argument. It can be argument help for positional argument or option help
   * for option argument.
   *
   * @param argument argument to search help for
   * @return help for argument or null if not found
   */
  @Nullable
  static Help findBestHelp(@NotNull final CommandLineArgument argument) {
    final Option option = argument.findOptionForOptionArgument();
    if (option != null) {
      return option.getHelp();
    }
    final Argument realArgument = argument.findRealArgument();
    return (realArgument != null ? realArgument.getHelp() : null);
  }

  /**
   * Finds real argument based on psi argument
   *
   * @param argument psi argument
   * @return real argument (if any)
   */
  @Nullable
  static Argument findRealArgument(@NotNull final CommandLineArgument argument) {
    final ValidationResult validationResult = getValidationResult(argument);
    if (validationResult == null) {
      return null;
    }
    return validationResult.getArgument(argument);
  }

  /**
   * Finds option if argument is option argument
   *
   * @param argument argument to check
   * @return option (if option argument) or null if not
   */
  @Nullable
  static Option findOptionForOptionArgument(@NotNull final CommandLineArgument argument) {
    final ValidationResult validationResult = getValidationResult(argument);
    if (validationResult == null) {
      return null;
    }
    return validationResult.getOptionForOptionArgument(argument);
  }

  /**
   * Searches for validation result for command line
   *
   * @param commandLinePart command line part
   * @return validation result (if any)
   */
  @Nullable
  private static ValidationResult getValidationResult(@NotNull final CommandLinePart commandLinePart) {
    final CommandLineFile commandLineFile = commandLinePart.getCommandLineFile();
    if (commandLineFile == null) {
      return null;
    }
    final ValidationResult validationResult = commandLineFile.getValidationResult();
    if (validationResult == null) {
      return null;
    }
    return validationResult;
  }

  /**
   * Returns option name regardless it is long or short.
   *
   * @param o option
   * @return name (if any)
   */
  @Nullable
  @NonNls
  static String getOptionName(@NotNull final CommandLineOption o) {
    final PsiElement longNameToken = o.getLongOptionNameToken();
    if (longNameToken != null) {
      return longNameToken.getText();
    }
    final PsiElement shortOptionNameToken = o.getShortOptionNameToken();
    if (shortOptionNameToken != null) {
      return shortOptionNameToken.getText();
    }
    return null;
  }
}


text:
'''
================================
extract - Extract UMI from fastq
================================

*Extract UMI barcode from a read and add it to the read name, leaving
any sample barcode in place*

Can deal with paired end reads and UMIs
split across the paired ends. Can also optionally extract cell
barcodes and append these to the read name also. See the section below
for an explanation for how to encode the barcode pattern(s) to
specficy the position of the UMI +/- cell barcode.

Usage:
------

For single ended reads, the following reads from stdin and outputs to
stdout::

        umi_tools extract --extract-method=string
        --bc-pattern=[PATTERN] -L extract.log [OPTIONS]

For paired end reads, the following reads end one from stdin and end
two from FASTQIN and outputs end one to stdout and end two to
FASTQOUT::

        umi_tools extract --extract-method=string
        --bc-pattern=[PATTERN] --bc-pattern2=[PATTERN]
        --read2-in=[FASTQIN] --read2-out=[FASTQOUT] -L extract.log [OPTIONS]

Using regex and filtering against a whitelist of cell barcodes::

        umi_tools extract --extract-method=regex
        --bc-pattern=[REGEX] --whitlist=[WHITELIST_TSV]
        -L extract.log [OPTIONS]

Filtering and correcting cell barcodes
--------------------------------------

umi_tools extract can optionally filter cell barcodes against a user-supplied
whitelist (``--whitelist``). If a whitelist is not available for your data, e.g
if you have performed droplet-based scRNA-Seq, you can use the
whitelist tool.

Cell barcodes which do not match the whitelist (user-generated or
automatically generated) can also be optionally corrected using the
``--error-correct-cell`` option.

""""""""""""""""""""""""
``--error-correct-cell``
""""""""""""""""""""""""
     Error correct cell barcodes to the whitelist (see ``--whitelist``)

"""""""""""""""
``--whitelist``
"""""""""""""""
     Whitelist of accepted cell barcodes. The whitelist should be in
     the following format (tab-separated)::

        AAAAAA    AGAAAA
        AAAATC
        AAACAT
        AAACTA    AAACTN,GAACTA
        AAATAC
        AAATCA    GAATCA
        AAATGT    AAAGGT,CAATGT

    Where column 1 is the whitelisted cell barcodes and column 2 is
    the list (comma-separated) of other cell barcodes which should be
    corrected to the barcode in column 1. If the ``--error-correct-cell``
    option is not used, this column will be ignored. Any additional columns
    in the whitelist input, such as the counts columns from the output of
    umi_tools whitelist, will be ignored.

"""""""""""""""
``--blacklist``
"""""""""""""""
    BlackWhitelist of cell barcodes to discard

""""""""""""""""""""""
``--subset-reads=[N]``
""""""""""""""""""""""
    Only parse the first N reads

""""""""""""""""""""""""""""""
``--quality-filter-threshold``
""""""""""""""""""""""""""""""
    Remove reads where any UMI base quality score falls below this threshold

"""""""""""""""""""""""""
``--quality-filter-mask``
"""""""""""""""""""""""""
    If a UMI base has a quality below this threshold, replace the base with 'N'

""""""""""""""""""""""
``--quality-encoding``
""""""""""""""""""""""
    Quality score encoding. Choose from:
     - 'phred33' [33-77]
     - 'phred64' [64-106]
     - 'solexa' [59-106]

"""""""""""""""""""""
``--reconcile-pairs``
"""""""""""""""""""""
    Allow read 2 infile to contain reads not in read 1 infile. This
    enables support for upstream protocols where read one contains
    cell barcodes, and the read pairs have been filtered and corrected
    without regard to the read2s

Experimental options
--------------------

.. note:: These options have not been extensively testing to ensure behaviour is as expected. If you have some suitable input files which we can use for testing, please `contact us <https://github.com/CGATOxford/UMI-tools/issues>`_.

If you have a library preparation method where the UMI may be in
either read, you can use the following options to search for the UMI
in either read::

       --either-read --extract-method --bc-pattern=[PATTERN1] --bc-pattern2=[PATTERN2]

Where both patterns match, the default behaviour is to discard both
reads. If you want to select the read with the UMI with highest
sequence quality, provide ``--either-read-resolve=quality.``

'''
from __future__ import absolute_import
import sys
import regex
import collections
import optparse

# python 3 doesn't require izip
try:
    # Python 2
    from itertools import izip
except ImportError:
    # Python 3
    izip = zip

import umi_tools.Utilities as U
import umi_tools.Documentation as Documentation
import umi_tools.umi_methods as umi_methods
import umi_tools.extract_methods as extract_methods
import umi_tools.whitelist_methods as whitelist_methods

# add the generic docstring text
__doc__ = __doc__ + Documentation.GENERIC_DOCSTRING_WE
usage = '''
extract - Extract UMI from fastq

Usage:

   Single-end:
      umi_tools extract [OPTIONS] -p PATTERN [-I IN_FASTQ[.gz]] [-S OUT_FASTQ[.gz]]

   Paired end:
      umi_tools extract [OPTIONS] -p PATTERN [-I IN_FASTQ[.gz]] [-S OUT_FASTQ[.gz]] --read2-in=IN2_FASTQ[.gz] --read2-out=OUT2_FASTQ[.gz]

   note: If -I/-S are ommited standard in and standard out are used
         for input and output.  To generate a valid BAM file on
         standard out, please redirect log with --log=LOGFILE or
         --log2stderr. Input/Output will be (de)compressed if a
         filename provided to -S/-I/--read2-in/read2-out ends in .gz
         '''

def main(argv=None):
    """script main.

    parses command line options in sys.argv, unless *argv* is given.
    """

    if argv is None:
        argv = sys.argv

    # setup command line parser
    parser = U.OptionParser(version="%prog version: $Id$",
                            usage=usage,
                            description=globals()["__doc__"])

    if len(argv) == 1:
        parser.print_usage()
        print ("Required options missing, see --help for more details")
        return 1

    group = U.OptionGroup(parser, "extract-specific options")

    # (Experimental option) Retain the UMI in the sequence read"
    group.add_option("--retain-umi", dest="retain_umi", action="store_true",
                     help=optparse.SUPPRESS_HELP)
    group.add_option("--read2-out", dest="read2_out", type="string",
                     help="file to output processed paired read to")
    group.add_option("--read2-stdout", dest="read2_stdout",
                     action="store_true",
                     help="Paired reads, send read2 to stdout, discarding read1")
    group.add_option("--quality-filter-threshold",
                     dest="quality_filter_threshold", type="int",
                     help=("Remove reads where any UMI base quality score "
                           "falls below this threshold"))
    group.add_option("--quality-filter-mask",
                     dest="quality_filter_mask", type="int",
                     help=("If a UMI base has a quality below this threshold, "
                           "replace the base with 'N'"))
    group.add_option("--quality-encoding",
                     dest="quality_encoding", type="choice",
                     choices=["phred33", "phred64", "solexa"],
                     help=("Quality score encoding. Choose from 'phred33'"
                           "[33-77] 'phred64' [64-106] or 'solexa' [59-106]"))
    group.add_option("--filter-cell-barcode",
                     dest="filter_cell_barcode",
                     action="store_true",
                     help=optparse.SUPPRESS_HELP)
    group.add_option("--error-correct-cell",
                     dest="error_correct_cell",
                     action="store_true",
                     help=("Correct errors in the cell barcode"))
    group.add_option("--whitelist",
                     dest="whitelist", type="string",
                     help=("A whitelist of accepted cell barcodes"))
    group.add_option("--blacklist",
                     dest="blacklist", type="string",
                     help=("A blacklist of rejected cell barcodes"))
    group.add_option("--filter-umi",
                     dest="filter_umi",
                     action="store_true",
                     #help="Filter the UMIs"
                     help=optparse.SUPPRESS_HELP)
    group.add_option("--umi-whitelist", dest="umi_whitelist",
                     type="string", default=None,
                     #help="A whitelist of accepted UMIs [default=%default]"
                     help=optparse.SUPPRESS_HELP)
    group.add_option("--umi-whitelist-paired", dest="umi_whitelist_paired",
                     type="string", default=None,
                     #help="A whitelist of accepted UMIs for read2[default=%default]"
                     help=optparse.SUPPRESS_HELP)
    group.add_option("--correct-umi-threshold", dest="correct_umi_threshold",
                     type="int", default=0,
                     #help="Correct errors in UMIs to the whitelist(s) provided"
                     #"if within threshold [default=%default]"
                     help=optparse.SUPPRESS_HELP)
    group.add_option("--umi-correct-log", dest="umi_correct_log",
                     type="string", default=None,
                     #help="File logging UMI error correction",
                     help=optparse.SUPPRESS_HELP)
    group.add_option("--subset-reads", "--reads-subset",
                     dest="reads_subset", type="int",
                     help=("Only extract from the first N reads. If N is "
                           "greater than the number of reads, all reads will "
                           "be used"))
    group.add_option("--reconcile-pairs",
                     dest="reconcile", action="store_true",
                     help=("Allow the presences of reads in read2 input that "
                           "are not present in read1 input. This allows cell "
                           "barcode filtering of read1s without "
                           "considering read2s"))
    group.add_option("--umi-separator",
                     dest="umi_separator", type="string",
                     help=("Separator to use to add UMI to the read name. Default: _"))
    parser.add_option_group(group)

    group = U.OptionGroup(parser, "[EXPERIMENTAl] barcode extraction options")

    group.add_option("--either-read", dest="either_read", action="store_true",
                     help="UMI may be on either read (see "
                     "--either-read-resolve) for options to resolve cases where"
                     "UMI is on both reads")
    group.add_option("--either-read-resolve",
                     dest="either_read_resolve", type="choice",
                     choices=["discard", "quality"],
                     help=("How to resolve instances where both reads "
                           "contain a UMI but using --either-read."
                           "Choose from 'discard' or 'quality'"
                           "(use highest quality). default=dicard"))

    parser.add_option_group(group)

    parser.set_defaults(extract_method="string",
                        filter_cell_barcodes=False,
                        whitelist=None,
                        blacklist=None,
                        error_correct_cell=False,
                        pattern=None,
                        pattern2=None,
                        read2_in=None,
                        read2_out=False,
                        read2_stdout=False,
                        quality_filter_threshold=None,
                        quality_encoding=None,
                        reconcile=False,
                        either_read=False,
                        either_read_resolve="discard",
                        umi_separator="_",
                        ignore_suffix=False)

    # add common options (-h/--help, ...) and parse command line

    (options, args) = U.Start(parser, argv=argv,
                              add_extract_options=True,
                              add_group_dedup_options=False,
                              add_umi_grouping_options=False,
                              add_sam_options=False)

    if options.filter_cell_barcode:
        U.info('Use of --whitelist ensures cell barcodes are filtered. '
               '--filter-cell-barcode is no longer required and may be '
               'removed in future versions.')

    if options.whitelist is not None:
        options.filter_cell_barcode = True

    if options.retain_umi and not options.extract_method == "regex":
        U.error("option --retain-umi only works with --extract-method=regex")

    if (options.filtered_out and not options.extract_method == "regex" and
        options.whitelist is None):
        U.error("Reads will not be filtered unless extract method is"
                "set to regex (--extract-method=regex) or cell"
                "barcodes are filtered (--whitelist)")

    if options.quality_filter_threshold or options.quality_filter_mask:
        if not options.quality_encoding:
            U.error("must provide a quality encoding (--quality-"
                    "encoding) to filter UMIs by quality (--quality"
                    "-filter-threshold) or mask low quality bases "
                    "with (--quality-filter-mask)")

    extract_cell, extract_umi = U.validateExtractOptions(options)

    if options.either_read:
        if extract_cell:
            U.error("Option to extract from either read (--either-read) "
                    "is not currently compatible with cell barcode extraction")
        if not options.extract_method == "regex":
            U.error("Option to extract from either read (--either-read)"
                    "requires --extract-method=regex")
        if not options.pattern or not options.pattern2:
            U.error("Option to extract from either read (--either-read)"
                    "requires --bc-pattern=[PATTERN1] and"
                    "--bc-pattern2=[PATTERN2]")

    if options.filter_umi:

        if not options.umi_whitelist:
                U.error("must provide a UMI whitelist (--umi-whitelist) if using "
                        "--filter-umi option")
        if options.pattern2 and not options.umi_whitelist_paired:
                U.error("must provide a UMI whitelist for paired end "
                        "(--umi-whitelist-paired) if using --filter-umi option"
                        "with paired end data")
        if not extract_umi:
            if options.extract_method == "string":
                U.error("barcode pattern(s) do not include any umi bases "
                        "(marked with 'Ns') %s, %s" % (
                            options.pattern, options.pattern2))
            elif options.extract_method == "regex":
                U.error("barcode regex(es) do not include any umi groups "
                        "(starting with 'umi_') %s, %s" % (
                            options.pattern, options.pattern2))

    if options.whitelist:

        if not extract_cell:
            if options.extract_method == "string":
                U.error("barcode pattern(s) do not include any cell bases "
                        "(marked with 'Cs') %s, %s" % (
                            options.pattern, options.pattern2))
            elif options.extract_method == "regex":
                U.error("barcode regex(es) do not include any cell groups "
                        "(starting with 'cell_') %s, %s" % (
                            options.pattern, options.pattern2))

    read1s = umi_methods.fastqIterate(options.stdin)

    # set up read extractor
    ReadExtractor = extract_methods.ExtractFilterAndUpdate(
        options.extract_method,
        options.pattern,
        options.pattern2,
        options.prime3,
        extract_cell,
        options.quality_encoding,
        options.quality_filter_threshold,
        options.quality_filter_mask,
        options.filter_umi,
        options.filter_cell_barcode,
        options.retain_umi,
        options.either_read,
        options.either_read_resolve,
        options.umi_separator)

    if options.filter_umi:
        umi_whitelist, false_to_true_map = whitelist_methods.getUserDefinedBarcodes(
            options.umi_whitelist,
            options.umi_whitelist_paired,
            deriveErrorCorrection=True,
            threshold=options.correct_umi_threshold)

        U.info("Length of whitelist: %i" % len(umi_whitelist))
        U.info("Length of 'correctable' whitelist: %i" % len(false_to_true_map))

        ReadExtractor.umi_whitelist = umi_whitelist
        ReadExtractor.umi_false_to_true_map = false_to_true_map
        ReadExtractor.umi_whitelist_counts = collections.defaultdict(
            lambda: collections.Counter())

    if options.whitelist:
        cell_whitelist, false_to_true_map = whitelist_methods.getUserDefinedBarcodes(
            options.whitelist,
            getErrorCorrection=options.error_correct_cell)

        ReadExtractor.cell_whitelist = cell_whitelist
        ReadExtractor.false_to_true_map = false_to_true_map

    if options.blacklist:
        blacklist = set()
        with U.openFile(options.blacklist, "r") as inf:
            for line in inf:
                blacklist.add(line.strip().split("\t")[0])
        ReadExtractor.cell_blacklist = blacklist

    # variables for progress monitor
    progCount = 0
    displayMax = 100000
    U.info("Starting barcode extraction")

    if options.filtered_out:
        filtered_out = U.openFile(options.filtered_out, "w")

    if options.read2_in is None:
        for read in read1s:

            # incrementing count for monitoring progress
            progCount += 1

            # Update display in every 100kth iteration
            if progCount % displayMax == 0:
                U.info("Parsed {} reads".format(progCount))

            new_read = ReadExtractor(read)

            if options.reads_subset:
                if (ReadExtractor.read_counts['Input Reads'] >
                    options.reads_subset):
                    break

            if not new_read:
                if options.filtered_out:
                    filtered_out.write(str(read) + "\n")
                continue

            options.stdout.write(str(new_read) + "\n")

    else:

        if options.filtered_out2:
            filtered_out2 = U.openFile(options.filtered_out2, "w")

        read2s = umi_methods.fastqIterate(U.openFile(options.read2_in))

        if options.read2_out:
            read2_out = U.openFile(options.read2_out, "w")

        if options.reconcile:
            strict = False
        else:
            strict = True

        for read1, read2 in umi_methods.joinedFastqIterate(
                read1s, read2s, strict, options.ignore_suffix):

            # incrementing count for monitoring progress
            progCount += 1

            # Update display in every 100kth iteration
            if progCount % displayMax == 0:
                U.info("Parsed {} reads".format(progCount))
                sys.stdout.flush()

            reads = ReadExtractor(read1, read2)

            if options.reads_subset:
                if (ReadExtractor.read_counts['Input Reads'] >
                    options.reads_subset):
                    break

            if not reads:
                if options.filtered_out:
                    filtered_out.write(str(read1) + "\n")
                if options.filtered_out2:
                    filtered_out2.write(str(read2) + "\n")
                continue
            else:
                new_read1, new_read2 = reads

            if options.read2_stdout:
                options.stdout.write(str(new_read2) + "\n")
            else:
                options.stdout.write(str(new_read1) + "\n")

                if options.read2_out:
                    read2_out.write(str(new_read2) + "\n")

    if options.read2_out:
        read2_out.close()
    if options.filtered_out:
        filtered_out.close()
    if options.filtered_out2:
        filtered_out2.close()

    for k, v in ReadExtractor.getReadCounts().most_common():
        U.info("%s: %s" % (k, v))

    if options.umi_correct_log:
        with U.openFile(options.umi_correct_log, "w") as outf:
            outf.write("umi\tcount_no_errors\tcount_errors\n")
            for umi, counts in ReadExtractor.umi_whitelist_counts.items():
                outf.write("%s\t%i\t%i\n" % (
                    umi, counts["no_error"], counts["error"]))
        outf.close()

    U.Stop()

if __name__ == "__main__":
    sys.exit(main(sys.argv))


text:
Here I will put the documentation of the project.

It will most likely be written using the markdown (https://help.github.com/articles/github-flavored-markdown/)