<|start_filename|>Parcel/Parcel.fs<|end_filename|>
﻿module Parcel
    open FParsec

    (*
     * PRIVATE IMPLEMENTATIONS
     *)

    // wrapper for success/failure
    let private test p str =
        match runParserOnString p (AST.Env("", "", "")) "" str with
        | Success(result, _, _)     -> printfn "Success: %A" result
        | Failure(errorMsg, _, _)   -> printfn "Failure: %s" errorMsg

    let private getAddress(formula: string)(path: string)(wbname: string)(wsname: string): AST.Address =
        match runParserOnString (Grammar.AddrR1C1 .>> eof) (AST.Env(path, wbname, wsname)) "" formula with
        | Success(addr, _, _)       -> addr
        | Failure(errorMsg, _, _)   -> failwith errorMsg

    let private getRange(formula: string)(path: string)(wbname: string)(wsname: string): AST.Range option =
        match runParserOnString (Grammar.RangeR1C1 .>> eof) (AST.Env(path, wbname, wsname)) "" formula with
        | Success(range, _, _)      -> Some(range)
        | Failure(errorMsg, _, _)   -> None

    let private getReference(formula: string)(path: string)(wbname: string)(wsname: string): AST.Reference option =
        match runParserOnString (Grammar.Reference Grammar.RangeAny .>> eof) (AST.Env(path, wbname, wsname)) "" formula with
        | Success(reference, _, _)  -> Some(reference)
        | Failure(errorMsg, _, _)   -> None

    (*
     * PUBLIC API
     *)
    let isNumeric(str: string): bool =
        match run (pfloat .>> eof) str with
        | Success(number, _, _)     -> true
        | Failure(errorMsg, _, _)   -> false

    let parseFormula(formula: string)(path: string)(wbname: string)(wsname: string): AST.Expression option =
        try
            match runParserOnString Grammar.Formula (AST.Env(path, wbname, wsname)) "" formula with
            | Success(formula, _, _) -> Some(formula)
            | Failure(errorMsg, _, _) -> None
        with
        | :? AST.IndirectAddressingNotSupportedException as e -> raise e    // rethrow
        | :? AST.ParseException as e -> raise e                             // rethrow
        | e ->                                                              // wrap other exception
            let rec f = (fun (e:exn) ->
                            e.Message +
                            "\n\n" +
                            if (e.InnerException <> null) then f e.InnerException else ""
                        )
            raise (AST.ParseException (f e))
            

    let parseFormulaAtAddress(fAddr: AST.Address)(formula: string): AST.Expression =
        match parseFormula formula fAddr.Path fAddr.WorkbookName fAddr.WorksheetName with
        | Some ast -> ast
        | None -> failwith ("Parse error on formula: " + formula)

    // The parser REPL calls this; note that the
    // Formula parser looks for EOF
    let consoleParser(s: string) = test Grammar.Formula s

    // Call this for simple address parsing
    let simpleReferenceParser(s: string, e: AST.Env) : AST.Reference =
        match runParserOnString (Grammar.Reference Grammar.RangeAny) e "" s with
        | Success(result, _, _) -> result
        | Failure(errorMsg, _, _) -> failwith ("String \"" + s + "\" does not appear to be a Reference:\n" + errorMsg)

    let rangeReferencesFromExpr(tree: AST.Expression) : AST.Range[] =
        RangeVisitor.rangesFromExpr tree |> Seq.toArray

    let rangeReferencesFromFormula(formula: string, path: string, workbook: string, worksheet: string, ignore_parse_errors: bool) : AST.Range[] =
        try
            match (parseFormula formula path workbook worksheet),ignore_parse_errors with
            | Some(tree),_ -> rangeReferencesFromExpr tree
            | None,false -> raise (AST.ParseException(formula))
            | None,true -> [||]    // just ignore parse exceptions for now
        with
        // right now, we recognize indirect addresses but do not correctly dereference them,
        // which requires a program interpreter and input spreadsheet :(
        | :? AST.IndirectAddressingNotSupportedException -> [||]

    let addrReferencesFromExpr(tree: AST.Expression) : AST.Address[] =
        CellVisitor.addrsFromExpr tree |> Seq.toArray

    let addrReferencesFromFormula(formula: string, path: string, wb: string, ws: string, ignore_parse_errors: bool) : AST.Address[] =
        match (parseFormula formula path wb ws),ignore_parse_errors with
        | Some(tree),_ -> addrReferencesFromExpr tree
        | None,false -> raise (AST.ParseException formula)
        | None,true -> [||]    // just ignore parse exceptions for now

    let constantsFromExpr(e: AST.Expression) : AST.ReferenceConstant[] =
        CellVisitor.constantsFromExpr e |> Seq.toArray

    let constantsFromFormula(formula: string, path: string, wb: string, ws: string, ignore_parse_errors: bool) : AST.ReferenceConstant[] =
        match (parseFormula formula path wb ws),ignore_parse_errors with
        | Some(ast),_ -> constantsFromExpr ast
        | None,false -> raise (AST.ParseException formula)
        | None,true -> [||]    // just ignore parse exceptions for now

    let rec operatorNamesFromExpr(ast: AST.Expression): string list =
        match ast with
        | AST.ReferenceExpr(r) -> operatorNamesFromRef(r)
        | AST.BinOpExpr(op, e1, e2) -> op :: operatorNamesFromExpr(e1) @ operatorNamesFromExpr(e2)
        | AST.UnaryOpExpr(op, e) -> op.ToString() :: operatorNamesFromExpr(e)
        | AST.ParensExpr(e) -> operatorNamesFromExpr(e)

    and operatorNamesFromRef(ref: AST.Reference): string list =
        match ref with
        | :? AST.ReferenceFunction as r -> [r.FunctionName]
        | _ -> []


    let rec formulaNamesFromExpr(ast: AST.Expression): string list =
        match ast with
        | AST.ReferenceExpr(r) -> formulaNamesFromRef(r)
        | AST.BinOpExpr(op, e1, e2) -> formulaNamesFromExpr(e1) @ formulaNamesFromExpr(e2)
        | AST.UnaryOpExpr(op, e) -> formulaNamesFromExpr(e)
        | AST.ParensExpr(e) -> formulaNamesFromExpr(e)

    and formulaNamesFromRef(ref: AST.Reference): string list =
        match ref with
        | :? AST.ReferenceFunction as r -> [r.FunctionName]
        | _ -> []

    let formulaNamesFromFormula(formula: string, path: string, wb: string, ws: string, ignore_parse_errors: bool) : seq<string> =
        let abspath = System.IO.Path.GetDirectoryName(wb)
        match (parseFormula formula abspath wb ws),ignore_parse_errors with
        | Some(ast),_ -> formulaNamesFromExpr(ast) |> Seq.ofList
        | None,false -> raise (AST.ParseException formula)
        | None,true -> Seq.empty    // just ignore parse exceptions for now



<|start_filename|>Parcel/ExcelIO/IO.cs<|end_filename|>
﻿using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Excel = Microsoft.Office.Interop.Excel;
using System.Reflection;
using System.Diagnostics;
using System.Runtime.InteropServices;
using Microsoft.Win32;
using System.Globalization;

namespace ExcelIO
{
    public class InvalidURLZoneException : Exception { }

    public class TrustedPathAlreadySetException : Exception { }

    public class AppWrap : IDisposable
    {
        public class WorkbookWrap : IDisposable
        {
            private Excel.Workbook _wb;
            private List<Excel.Worksheet> _ws = new List<Excel.Worksheet>();
            private List<Excel.Range> _rs = new List<Excel.Range>();
            private Regex _formula_filter = new Regex("^=", RegexOptions.Compiled);

            public WorkbookWrap(Excel.Workbook wb)
            {
                _wb = wb;
            }
            public IEnumerable<string> GetFormulas() {
                var fs = new List<string>();
                foreach (Excel.Worksheet ws in _wb.Worksheets)
                {
                    // track touched workbooks
                    _ws.Add(ws);

                    // track touched ranges
                    Excel.Range ur = ws.UsedRange;
                    _rs.Add(ur);

                    // if a single cell
                    if (ur.Columns.Count == 1 && ur.Rows.Count == 1)
                    {
                        String s = ur.Formula;
                        if (!String.IsNullOrEmpty(s) && _formula_filter.IsMatch(s))
                        {
                            fs.Add(s);
                        }
                    }
                    // if an array
                    else
                    {

                        // array read formulas
                        object[,] formulas = ur.Formula;

                        // grab formula strings
                        // danger: one-based array
                        for (int i = 1; i <= formulas.GetLength(0); i++)
                        {
                            for (int j = 1; j <= formulas.GetLength(1); j++)
                            {
                                String s = (String)formulas[i, j];
                                if (!String.IsNullOrEmpty(s) && _formula_filter.IsMatch(s))
                                {
                                    fs.Add(s);
                                }
                            }
                        }
                    }
                }
                return fs;
            }
            public void Dispose()
            {
                // dispose of ranges
                foreach (Excel.Range r in _rs)
                {
                    Marshal.ReleaseComObject(r);
                }
                _rs = null;

                // dispose of worksheets
                foreach (Excel.Worksheet ws in _ws)
                {
                    Marshal.ReleaseComObject(ws);
                    
                }
                _ws = null;

                // dispose of workbooks
                _wb.Close();
                Marshal.ReleaseComObject(_wb);
                _wb = null;

                // force collection
                GC.Collect();
            }
        }

        private Excel.Application _app;
        public AppWrap()
        {
            _app = new Excel.Application();
        }
        public WorkbookWrap OpenWorkbook(string file) {
            // clear zone data
            IO.DeleteZoneData(file);

            return new WorkbookWrap(IO.OpenWorkbook(file, _app));
        }
        public void Dispose()
        {
            _app.Quit();
        }
    }

    public class IO
    {
        public static readonly string KEY_PREFIX = @"HKEY_CURRENT_USER\";
        public static readonly string KEY_BASE = @"Software\Microsoft\Office\14.0\Excel\Security\Trusted Locations\";

        public enum URLZone
        {
            URLZONE_INVALID = -1,
            URLZONE_PREDEFINED_MIN = 0,
            URLZONE_LOCAL_MACHINE = 0,
            URLZONE_INTRANET,
            URLZONE_TRUSTED,
            URLZONE_INTERNET,
            URLZONE_UNTRUSTED,
            URLZONE_PREDEFINED_MAX = 999,
            URLZONE_USER_MIN = 1000,
            URLZONE_USER_MAX = 10000
        }

        // All of the following private enums are poorly documented
        private enum XlCorruptLoad
        {
            NormalLoad = 0,
            RepairFile = 1,
            ExtractData = 2
        }

        private enum XlUpdateLinks
        {
            Yes = 2,
            No = 0
        }

        private enum XlPlatform
        {
            Macintosh = 1,
            Windows = 2,
            MSDOS = 3
        }

        [DllImport(@"..\..\..\Parcel\Debug\TrustedZoneHandler.dll", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Unicode)]
        private static extern void EraseZoneData(string filename);

        [DllImport(@"..\..\..\Parcel\Debug\TrustedZoneHandler.dll", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Unicode)]
        private static extern uint ReadZoneData(string filename);

        [DllImport(@"..\..\..\Parcel\Debug\TrustedZoneHandler.dll", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Unicode)]
        private static extern uint SetZoneData(string filename, uint url_zone);

        public static URLZone GetZoneData(string filename)
        {
            // ReadZoneData requires an absolute path
            return (URLZone)ReadZoneData(System.IO.Path.GetFullPath(filename));
        }

        public static void DeleteZoneData(string filename)
        {
            // EraseZoneData requires an absolute path
            EraseZoneData(System.IO.Path.GetFullPath(filename));
        }

        public static void UpdateZoneData(string filename, URLZone urlz)
        {
            if (urlz < URLZone.URLZONE_PREDEFINED_MIN || urlz > URLZone.URLZONE_USER_MAX)
            {
                throw new InvalidURLZoneException();
            }

            // EraseZoneData requires an absolute path
            SetZoneData(System.IO.Path.GetFullPath(filename), (uint)urlz);
        }

        /// <summary>
        /// Returns the next available trusted location ID.
        /// </summary>
        /// <returns>An integer representing the trusted location ID. Throws an exception if the key already exists.</returns>
        public static int NextTrustedLocationID(string path)
        {
            var cpath = System.IO.Path.GetFullPath(path);
            Regex r = new Regex("Location([0-9]+)", RegexOptions.Compiled);
            var max = 0;
            var no_locations = true;
            var already_defined = false;

            using (RegistryKey key = Registry.CurrentUser.OpenSubKey(KEY_BASE))
            {
                foreach (var location in key.GetSubKeyNames())
                {
                    no_locations = false;

                    // extract numeric ID
                    var id = System.Convert.ToInt32(r.Match(location).Groups[1].Value);

                    // check to see if the location is already defined
                    var trusted_path = System.IO.Path.GetFullPath((String)Registry.GetValue(KEY_PREFIX + KEY_BASE + "Location" + id, "Path", "empty"));
                    if (cpath == trusted_path)
                    {
                        throw new TrustedPathAlreadySetException();
                    }

                    // update max
                    if (id > max)
                    {
                        max = id;
                    }
                }
            }

            if (no_locations)
            {
                return max;
            }
            else
            {
                return max + 1;
            }
        }

        /// <summary>
        /// Adds a trusted location to the Windows registry.
        /// </summary>
        /// <param name="path">An absolute path string.</param>
        /// <param name="allow_subfolders">Subfolders of the path are trusted when true.</param>
        public static void AddTrustedLocation(string path, bool allow_subfolders, string description)
        {
            int id;
            try
            {
                id = NextTrustedLocationID(path);
            }
            catch (TrustedPathAlreadySetException)
            {
                return;
            }

            var dt = DateTime.Now;
            Registry.SetValue(KEY_PREFIX + KEY_BASE + "Location" + id, "Path", path);
            Registry.SetValue(KEY_PREFIX + KEY_BASE + "Location" + id, "AllowSubfolders", allow_subfolders, RegistryValueKind.DWord);
            Registry.SetValue(KEY_PREFIX + KEY_BASE + "Location" + id, "Date", dt.ToString("g", CultureInfo.CreateSpecificCulture("en-US")));
            Registry.SetValue(KEY_PREFIX + KEY_BASE + "Location" + id, "Description", description);
        }

        public static Excel.Workbook OpenWorkbook(string filename, Excel.Application app)
        {
            // we need to disable all alerts, e.g., password prompts, etc.
            app.DisplayAlerts = false;

            // disable macros
            app.AutomationSecurity = Microsoft.Office.Core.MsoAutomationSecurity.msoAutomationSecurityForceDisable;

            // This call is stupid.  See:
            // http://msdn.microsoft.com/en-us/library/microsoft.office.interop.excel.workbooks.open%28v=office.11%29.aspx
            app.Workbooks.Open(filename,                    // FileName (String)
                               XlUpdateLinks.Yes,           // UpdateLinks (XlUpdateLinks enum)
                               true,                        // ReadOnly (Boolean)
                               Missing.Value,               // Format (int?)
                               "<PASSWORD>",        // Password (String)
                               Missing.Value,               // WriteResPassword (String)
                               true,                        // IgnoreReadOnlyRecommended (Boolean)
                               Missing.Value,               // Origin (XlPlatform enum)
                               Missing.Value,               // Delimiter; if the filetype is txt (String)
                               Missing.Value,               // Editable; not what you think (Boolean)
                               false,                       // Notify (Boolean)
                               Missing.Value,               // Converter(int)
                               false,                       // AddToMru (Boolean)
                               Missing.Value,               // Local; really "use my locale?" (Boolean)
                               XlCorruptLoad.RepairFile);   // CorruptLoad (XlCorruptLoad enum)

            return app.Workbooks[1];
        }

        /// <summary>
        /// Enumerates spreadsheets found at the given path.  Recursively
        /// descends through all directories in the directory hierarchy.
        /// </summary>
        /// <param name="path">The starting point in the search.</param>
        /// <param name="path_filters">Only include matches from directories in the path matching this pattern.</param>
        /// <returns>IEnumerable collection of spreadsheet names</returns>
        public static IEnumerable<string> EnumerateSpreadsheets(string path, Regex path_filter)
        {
            var xlfiles = Directory.EnumerateFiles(path, "*.xls", SearchOption.AllDirectories);
            return xlfiles.Where(xlfile => path_filter.IsMatch(xlfile));
        }
    }
}


<|start_filename|>FParsec/CharParsers.fs<|end_filename|>
﻿// Copyright (c) <NAME> 2007-2015
// License: Simplified BSD License. See accompanying documentation.

[<AutoOpen>]
module FParsec.CharParsers

open System.Diagnostics
open System.Text
open System.Text.RegularExpressions
open System.Runtime.CompilerServices // for MethodImplAttribute

#if LOW_TRUST
#else
open Microsoft.FSharp.NativeInterop
#endif

open FParsec
open FParsec.Internals
open FParsec.Error
open FParsec.Primitives

#nowarn "9" // "Uses of this construct may result in the generation of unverifiable .NET IL code."
#nowarn "51" // "The address-of operator may result in non-verifiable code."

// ================
// Helper functions
// ================

[<Literal>]
let EOS = '\uffff'

let foldCase = Text.FoldCase : string -> string
let normalizeNewlines = Text.NormalizeNewlines

let floatToHexString = HexFloat.DoubleToHexString
let floatOfHexString = HexFloat.DoubleFromHexString

let float32ToHexString = HexFloat.SingleToHexString
let float32OfHexString = HexFloat.SingleFromHexString

// ========================
// Running parsers on input
// ========================

[<StructuredFormatDisplay("{StructuredFormatDisplay}")>]
type ParserResult<'Result,'UserState> =
     | Success of 'Result * 'UserState * Position
     | Failure of string * ParserError * 'UserState
     with
        member private t.StructuredFormatDisplay =
            match t with
            | Success(r,_,_) ->
                if typeof<'Result> = typeof<unit> then "Success: ()"
                else sprintf "Success: %A" r
            | Failure(msg,_,_) ->
                sprintf "Failure:\n%s" msg

let internal applyParser (parser: Parser<'Result,'UserState>) (stream: CharStream<'UserState>) =
    let reply = parser stream
    if reply.Status = Ok then
        Success(reply.Result, stream.UserState, stream.Position)
    else
        let error = ParserError(stream.Position, stream.UserState, reply.Error)
        Failure(error.ToString(stream), error, stream.UserState)

let runParserOnString (parser: Parser<'Result,'UserState>) (ustate: 'UserState) (streamName: string) (chars: string) =
    CharStream.ParseString(chars, 0, chars.Length, applyParser parser, ustate, streamName)

let runParserOnSubstring (parser: Parser<'Result,'UserState>) (ustate: 'UserState) (streamName: string) (chars: string) (index: int) length =
    CharStream.ParseString(chars, index, length, applyParser parser, ustate, streamName)

let runParserOnStream (parser: Parser<'Result,'UserState>) (ustate: 'UserState) (streamName: string) (byteStream: System.IO.Stream) (encoding: System.Text.Encoding) =
#if LOW_TRUST
    let
#else
    use
#endif
        stream = new CharStream<'UserState>(byteStream, encoding)
    stream.UserState <- ustate
    stream.Name <- streamName
    applyParser parser stream

#if PCL
#else
let runParserOnFile (parser: Parser<'Result,'UserState>) (ustate: 'UserState) (path: string) (encoding: System.Text.Encoding) =
#if LOW_TRUST
    let
#else
    use
#endif
        stream = new CharStream<'UserState>(path, encoding)
    stream.UserState <- ustate
    applyParser parser stream
#endif

let run parser (string: string) =
    runParserOnString parser () "" string

// =======
// Parsers
// =======

// -------------------------------------------------------------
// Reading the input stream position and handling the user state
// -------------------------------------------------------------

let getPosition : Parser<Position,'u> =
    fun stream -> Reply(stream.Position)

let getUserState : Parser<'u,'u> =
    fun stream -> Reply(stream.UserState)

let setUserState (newUserState: 'u) : Parser<unit,'u> =
    fun stream ->
        stream.UserState <- newUserState
        Reply(())

let updateUserState (f: 'u -> 'u) : Parser<unit,'u> =
    fun stream ->
        stream.UserState <- f stream.UserState
        Reply(())

let userStateSatisfies f : Parser<unit,'u> =
    fun stream ->
        let status = if f stream.UserState then Ok else Error
        Reply(status, (), NoErrorMessages)

// --------------------
// Parsing single chars
// --------------------

let newlineReturn result : Parser<_,'u> =
    fun stream ->
        if stream.SkipNewline() then Reply(result)
        else Reply(Error, Errors.ExpectedNewline)

let newline<'u>     = newlineReturn '\n' : Parser<_,'u>
let skipNewline<'u> = newlineReturn  ()  : Parser<_,'u>

let unicodeNewlineReturn result : Parser<_,'u> =
    fun stream ->
        if stream.SkipUnicodeNewline() then Reply(result)
        else Reply(Error, Errors.ExpectedNewline)

let unicodeNewline<'u>     = unicodeNewlineReturn '\n' : Parser<_,'u>
let skipUnicodeNewline<'u> = unicodeNewlineReturn  ()  : Parser<_,'u>

let internal charReturnE (c: char) result error : Parser<'a,'u> =
    fun stream ->
        if stream.Skip(c) then Reply(result)
        else Reply(Error, error)

let charReturn c result : Parser<'a,'u> =
    match c with
    | '\r' | '\n' -> newlineReturn result
    | EOS -> invalidArg "c" "The char '\uffff' (EOS) is not a valid argument for the pchar/skipChar/charReturn parser. If you want to check for the end of the stream, consider using the `eof` parser."
    | _   -> charReturnE c result (expectedString (string c))

let pchar    c = charReturn c c
let skipChar c = charReturn c ()


/// returns true for chars '\u000E' - '\ufffe'
let inline internal isCertainlyNoNLOrEOS (c: char) =
    // '\n' = '\u000A', '\r' = '\u000D'
    unativeint c - 0xEun < unativeint EOS - 0xEun

let anyChar : Parser<char,'u> =
    fun stream ->
        let c = stream.ReadCharOrNewline()
        if c <> EOS then Reply(c)
        else Reply(Error, Errors.ExpectedAnyChar)

let skipAnyChar : Parser<unit,'u> =
    fun stream ->
        if stream.ReadCharOrNewline() <> EOS then Reply(())
        else Reply(Error, Errors.ExpectedAnyChar)


// doesn't check for newlines or EOS
let
#if NOINLINE
#else
    inline
#endif
           internal fastInlineSatisfyE f error : Parser<char,'u> =
    fun stream ->
        let c = stream.Peek()
        if f c then
            stream.Skip()
            Reply(c)
        else
            Reply(Error, error)

let internal satisfyE f error : Parser<char,'u> =
    fun stream ->
        let mutable reply = Reply()
        match stream.Peek() with
        | c when isCertainlyNoNLOrEOS c ->
            if f c then
                stream.Skip()
                reply.Status <- Ok
                reply.Result <- c
            else
                reply.Error <- error
        | '\r' | '\n' ->
            if f '\n' then
                stream.SkipNewline() |> ignore
                reply.Status <- Ok
                reply.Result <- '\n'
            else
                reply.Error <- error
        | c ->
             if c <> EOS && f c then
                stream.Skip()
                reply.Status <- Ok
                reply.Result <- c
             else
                reply.Error <- error
        reply

let internal skipSatisfyE f error : Parser<unit,'u> =
    fun stream ->
        let mutable reply = Reply()
        match stream.Peek() with
        | c when isCertainlyNoNLOrEOS c ->
            if f c then
                stream.Skip()
                reply.Status <- Ok
            else
                reply.Error <- error
        | '\r' | '\n' ->
            if f '\n' then
                stream.SkipNewline() |> ignore
                reply.Status <- Ok
            else
                reply.Error <- error
        | c ->
             if c <> EOS && f c then
                stream.Skip()
                reply.Status <- Ok
             else
                reply.Error <- error
        reply

let satisfy f        = satisfyE f NoErrorMessages
let satisfyL f label = satisfyE f (expected label)

let skipSatisfy f        = skipSatisfyE f NoErrorMessages
let skipSatisfyL f label = skipSatisfyE f (expected label)


let private charsToString (chars: seq<char>) =
#if PCL
    match box chars with
#else
    match chars with
#endif
    | :? string as str -> str
    | _ -> new string(Array.ofSeq chars)

let private stringToChars (str: string) =
#if PCL
    match box str with
    | :? seq<char> as chars -> chars
    | _ -> seq { for i = 0 to str.Length - 1 do yield str.[i] }
#else
    str
#endif


let isAnyOf (chars: seq<char>) =
#if LOW_TRUST
    let cs = new CharSet(charsToString chars)
    fun c -> cs.Contains(c)
#else
    #if USE_STATIC_MAPPING_FOR_IS_ANY_OF
         StaticMapping.createStaticCharIndicatorFunction false chars
    #else
        let cs = new CharSet(charsToString chars)
        fun c -> cs.Contains(c)
    #endif
#endif

let isNoneOf (chars: seq<char>) =
#if LOW_TRUST
    let cs = new CharSet(charsToString chars)
    fun c -> not (cs.Contains(c))
#else
    #if USE_STATIC_MAPPING_FOR_IS_ANY_OF
        StaticMapping.createStaticCharIndicatorFunction true chars
    #else
        let cs = new CharSet(charsToString chars)
        fun c -> not (cs.Contains(c))
    #endif
#endif

let anyOf (chars: seq<char>) =
    let str = charsToString chars
    let chars = stringToChars str // PCL workaround
    satisfyE (isAnyOf chars) (Errors.ExpectedAnyCharIn(str))

let skipAnyOf (chars: seq<char>) =
    let str = charsToString chars
    let chars = stringToChars str // PCL workaround
    skipSatisfyE (isAnyOf chars) (Errors.ExpectedAnyCharIn(str))

let noneOf (chars: seq<char>) =
    let str = charsToString chars
    let chars = stringToChars str // PCL workaround
    satisfyE (isNoneOf chars) (Errors.ExpectedAnyCharNotIn(str))

let skipNoneOf (chars: seq<char>) =
    let str = charsToString chars
    let chars = stringToChars str // PCL workaround
    skipSatisfyE (isNoneOf chars) (Errors.ExpectedAnyCharNotIn(str))

let inline isAsciiUpper (c: char) =
    uint32 c - uint32 'A' <= uint32 'Z' - uint32 'A'

let inline isAsciiLower (c: char) =
    uint32 c - uint32 'a' <= uint32 'z' - uint32 'a'

let inline isAsciiLetter (c: char) =
    let cc = uint32 c ||| uint32 ' '
    cc - uint32 'a' <= uint32 'z' - uint32 'a'

let inline isUpper (c: char) =
    isAsciiUpper c || (c > '\u007F' && System.Char.IsUpper(c))

let inline isLower (c: char) =
    isAsciiLower c || (c > '\u007F' && System.Char.IsLower(c))

let inline isLetter (c: char) =
    isAsciiLetter c || (c > '\u007F' && System.Char.IsLetter(c))

let inline isDigit (c: char) =
    uint32 c - uint32 '0' <= uint32 '9' - uint32 '0'

let inline isHex (c: char) =
    let cc = uint32 c ||| uint32 ' '
    isDigit c || cc - uint32 'a' <= uint32 'f' - uint32 'a'

let inline isOctal (c: char) =
    uint32 c - uint32 '0' <= uint32 '7' - uint32 '0'

let asciiUpper  stream = fastInlineSatisfyE isAsciiUpper  Errors.ExpectedAsciiUppercaseLetter stream
let asciiLower  stream = fastInlineSatisfyE isAsciiLower  Errors.ExpectedAsciiLowercaseLetter stream
let asciiLetter stream = fastInlineSatisfyE isAsciiLetter Errors.ExpectedAsciiLetter stream

// unicode is the default for letters and ascii the default for numbers
let upper  stream = fastInlineSatisfyE isUpper  Errors.ExpectedUppercaseLetter stream
let lower  stream = fastInlineSatisfyE isLower  Errors.ExpectedLowercaseLetter stream
let letter stream = fastInlineSatisfyE isLetter Errors.ExpectedLetter          stream

let digit  stream = fastInlineSatisfyE isDigit  Errors.ExpectedDecimalDigit     stream
let hex    stream = fastInlineSatisfyE isHex    Errors.ExpectedHexadecimalDigit stream
let octal  stream = fastInlineSatisfyE isOctal  Errors.ExpectedOctalDigit       stream

let tab stream = fastInlineSatisfyE ((=) '\t') Errors.ExpectedTab stream

let spaces : Parser<unit,'u> =
    fun stream ->
        stream.SkipWhitespace() |> ignore
        Reply(())

let spaces1 : Parser<unit,'u> =
    fun stream ->
        if stream.SkipWhitespace() then Reply(())
        else Reply(Error, Errors.ExpectedWhitespace)

let unicodeSpaces : Parser<unit,'u> =
    fun stream ->
        stream.SkipUnicodeWhitespace() |> ignore
        Reply(())

let unicodeSpaces1 : Parser<unit,'u> =
    fun stream ->
        if stream.SkipUnicodeWhitespace() then Reply(())
        else Reply(Error, Errors.ExpectedWhitespace)

let eof : Parser<unit,'u>=
    fun stream ->
        if stream.IsEndOfStream then Reply(())
        else Reply(Error, Errors.ExpectedEndOfInput)


// ------------------------
// Parsing strings directly
// ------------------------

let internal newlineOrEOSCharInStringArg name (arg: string) i =
    let msg2 = match arg.[i] with
               |'\r'|'\n' -> " may not contain newline chars ('\r' or '\n')."
               | EOS      -> " may not contain the char '\uffff' (EOS)"
               | _        -> failwith "newlineOrEOSCharInStringArg"
    raise (System.ArgumentException(concat3 "The string argument to " name msg2))

let internal checkStringContainsNoNewlineOrEOSChar s name =
    let i = findNewlineOrEOSChar s
    if i >= 0 then newlineOrEOSCharInStringArg name s i

let stringReturn s result : Parser<'a,'u> =
    let inline checkNoNewlineOrEOSChar c i =
        if not (isCertainlyNoNLOrEOS c) then
            match c with
            |'\r'|'\n'|EOS -> newlineOrEOSCharInStringArg "pstring/skipString/stringReturn" s i
            | _ -> ()

    let error = expectedString s
    match s.Length with
    | 0 -> preturn result
    | 1 ->
        let c = s.[0]
        checkNoNewlineOrEOSChar c 0
        charReturnE c result error
    | 2 ->
        let c0, c1 = s.[0], s.[1]
        checkNoNewlineOrEOSChar c0 0
        checkNoNewlineOrEOSChar c1 1
        let cs = TwoChars(c0, c1)
        fun stream ->
            if stream.Skip(cs) then Reply(result)
            else Reply(Error, error)
    | _ ->
        checkStringContainsNoNewlineOrEOSChar s "pstring/skipString/stringReturn"
        fun stream ->
            if stream.Skip(s) then Reply(result)
            else Reply(Error, error)

let pstring s    = stringReturn s s
let skipString s = stringReturn s ()

let pstringCI s : Parser<string,'u> =
    checkStringContainsNoNewlineOrEOSChar s "pstringCI"
    let error = expectedStringCI s
    let cfs = foldCase s
    fun stream ->
        let index0 = stream.IndexToken
        if stream.SkipCaseFolded(cfs) then
             Reply(stream.ReadFrom(index0))
        else Reply(Error, error)

let stringCIReturn (s: string) result : Parser<'a,'u> =
    let error = expectedStringCI s
    if s.Length = 1 then
        let c = s.[0]
        if not (isCertainlyNoNLOrEOS c) then
            match c with '\r'|'\n'|EOS -> newlineOrEOSCharInStringArg "skipStringCI/stringCIReturn"  s 0 | _ -> ()
        let cfc = Text.FoldCase(c)
        fun stream ->
            if stream.SkipCaseFolded(cfc) then Reply(result)
            else Reply(Error, error)
    else
        checkStringContainsNoNewlineOrEOSChar s "skipStringCI/stringCIReturn"
        let cfs = foldCase s
        fun stream ->
            if stream.SkipCaseFolded(cfs) then Reply(result)
            else Reply(Error, error)

let skipStringCI s = stringCIReturn s ()


let anyString n : Parser<string,'u> =
    let error = Errors.ExpectedAnySequenceOfNChars(n)
    fun stream ->
        let state = stream.State
        let str = stream.ReadCharsOrNewlines(n, true)
        if str.Length = n then Reply(str)
        else
            stream.BacktrackTo(state)
            Reply(Error, error)

let skipAnyString n : Parser<unit,'u> =
    let error = Errors.ExpectedAnySequenceOfNChars(n)
    fun stream ->
        let state = stream.State
        if stream.SkipCharsOrNewlines(n) = n then Reply(())
        else
            stream.BacktrackTo(state)
            Reply(Error, error)

let restOfLine skipNewline : Parser<_,_> =
    fun stream ->
        Reply(stream.ReadRestOfLine(skipNewline))

let skipRestOfLine skipNewline : Parser<_,_> =
    fun stream ->
         stream.SkipRestOfLine(skipNewline)
         Reply(())

let charsTillString (s: string) skipString maxCount : Parser<string,'u> =
    checkStringContainsNoNewlineOrEOSChar s "charsTillString"
    if maxCount < 0 then raise (System.ArgumentOutOfRangeException("maxCount", "maxCount is negative."))
    let error = Errors.CouldNotFindString(s)
    fun stream ->
        let mutable charsBeforeString = null
        stream.SkipCharsOrNewlinesUntilString(s, maxCount, true, &charsBeforeString) |> ignore
        if isNotNull charsBeforeString then
            if skipString then stream.Skip(s.Length)
            Reply(charsBeforeString)
        else
            Reply(Error, error)

let charsTillStringCI (s: string) skipString maxCount : Parser<string,'u> =
    checkStringContainsNoNewlineOrEOSChar s "charsTillStringCI"
    if maxCount < 0 then raise (System.ArgumentOutOfRangeException("maxCount", "maxCount is negative."))
    let cfs = foldCase s
    let error = Errors.CouldNotFindCaseInsensitiveString(s)
    fun stream ->
        let mutable charsBeforeString = null
        stream.SkipCharsOrNewlinesUntilCaseFoldedString(cfs, maxCount, true, &charsBeforeString) |> ignore
        if isNotNull charsBeforeString then
            if skipString then stream.Skip(s.Length)
            Reply(charsBeforeString)
        else
            Reply(Error, error)


let skipCharsTillString (s: string) skipString maxCount : Parser<unit,'u> =
    checkStringContainsNoNewlineOrEOSChar s "skipCharsTillString"
    if maxCount < 0 then raise (System.ArgumentOutOfRangeException("maxCount", "maxCount is negative."))
    let error = Errors.CouldNotFindString(s)
    fun stream ->
        let mutable foundString = false
        stream.SkipCharsOrNewlinesUntilString(s, maxCount, &foundString) |> ignore
        if foundString then
            if skipString then stream.Skip(s.Length)
            Reply(())
        else
            Reply(Error, error)

let skipCharsTillStringCI (s: string) skipString maxCount : Parser<unit,'u> =
    checkStringContainsNoNewlineOrEOSChar s "skipCharsTillStringCI"
    if maxCount < 0 then raise (System.ArgumentOutOfRangeException("maxCount", "maxCount is negative."))
    let cfs = foldCase s
    let error = Errors.CouldNotFindCaseInsensitiveString(s)
    fun stream ->
        let mutable foundString = false
        stream.SkipCharsOrNewlinesUntilCaseFoldedString(cfs, maxCount, &foundString) |> ignore
        if foundString then
            if skipString then stream.Skip(s.Length)
            Reply(())
        else
            Reply(Error, error)

let
#if NOINLINE
#else
    inline
#endif
           internal manySatisfyImpl require1 (f1: char -> bool) (f: char -> bool) error : Parser<string,'u> =
    fun stream ->
        let str = stream.ReadCharsOrNewlinesWhile(f1, f, true)
        if not require1 || str.Length <> 0 then Reply(str)
        else Reply(Error, error)

let
#if NOINLINE
#else
    inline
#endif
           internal skipManySatisfyImpl require1 (f1: char -> bool) (f: char -> bool) error : Parser<unit,'u> =
    fun stream ->
        let n = stream.SkipCharsOrNewlinesWhile(f1, f)
        if not require1 || n <> 0 then Reply(())
        else Reply(Error, error)

let manySatisfy2   f1 f       = manySatisfyImpl false f1 f NoErrorMessages
let many1Satisfy2  f1 f       = manySatisfyImpl true  f1 f NoErrorMessages
let many1Satisfy2L f1 f label = manySatisfyImpl true  f1 f (expected label)

let skipManySatisfy2   f1 f       = skipManySatisfyImpl false f1 f NoErrorMessages
let skipMany1Satisfy2  f1 f       = skipManySatisfyImpl true  f1 f NoErrorMessages
let skipMany1Satisfy2L f1 f label = skipManySatisfyImpl true  f1 f (expected label)

let manySatisfy   f       = manySatisfy2   f f
let many1Satisfy  f       = many1Satisfy2  f f
let many1SatisfyL f label = many1Satisfy2L f f label

let skipManySatisfy   f       = skipManySatisfy2   f f
let skipMany1Satisfy  f       = skipMany1Satisfy2  f f
let skipMany1SatisfyL f label = skipMany1Satisfy2L f f label


let internal manyMinMaxSatisfy2E minCount maxCount f1 f error : Parser<string,'u> =
    if maxCount < 0 then raise (System.ArgumentOutOfRangeException("maxCount", "maxCount is negative."))
    if minCount > 0 then
        fun stream ->
            let str = stream.ReadCharsOrNewlinesWhile(f1, f, minCount, maxCount, true)
            if str.Length <> 0 then Reply(str)
            else Reply(Error, error)
    else
        fun stream ->
            Reply(stream.ReadCharsOrNewlinesWhile(f1, f, 0, maxCount, true))

let internal skipManyMinMaxSatisfy2E minCount maxCount f1 f error : Parser<unit,'u> =
    if maxCount < 0 then raise (System.ArgumentOutOfRangeException("maxCount", "maxCount is negative."))
    if minCount > 0 then
        fun stream ->
            let n = stream.SkipCharsOrNewlinesWhile(f1, f, minCount, maxCount)
            if n <> 0 then Reply(())
            else Reply(Error, error)
    else
        fun stream ->
            stream.SkipCharsOrNewlinesWhile(f1, f, 0, maxCount) |> ignore
            Reply(())

let manyMinMaxSatisfy   minCount maxCount    f       = manyMinMaxSatisfy2E minCount maxCount f  f NoErrorMessages
let manyMinMaxSatisfyL  minCount maxCount    f label = manyMinMaxSatisfy2E minCount maxCount f  f (expected label)
let manyMinMaxSatisfy2  minCount maxCount f1 f       = manyMinMaxSatisfy2E minCount maxCount f1 f NoErrorMessages
let manyMinMaxSatisfy2L minCount maxCount f1 f label = manyMinMaxSatisfy2E minCount maxCount f1 f (expected label)

let skipManyMinMaxSatisfy   minCount maxCount    f       = skipManyMinMaxSatisfy2E minCount maxCount f  f NoErrorMessages
let skipManyMinMaxSatisfyL  minCount maxCount    f label = skipManyMinMaxSatisfy2E minCount maxCount f  f (expected label)
let skipManyMinMaxSatisfy2  minCount maxCount f1 f       = skipManyMinMaxSatisfy2E minCount maxCount f1 f NoErrorMessages
let skipManyMinMaxSatisfy2L minCount maxCount f1 f label = skipManyMinMaxSatisfy2E minCount maxCount f1 f (expected label)


let internal regexE pattern error : Parser<string,'u> =
    let regex = new Regex("\\A" + pattern, RegexOptions.Multiline |||
                                           RegexOptions.ExplicitCapture)
    fun stream ->
        let m = stream.Match(regex)
        if m.Success then
            let str = m.Value
            if findNewlineOrEOSChar str < 0 then
                if str.Length <> 0 then stream.Skip(str.Length)
                Reply(str)
            else
                let nStr = normalizeNewlines str
                let mutable nSkippedChars = 0
                let n = stream.SkipCharsOrNewlines(nStr.Length)
                if n = nStr.Length then Reply(nStr)
                else Reply(FatalError, messageError "Internal error in the regex parser. Please report this error to <EMAIL>.")
        else Reply(Error, error)

let regex  pattern       = regexE pattern (Errors.ExpectedStringMatchingRegex(pattern))
let regexL pattern label = regexE pattern (expected label)

type private IdFlags = IdentifierValidator.IdentifierCharFlags

type IdentifierOptions(?isAsciiIdStart, ?isAsciiIdContinue,
                   #if PCL
                   #else
                       ?normalization,
                       ?normalizeBeforeValidation,
                   #endif
                       ?allowJoinControlChars, ?preCheckStart, ?preCheckContinue, ?allowAllNonAsciiCharsInPreCheck, ?label, ?invalidCharMessage) =
    // we use match instead of defaultArg here, so that the function wrapper objects only get constructed when needed
    let isAsciiIdStart    = match isAsciiIdStart    with Some v -> v | _ -> IdentifierValidator.IsXIdStartOrSurrogate
    let isAsciiIdContinue = match isAsciiIdContinue with Some v -> v | _ -> IdentifierValidator.IsXIdContinueOrSurrogate
#if PCL
#else
    let normalizationForm = defaultArg normalization (enum<NormalizationForm> 0)
    let normalizeBeforeValidation = defaultArg normalizeBeforeValidation false
#endif
    let allowJoinControlChars = defaultArg allowJoinControlChars false
    let expectedIdentifierError = expected (defaultArg label Strings.Identifier)
    let invalidCharError = messageError (defaultArg invalidCharMessage Strings.IdentifierContainsInvalidCharacterAtIndicatedPosition)
    let allowAllNonAsciiCharsInPreCheck = defaultArg allowAllNonAsciiCharsInPreCheck false

    let preCheckStart = if preCheckStart.IsSome then preCheckStart.Value
                        elif allowAllNonAsciiCharsInPreCheck then isAsciiIdStart
                        else Unchecked.defaultof<_>
    let preCheckContinue = if preCheckContinue.IsSome then preCheckContinue.Value
                           elif allowAllNonAsciiCharsInPreCheck then isAsciiIdContinue
                           else Unchecked.defaultof<_>

    let asciiOptions = Array.zeroCreate 128
    do for i = 1 to 127 do
          let c = char i
          let mutable v = IdFlags.None
          if isAsciiIdStart c then v <- v ||| IdFlags.NonContinue
          if isAsciiIdContinue c then v <- v ||| IdFlags.Continue
          if allowAllNonAsciiCharsInPreCheck then
             if preCheckStart c then v <- v ||| IdFlags.PreCheckNonContinue
             if preCheckContinue c then v <- v ||| IdFlags.PreCheckContinue
          asciiOptions.[i] <- v

    let iv = new IdentifierValidator(asciiOptions)
    do
    #if PCL
    #else
       iv.NormalizationForm <- normalizationForm
       iv.NormalizeBeforeValidation <- normalizeBeforeValidation
    #endif
       iv.AllowJoinControlCharsAsIdContinueChars <- allowJoinControlChars

    let preCheck1 =
        if allowAllNonAsciiCharsInPreCheck then
            fun c -> let i = int c
                     if i <= 0x7f then
                         // not (x = y) currently yields better code here than (x <> y)
                         not (asciiOptions.[int c] &&& IdFlags.PreCheckNonContinue = IdFlags.None)
                     else true
        elif isNotNull preCheckStart then preCheckStart
        else iv.IsIdStartOrSurrogateFunc

    let preCheck =
        if allowAllNonAsciiCharsInPreCheck then
            fun c -> let i = int c
                     if i <= 0x7f then
                         not (asciiOptions.[i] &&& IdFlags.PreCheckContinue = IdFlags.None)
                     else true
        elif isNotNull preCheckContinue then preCheckContinue
        else iv.IsIdContinueOrJoinControlOrSurrogateFunc

    member internal t.IdentifierValidator = iv
    member internal t.PreCheck1 = preCheck1
    member internal t.PreCheck  = preCheck
    member internal t.ExpectedIdentifierError = expectedIdentifierError
    member internal t.InvalidCharError = invalidCharError

let identifier (identifierOptions: IdentifierOptions) : Parser<string, _> =
    let validator = identifierOptions.IdentifierValidator
    let preCheck1 = identifierOptions.PreCheck1
    let preCheck  = identifierOptions.PreCheck
    let expectedIdentifierError = identifierOptions.ExpectedIdentifierError
    let invalidCharError = identifierOptions.InvalidCharError
    fun stream ->
        let str = stream.ReadCharsOrNewlinesWhile(preCheck1, preCheck, true)
        if str.Length <> 0 then
            let mutable errorPos = 0
            let nstr = validator.ValidateAndNormalize(str, &errorPos)
            if isNotNull nstr then Reply(nstr)
            else
                stream.Skip(errorPos - str.Length)
                Reply(FatalError, invalidCharError)
        else
            Reply(Error, expectedIdentifierError)

// ----------------------------------------------
// Parsing strings with the help of other parsers
// ----------------------------------------------


let manyChars2 p1 p = ManyChars(p1, p).AsFSharpFunc
let manyChars     p = manyChars2 p p

let many1Chars2 p1 p = Many1Chars(p1, p).AsFSharpFunc
let many1Chars     p = many1Chars2 p p

let manyCharsTillApply2 p1 p endp f = ManyCharsTill(p1, p, endp, f).AsFSharpFunc
let manyCharsTillApply     p endp f = manyCharsTillApply2 p p endp f
let manyCharsTill2      p1 p endp   = manyCharsTillApply2 p1 p endp (fun str _ -> str)
let manyCharsTill          p endp   = manyCharsTill2 p p endp

let many1CharsTillApply2 p1 p endp f = Many1CharsTill(p1, p, endp, f).AsFSharpFunc
let many1CharsTillApply     p endp f = many1CharsTillApply2 p p endp f
let many1CharsTill2      p1 p endp   = many1CharsTillApply2 p1 p endp (fun str _ -> str)
let many1CharsTill          p endp   = many1CharsTill2 p p endp



let
#if NOINLINE
#else
    inline
#endif
           internal manyStringsImpl require1 (p1: Parser<string,'u>) (p: Parser<string,'u>) : Parser<string,'u> =
    fun stream ->
        let mutable stateTag = stream.StateTag
        let mutable reply = p1 stream
        if reply.Status = Ok then
            let result1 = reply.Result
            let mutable error  = reply.Error
            stateTag <- stream.StateTag
            reply <- p stream
            if reply.Status <> Ok then reply.Result <- result1
            else
                let result2 = reply.Result
                error <- reply.Error
                stateTag <- stream.StateTag
                reply <- p stream
                if reply.Status <> Ok then reply.Result <- result1 + result2
                else
                    let result3 = reply.Result
                    error <- reply.Error
                    stateTag <- stream.StateTag
                    reply <- p stream
                    if reply.Status <> Ok then reply.Result <- concat3 result1 result2 result3
                    else
                        let result4 = reply.Result
                        error <- reply.Error
                        stateTag <- stream.StateTag
                        reply <- p stream
                        if reply.Status <> Ok then reply.Result <- concat4 result1 result2 result3 result4
                        else
                            let n = 2*(result1.Length + result2.Length + result3.Length + result4.Length) + reply.Result.Length
                            let sb = new StringBuilder(n)
                            sb.Append(result1).Append(result2).Append(result3).Append(result4).Append(reply.Result) |> ignore
                            error <- reply.Error
                            stateTag <- stream.StateTag
                            reply <- p stream
                            while reply.Status = Ok do
                                if stateTag = stream.StateTag then
                                    raiseInfiniteLoopException "manyStrings" stream
                                error <- reply.Error
                                sb.Append(reply.Result) |> ignore
                                stateTag <- stream.StateTag
                                reply <- p stream
                            reply.Result <- sb.ToString()
            // We assume that the string parser changes the state when it succeeds,
            // so we don't need to merge more than 2 error message lists.
            if stateTag = stream.StateTag then
                if reply.Status = Error then
                    reply.Status <- Ok
                if isNotNull error then
                    reply.Error <- mergeErrors error reply.Error
        elif not require1 && reply.Status = Error && stateTag = stream.StateTag then
            reply.Status <- Ok
            reply.Result <- ""
        reply

let manyStrings2 p1 p = manyStringsImpl false p1 p
let manyStrings p = manyStrings2 p p
let many1Strings2 p1 p = manyStringsImpl true p1 p
let many1Strings p = many1Strings2 p p

let stringsSepBy (p: Parser<string,'u>) (sep: Parser<string,'u>) : Parser<string,'u> =
    fun stream ->
        let mutable stateTag = stream.StateTag
        let mutable reply = p stream
        if reply.Status = Ok then
            let result1 = reply.Result
            let mutable error = reply.Error
            stateTag <- stream.StateTag
            reply <- sep stream
            if reply.Status <> Ok then
                if stateTag = stream.StateTag then
                    if reply.Status = Error then
                        reply.Status <- Ok
                        reply.Result <- result1
                    if isNotNull error then
                        reply.Error <- mergeErrors error reply.Error
            else
                // We assume that at least one of the parsers sep and p consume
                // input when both are called consecutively and succeed. This
                // way we only have to merge a maximum of 3 error message lists.
                let mutable result = null
                let mutable error0 = error
                let mutable stateTag0 = stateTag
                let result2 = reply.Result
                error <- reply.Error
                stateTag <- stream.StateTag
                reply <- p stream
                if reply.Status = Ok then
                    let result3 = reply.Result
                    error0 <- error
                    stateTag0 <- stateTag
                    error <- reply.Error
                    stateTag <- stream.StateTag
                    reply <- sep stream
                    if reply.Status <> Ok then result <- concat3 result1 result2 result3
                    else
                        let result4 = reply.Result
                        error0 <- error
                        stateTag0 <- stateTag
                        error <- reply.Error
                        stateTag <- stream.StateTag
                        reply <- p stream
                        if reply.Status = Ok then
                            let n = 2*(result1.Length + result2.Length + result3.Length + result4.Length) + reply.Result.Length
                            let sb = new StringBuilder(n)
                            sb.Append(result1).Append(result2).Append(result3).Append(result4) |> ignore
                            while reply.Status = Ok do
                                sb.Append(reply.Result) |> ignore
                                error0 <- error
                                stateTag0 <- stateTag
                                error <- reply.Error
                                stateTag <- stream.StateTag
                                reply <- sep stream
                                if reply.Status <> Ok then result <- sb.ToString()
                                else
                                    sb.Append(reply.Result) |> ignore
                                    if stateTag0 = stream.StateTag then
                                        raiseInfiniteLoopException "stringsSepBy" stream
                                    error0 <- error
                                    stateTag0 <- stateTag
                                    error <- reply.Error
                                    stateTag <- stream.StateTag
                                    reply <- p stream
                if stateTag = stream.StateTag then
                    if isNotNull result && reply.Status = Error then
                        reply.Status <- Ok
                        reply.Result <- result
                    error <- mergeErrors error reply.Error
                    if stateTag0 = stateTag then
                        error <- mergeErrors error0 error
                    reply.Error <- error
        elif reply.Status = Error && stateTag = stream.StateTag then
            reply.Status <- Ok
            reply.Result <- ""
        reply

let skipped (p: Parser<unit,'u>) : Parser<string,'u> =
    fun stream ->
        let index0 = stream.IndexToken
        let line0 = stream.Line
        let reply = p stream
        if reply.Status = Ok then
            let str = stream.ReadFrom(index0)
            let nstr = if line0 = stream.Line then str
                       else Text.NormalizeNewlines(str)
            Reply(Ok, nstr, reply.Error)
        else
            Reply(reply.Status, reply.Error)

let withSkippedString (f: string -> 'a -> 'b) (p: Parser<'a,'u>) : Parser<'b,'u> =
    let optF = OptimizedClosures.FSharpFunc<_,_,_>.Adapt(f)
    fun stream ->
        let index0 = stream.IndexToken
        let line0 = stream.Line
        let reply = p stream
        if reply.Status = Ok then
            let str = stream.ReadFrom(index0)
            let nstr = if line0 = stream.Line then str
                       else Text.NormalizeNewlines(str)
            let result = optF.Invoke(nstr, reply.Result)
            Reply(Ok, result, reply.Error)
        else
            Reply(reply.Status, reply.Error)

// ---------------
// Parsing numbers
// ---------------

[<System.Flags>]
type NumberLiteralOptions =
     | None                             = 0
     | AllowSuffix                      = 0b000000000001
     | AllowMinusSign                   = 0b000000000010
     | AllowPlusSign                    = 0b000000000100
     | AllowFraction                    = 0b000000001000
     | AllowFractionWOIntegerPart       = 0b000000010000
     | AllowExponent                    = 0b000000100000
     | AllowHexadecimal                 = 0b000001000000
     | AllowBinary                      = 0b000010000000
     | AllowOctal                       = 0b000100000000
     | AllowInfinity                    = 0b001000000000
     | AllowNaN                         = 0b010000000000

     | IncludeSuffixCharsInString       = 0b100000000000

     | DefaultInteger                   = 0b000111000110
     | DefaultUnsignedInteger           = 0b000111000000
     | DefaultFloat                     = 0b011001101110

type internal NLO = NumberLiteralOptions

[<System.Flags>]
type NumberLiteralResultFlags =
     | None             = 0
     | SuffixLengthMask = 0b0000000000001111
     | HasMinusSign     = 0b0000000000010000
     | HasPlusSign      = 0b0000000000100000
     | HasIntegerPart   = 0b0000000001000000
     | HasFraction      = 0b0000000010000000
     | HasExponent      = 0b0000000100000000
     | IsDecimal        = 0b0000001000000000
     | IsHexadecimal    = 0b0000010000000000
     | IsBinary         = 0b0000100000000000
     | IsOctal          = 0b0001000000000000
     | BaseMask         = 0b0001111000000000
     | IsInfinity       = 0b0010000000000000
     | IsNaN            = 0b0100000000000000

type internal NLF = NumberLiteralResultFlags

type NumberLiteral(string, info, suffixChar1, suffixChar2, suffixChar3, suffixChar4) =
    member t.String = string

    member t.SuffixLength = int (info &&& NLF.SuffixLengthMask)
    member t.SuffixChar1  = suffixChar1
    member t.SuffixChar2  = suffixChar2
    member t.SuffixChar3  = suffixChar3
    member t.SuffixChar4  = suffixChar4

    member t.Info = info

    member t.HasMinusSign   = int (info &&& NLF.HasMinusSign) <> 0
    member t.HasPlusSign    = int (info &&& NLF.HasPlusSign) <> 0
    member t.HasIntegerPart = int (info &&& NLF.HasIntegerPart) <> 0
    member t.HasFraction    = int (info &&& NLF.HasFraction) <> 0
    member t.HasExponent    = int (info &&& NLF.HasExponent) <> 0
    member t.IsInteger      = int (info &&& (NLF.HasFraction ||| NLF.HasExponent)) = 0 // HasIntegerPart must be set if HasFraction and HasExponent both aren't
    member t.IsDecimal      = int (info &&& NLF.IsDecimal) <> 0
    member t.IsHexadecimal  = int (info &&& NLF.IsHexadecimal) <> 0
    member t.IsBinary       = int (info &&& NLF.IsBinary) <> 0
    member t.IsOctal        = int (info &&& NLF.IsOctal) <> 0
    member t.IsNaN          = int (info &&& NLF.IsNaN) <> 0
    member t.IsInfinity     = int (info &&& NLF.IsInfinity) <> 0

    override t.Equals(other: obj) =
        match other with
        | :? NumberLiteral as other ->
               t.String = other.String
            && t.Info = other.Info
            && t.SuffixChar1 = other.SuffixChar1
            && t.SuffixChar2 = other.SuffixChar2
            && t.SuffixChar3 = other.SuffixChar3
            && t.SuffixChar4 = other.SuffixChar4
        | _ -> false

    override t.GetHashCode() =
        if isNotNull string then string.GetHashCode() else 0

let numberLiteralE (opt: NumberLiteralOptions) (errorInCaseNoLiteralFound: ErrorMessageList) (stream: CharStream<'u>) =
    let index0 = stream.IndexToken
    let stateTag = stream.StateTag
    let mutable c = stream.Peek()
    let mutable error = NoErrorMessages
    let mutable flags = NLF.None

    if c = '-' && (opt &&& NLO.AllowMinusSign) <> NLO.None then
        flags <- NLF.HasMinusSign
        c <- stream.SkipAndPeek()
    elif c = '+' && (opt &&& NLO.AllowPlusSign) <> NLO.None then
        flags <- NLF.HasPlusSign
        c <- stream.SkipAndPeek()

    let allowStartingPoint = NLO.AllowFraction ||| NLO.AllowFractionWOIntegerPart // for starting point both flags are required

    if isDigit c || (c = '.' && (opt &&& allowStartingPoint) = allowStartingPoint) then
        let mutable c1  = '\u0000'
        if    c <> '0'
           || (c1 <- stream.SkipAndPeek();
                  c1 <= '9'
               || (opt &&& (NLO.AllowBinary ||| NLO.AllowOctal ||| NLO.AllowHexadecimal)) = NLO.None
               || ((int c1 ||| int ' ') = int 'e'))
        then
            flags <- flags ||| NLF.IsDecimal
            if c <> '.' then
                flags <- flags ||| NLF.HasIntegerPart
                if c <> '0' then
                    c <- stream.SkipAndPeek()
                else
                    c <- c1
                while isDigit c do
                    c <- stream.SkipAndPeek()
            if c = '.' && (opt &&& NLO.AllowFraction) <> NLO.None then
                flags <- flags ||| NLF.HasFraction
                c <- stream.SkipAndPeek()
                if isDigit c then
                    c <- stream.SkipAndPeek()
                elif (flags &&& NLF.HasIntegerPart) = NLF.None then
                    // at least one digit before or after the . is required
                    error <- Errors.ExpectedDecimalDigit
                while isDigit c do
                    c <- stream.SkipAndPeek()
            if (int c ||| int ' ') = int 'e' && isNull error && (opt &&& NLO.AllowExponent) <> NLO.None then
                flags <- flags ||| NLF.HasExponent
                c <- stream.SkipAndPeek()
                if c = '-' || c = '+' then
                    c <- stream.SkipAndPeek()
                if not (isDigit c) then
                    error <- Errors.ExpectedDecimalDigit
                while isDigit c do
                    c <- stream.SkipAndPeek()
        else
            match int c1 ||| int ' ' with
            | 0x78 (* 'x' *) when (opt &&& NLO.AllowHexadecimal) <> NLO.None ->
                flags <- flags ||| NLF.IsHexadecimal
                c <- stream.SkipAndPeek()
                if isHex c then
                    flags <- flags ||| NLF.HasIntegerPart
                    c <- stream.SkipAndPeek()
                elif (opt &&& NLO.AllowFractionWOIntegerPart) = NLO.None then
                    // integer part required
                    error <- Errors.ExpectedHexadecimalDigit
                while isHex c do
                    c <- stream.SkipAndPeek()
                if c = '.' && isNull error && (opt &&& NLO.AllowFraction) <> NLO.None then
                    flags <- flags ||| NLF.HasFraction
                    c <- stream.SkipAndPeek()
                    if isHex c then
                        c <- stream.SkipAndPeek()
                    elif (flags &&& NLF.HasIntegerPart) = NLF.None then
                        // at least one digit before or after the . is required
                        error <- Errors.ExpectedHexadecimalDigit
                    while isHex c do
                        c <- stream.SkipAndPeek()
                elif (flags &&& NLF.HasIntegerPart) = NLF.None then
                    // we neither have an integer part nor a fraction
                    error <- Errors.ExpectedHexadecimalDigit
                if (int c ||| int ' ') = int 'p' && isNull error && (opt &&& NLO.AllowExponent) <> NLO.None then
                    flags <- flags ||| NLF.HasExponent
                    c <- stream.SkipAndPeek()
                    if c = '-' || c = '+' then
                        c <- stream.SkipAndPeek()
                    if not (isDigit c) then
                        error <- Errors.ExpectedDecimalDigit
                    while isDigit c do
                        c <- stream.SkipAndPeek()
            | 0x6f (* 'o' *) when (opt &&& NLO.AllowOctal) <> NLO.None ->
                flags <- flags ||| NLF.IsOctal
                c <- stream.SkipAndPeek()
                if isOctal c then
                    flags <- flags ||| NLF.HasIntegerPart
                    c <- stream.SkipAndPeek()
                else
                    error <- Errors.ExpectedOctalDigit
                while isOctal c do
                    c <- stream.SkipAndPeek()
            | 0x62 (* 'b' *) when (opt &&& NLO.AllowBinary) <> NLO.None ->
                flags <- flags ||| NLF.IsBinary
                c <- stream.SkipAndPeek()
                if c = '0' || c = '1' then
                    flags <- flags ||| NLF.HasIntegerPart
                    c <- stream.SkipAndPeek()
                else
                    error <- Errors.ExpectedBinaryDigit
                while c = '0' || c = '1' do
                    c <- stream.SkipAndPeek()
            | _ ->
                flags <- flags ||| (NLF.IsDecimal ||| NLF.HasIntegerPart)
                c <- c1

        if isNull error then
            if (opt &&& NLO.AllowSuffix) = NLO.None  || not (isAsciiLetter c) then
                let str = stream.ReadFrom(index0)
                Reply(NumberLiteral(str, flags, EOS, EOS, EOS, EOS))
            else
                let mutable str = if (opt &&& NLO.IncludeSuffixCharsInString) <> NLO.None then null
                                  else stream.ReadFrom(index0)
                let mutable nSuffix = 1
                let mutable s1 = c
                let mutable s2 = EOS
                let mutable s3 = EOS
                let mutable s4 = EOS
                c <- stream.SkipAndPeek()
                if isAsciiLetter c then
                    nSuffix <- 2
                    s2 <- c
                    c <- stream.SkipAndPeek()
                    if isAsciiLetter c then
                        nSuffix <- 3
                        s3 <- c
                        c <- stream.SkipAndPeek()
                        if isAsciiLetter c then
                            nSuffix <- 4
                            s4 <- c
                            c <- stream.SkipAndPeek()
                flags <- flags ||| (enum) nSuffix
                if (opt &&& NLO.IncludeSuffixCharsInString) <> NLO.None then
                    str <- stream.ReadFrom(index0)
                Reply(NumberLiteral(str, flags, s1, s2, s3, s4))
        else
            Reply(Error, error)
    else
       let cc = int c ||| int ' '
       if
           if cc = int 'i' then
                 (opt &&& NLO.AllowInfinity) <> NLO.None
              && stream.SkipCaseFolded("inf") && (flags <- flags ||| NLF.IsInfinity
                                                  stream.SkipCaseFolded("inity") |> ignore
                                                  true)
           elif cc = int 'n' then
                   (opt &&& NLO.AllowNaN) <> NLO.None
                && stream.SkipCaseFolded("nan") && (flags <- flags ||| NLF.IsNaN
                                                    true)
           else false
       then
           let str = stream.ReadFrom(index0)
           Reply(NumberLiteral(str, flags, EOS, EOS, EOS, EOS))
       else
           if flags &&& (NLF.HasMinusSign |||  NLF.HasPlusSign) <> NLF.None then
              stream.Seek(index0)
              stream.StateTag <- stateTag
           Reply(Error, errorInCaseNoLiteralFound)

let numberLiteral opt label = numberLiteralE opt (expected label)

let pfloat : Parser<float,'u> =
    fun stream ->
        let reply = numberLiteralE NLO.DefaultFloat Errors.ExpectedFloatingPointNumber stream
        if reply.Status = Ok then
            let nl = reply.Result
            try
                let d = if nl.IsDecimal then
                            System.Double.Parse(nl.String, System.Globalization.CultureInfo.InvariantCulture)
                        elif nl.IsHexadecimal then
                            floatOfHexString nl.String
                        elif nl.IsInfinity then
                            if nl.HasMinusSign then System.Double.NegativeInfinity else System.Double.PositiveInfinity
                        else
                            System.Double.NaN
                Reply(d)
            with e ->
                let error = if   (e :? System.OverflowException) then Errors.NumberOutsideOfDoubleRange
                            elif (e :? System.FormatException) then messageError "The floating-point number has an invalid format (this error is unexpected, please report this error message to <EMAIL>)."
                            else reraise()
                stream.Skip(-nl.String.Length)
                Reply(FatalError, error)
        else
            Reply(reply.Status, reply.Error)

let internal parseUInt64 (c0: char) (stream: CharStream<'u>) (status: ReplyStatus byref) (error: ErrorMessageList byref) =
    Debug.Assert(isDigit c0 && (status = Ok))

    // we rely on the compiler eliminating inactive branches
    let opt = NumberLiteralOptions.DefaultUnsignedInteger
    let limit10  = 1844674407370955160UL //(System.UInt64.MaxValue - 9UL)/10UL
    let maxDiv10 = 1844674407370955161UL //System.UInt64.MaxValue/10UL
    let maxMod10 = 5u //System.UInt64.MaxValue%10UL

    let limit16  = 1152921504606846975UL //(System.UInt64.MaxValue - 15UL)/16UL
    let maxDiv16 = 1152921504606846975UL //System.UInt64.MaxValue/16UL
    let maxMod16 = 15u //System.UInt64.MaxValue%16UL

    let limit8  = 2305843009213693951UL  //(System.UInt64.MaxValue - 7UL)/8UL
    let maxDiv8 = 2305843009213693951UL //System.UInt64.MaxValue/8UL
    let maxMod8 = 7u //System.UInt64.MaxValue%8UL

    let limit2  = 9223372036854775807UL //(System.UInt64.MaxValue - 1UL)/2UL
    let maxDiv2 = 9223372036854775807UL //System.UInt64.MaxValue/2UL
    let maxMod2 = 1u //System.UInt64.MaxValue%2UL

    let mutable n = 0UL
    let mutable c = c0
    let c1 = stream.SkipAndPeek()

    if    (opt &&& (NLO.AllowBinary ||| NLO.AllowOctal ||| NLO.AllowHexadecimal)) = NLO.None
       || c <> '0' || c1 <= '9'
    then
        n <- uint64 (uint32 c - uint32 '0')
        c <- c1
        while c >= '0' && c <= '9' do
            let nc = uint32 c - uint32 '0'
            if n <= limit10 || (maxMod10 < 9u && n = maxDiv10 && nc <= maxMod10) then
                n <- 10UL*n + uint64 nc
                c <- stream.SkipAndPeek()
            else
                status <- FatalError
                c <- '!' // break

    else
        let cc1 = uint32 c1 ||| uint32 ' '
        if (opt &&& NLO.AllowHexadecimal) <> NLO.None && cc1 = uint32 'x' then
            c <- stream.SkipAndPeek()
            let mutable nc = uint32 0
            if  (let cc = uint32 c ||| uint32 ' '
                 if c <= '9' then nc <- uint32 c - uint32 '0'; c >= '0'
                 else cc <= uint32 'f' && (nc <- cc - 0x57u; cc >= uint32 'a')) // 0x57u = uint32 'a' - 10u
            then
                n <- uint64 nc
                c <- stream.SkipAndPeek()
                while
                    (let cc = uint32 c ||| uint32 ' '
                     if c <= '9' then nc <- uint32 c - uint32 '0'; c >= '0'
                     else cc <= uint32 'f' && (nc <- cc - 0x57u; cc >= uint32 'a'))
                  do
                    if n <= limit16 || (maxMod16 < 15u && n = maxDiv16 && nc <= maxMod16) then
                        n <- 16UL*n + uint64 nc
                        c <- stream.SkipAndPeek()
                    else
                        status <- FatalError
                        c <- '!' // break
            else
                status <- Error
                error <- Errors.ExpectedHexadecimalDigit

        elif (opt &&& NLO.AllowOctal) <> NLO.None && cc1 = uint32 'o' then
            c <- stream.SkipAndPeek()
            let mutable nc = uint32 c - uint32 '0'
            if nc = (nc &&& 7u) then
                n <- uint64 nc
                c <- stream.SkipAndPeek()
                nc <- uint32 c - uint32 '0'
                while nc = (nc &&& 7u) do
                    if n <= limit8 || (maxMod8 < 7u && n = maxDiv8 && nc <= maxMod8) then
                        n <- 8UL*n + uint64 nc
                        c <- stream.SkipAndPeek()
                        nc <- uint32 c - uint32 '0'
                    else
                        status <- FatalError
                        nc <- 11u // break
            else
                status <- Error
                error <- Errors.ExpectedOctalDigit

        elif (opt &&& NLO.AllowBinary) <> NLO.None && cc1 = uint32 'b' then
            c <- stream.SkipAndPeek()
            let mutable nc = uint32 c - uint32 '0'
            if nc = (nc &&& 1u) then
                n <- uint64 nc
                c <- stream.SkipAndPeek()
                nc <- uint32 c - uint32 '0'
                while nc = (nc &&& 1u) do
                    if n <= limit2 || (maxMod2 = 0u && n = maxDiv2 && nc = 0u) then
                        n <- 2UL*n + uint64 nc
                        c <- stream.SkipAndPeek()
                        nc <- uint32 c - uint32 '0'
                    else
                        status <- FatalError
                        nc <- 11u // break
            else
                status <- Error
                error <- Errors.ExpectedBinaryDigit
        // else c = 0 && not (isDigit c1)
    n

let internal parseUInt32 (c0: char) (stream: CharStream<'u>) (status: ReplyStatus byref) (error: ErrorMessageList byref) =
    Debug.Assert(isDigit c0 && (status = Ok))

    // we rely on the compiler eliminating inactive branches
    let opt = NumberLiteralOptions.DefaultUnsignedInteger
    let limit10  = 429496728u  //(System.UInt32.MaxValue - 9u)/10u
    let maxDiv10 = 429496729u //System.UInt32.MaxValue/10u
    let maxMod10 = 5u //System.UInt32.MaxValue%10u

    let limit16  = 268435455u  //(System.UInt32.MaxValue - 15u)/16u
    let maxDiv16 = 268435455u //System.UInt32.MaxValue/16u
    let maxMod16 = 15u //System.UInt32.MaxValue%16u

    let limit8  = 536870911u  //(System.UInt32.MaxValue - 7u)/8u
    let maxDiv8 = 536870911u //System.UInt32.MaxValue/8u
    let maxMod8 = 7u //System.UInt32.MaxValue%8u

    let limit2  = 2147483647u //(System.UInt32.MaxValue - 1u)/2u
    let maxDiv2 = 2147483647u //System.UInt32.MaxValue/2u
    let maxMod2 = 1u //System.UInt32.MaxValue%2u

    let mutable n = 0u
    let mutable c = c0
    let c1 = stream.SkipAndPeek()

    if    (opt &&& (NLO.AllowBinary ||| NLO.AllowOctal ||| NLO.AllowHexadecimal)) = NLO.None
       || c <> '0' || c1 <= '9'
    then
        n <- uint32 c - uint32 '0'
        c <- c1
        while c >= '0' && c <= '9' do
            let nc = uint32 c - uint32 '0'
            if n <= limit10 || (maxMod10 < 9u && n = maxDiv10 && nc <= maxMod10) then
                n <- 10u*n + nc
                c <- stream.SkipAndPeek()
            else
                status <- FatalError
                c <- '!' // break

    else
        let cc1 = uint32 c1 ||| uint32 ' '
        if (opt &&& NLO.AllowHexadecimal) <> NLO.None && cc1 = uint32 'x' then
            c <- stream.SkipAndPeek()
            let mutable nc = uint32 0
            if  (let cc = uint32 c ||| uint32 ' '
                 if c <= '9' then nc <- uint32 c - uint32 '0'; c >= '0'
                 else cc <= uint32 'f' && (nc <- cc - 0x57u; cc >= uint32 'a')) // 0x57u = uint32 'a' - 10u
            then
                n <- uint32 nc
                c <- stream.SkipAndPeek()
                while
                    (let cc = uint32 c ||| uint32 ' '
                     if c <= '9' then nc <- uint32 c - uint32 '0'; c >= '0'
                     else cc <= uint32 'f' && (nc <- cc - 0x57u; cc >= uint32 'a'))
                  do
                    if n <= limit16 || (maxMod16 < 15u && n = maxDiv16 && nc <= maxMod16) then
                        n <- 16u*n + nc
                        c <- stream.SkipAndPeek()
                    else
                        status <- FatalError
                        c <- '!' // break
            else
                status <- Error
                error <- Errors.ExpectedHexadecimalDigit

        elif (opt &&& NLO.AllowOctal) <> NLO.None && cc1 = uint32 'o' then
            c <- stream.SkipAndPeek()
            let mutable nc = uint32 c - uint32 '0'
            if nc = (nc &&& 7u) then
                n <- uint32 nc
                c <- stream.SkipAndPeek()
                nc <- uint32 c - uint32 '0'
                while nc = (nc &&& 7u) do
                    if n <= limit8 || (maxMod8 < 7u && n = maxDiv8 && nc <= maxMod8) then
                        n <- 8u*n + nc
                        c <- stream.SkipAndPeek()
                        nc <- uint32 c - uint32 '0'
                    else
                        status <- FatalError
                        nc <- 11u // break
            else
                status <- Error
                error <- Errors.ExpectedOctalDigit

        elif (opt &&& NLO.AllowBinary) <> NLO.None && cc1 = uint32 'b' then
            c <- stream.SkipAndPeek()
            let mutable nc = uint32 c - uint32 '0'
            if nc = (nc &&& 1u) then
                n <- uint32 nc
                c <- stream.SkipAndPeek()
                nc <- uint32 c - uint32 '0'
                while nc = (nc &&& 1u) do
                    if n <= limit2 || (maxMod2 = 0u && n = maxDiv2 && nc = 0u) then
                        n <- 2u*n + nc
                        c <- stream.SkipAndPeek()
                        nc <- uint32 c - uint32 '0'
                    else
                        status <- FatalError
                        nc <- 11u // break
            else
                status <- Error
                error <- Errors.ExpectedBinaryDigit
        // else c = 0 && not (isDigit c1)
    n

[<MethodImplAttribute(MethodImplOptions.NoInlining)>]
let internal overflowError message =
    if isNotNull message then messageError message // isNotNull prevents fsc from inlining the function
    else NoErrorMessages

let inline internal pint (opt: NumberLiteralOptions) (max: 'uint) (uint64_: 'uint -> uint64) (uint: int -> 'uint) (uint_: uint32 -> 'uint) (uint__: uint64 -> 'uint) (int: 'uint -> 'int) (int_: int -> 'int) (errorInCaseNoLiteralFound: ErrorMessageList) (outOfRangeError: ErrorMessageList) (stream: CharStream<'u>) =
    // we rely on the compiler eliminating inactive branches after inlining

    let minusIsAllowed = (opt &&& NLO.AllowMinusSign) <> NLO.None

    let index = stream.IndexToken
    let stateTag = stream.StateTag
    let mutable c = stream.Peek()

    let mutable plusMinus1  = 1
    let mutable signPresent = false
    if minusIsAllowed && c = '-' then
        plusMinus1 <- -1
        signPresent <- true
        c <- stream.SkipAndPeek()
    elif (opt &&& NLO.AllowPlusSign) <> NLO.None && c = '+' then
        signPresent <- true
        c <- stream.SkipAndPeek()

    let mutable status = Ok
    let mutable error = NoErrorMessages
    let mutable result = Unchecked.defaultof<_>
    if c >= '0' && c <= '9' then
        let n = if uint64_ max <= uint64 System.UInt32.MaxValue then
                    uint_  (parseUInt32 c stream (&status) (&error))
                else
                    uint__ (parseUInt64 c stream (&status) (&error))
        let isUInt32Or64 = uint64_ max = uint64 System.UInt32.MaxValue || uint64_ max = System.UInt64.MaxValue
        if status = Ok && (isUInt32Or64 || (n <= max || (minusIsAllowed && plusMinus1 = -1 && n = max + uint 1))) then
            result <- if minusIsAllowed then int_ plusMinus1 * int n else int n
        elif status <> Error then
            status <- FatalError
            stream.Seek(index)
            stream.StateTag <- stateTag
            error <- outOfRangeError
    else
        status <- Error
        error <- errorInCaseNoLiteralFound
        if signPresent then
            stream.Seek(index)
            stream.StateTag <- stateTag
    Reply(status, result, error)

let pint64 stream = pint NumberLiteralOptions.DefaultInteger (uint64 System.Int64.MaxValue)            uint64 uint64 uint64 uint64 int64 int64 Errors.ExpectedInt64 Errors.NumberOutsideOfInt64Range stream
let pint32 stream = pint NumberLiteralOptions.DefaultInteger (uint32 System.Int32.MaxValue)            uint64 uint32 uint32 uint32 int32 int32 Errors.ExpectedInt32 Errors.NumberOutsideOfInt32Range stream
                                                           // fsc's optimizer seems to have problems with literals of small int types
let pint16 stream = pint NumberLiteralOptions.DefaultInteger ((*uint32 System.Int16.MaxValue*)0x7fffu) uint64 uint32 uint32 uint32 int16 int16 Errors.ExpectedInt16 Errors.NumberOutsideOfInt16Range stream
let pint8  stream = pint NumberLiteralOptions.DefaultInteger ((*uint32 System.SByte.MaxValue*)0x7fu)   uint64 uint32 uint32 uint32 sbyte sbyte Errors.ExpectedInt8  Errors.NumberOutsideOfInt8Range stream

let puint64 stream = pint NumberLiteralOptions.DefaultUnsignedInteger System.UInt64.MaxValue uint64 uint64 uint64 uint64 uint64 uint64 Errors.ExpectedUInt64 Errors.NumberOutsideOfUInt64Range stream
let puint32 stream = pint NumberLiteralOptions.DefaultUnsignedInteger System.UInt32.MaxValue uint64 uint32 uint32 uint32 uint32 uint32 Errors.ExpectedUInt32 Errors.NumberOutsideOfUInt32Range stream
let puint16 stream = pint NumberLiteralOptions.DefaultUnsignedInteger 0xffffu                uint64 uint32 uint32 uint32 uint16 uint16 Errors.ExpectedUInt16 Errors.NumberOutsideOfUInt16Range stream
let puint8  stream = pint NumberLiteralOptions.DefaultUnsignedInteger 0xffu                  uint64 uint32 uint32 uint32 byte   byte   Errors.ExpectedUInt8  Errors.NumberOutsideOfUInt8Range stream



// -------------------
// Conditional parsing
// -------------------

let notFollowedByEof : Parser<unit,'u> =
    fun stream ->
        if not (stream.IsEndOfStream) then Reply(())
        else Reply(Error, Errors.UnexpectedEndOfInput)

let followedByNewline : Parser<unit,'u> =
    fun stream ->
        match stream.Peek() with
        |'\r' | '\n' -> Reply(())
        | _ -> Reply(Error, Errors.ExpectedNewline)

let notFollowedByNewline : Parser<unit,'u> =
    fun stream ->
        match stream.Peek() with
        |'\r' | '\n' -> Reply(Error, Errors.UnexpectedNewline)
        | _ -> Reply(())

let followedByString (str: string) : Parser<unit,'u> =
    checkStringContainsNoNewlineOrEOSChar str "followedByString"
    let error = expectedString str
    if str.Length = 1 then
        let chr = str.[0]
        fun stream ->
            if stream.Match(chr) then Reply(())
            else Reply(Error, error)
    else
        fun stream ->
            if stream.Match(str) then Reply(())
            else Reply(Error, error)

let followedByStringCI str : Parser<unit,'u> =
    checkStringContainsNoNewlineOrEOSChar str "followedByStringCI"
    let error = expectedStringCI str
    if str.Length = 1 then
        let cfChr = Text.FoldCase(str.[0])
        fun stream ->
            if stream.MatchCaseFolded(cfChr) then Reply(())
            else Reply(Error, error)
    else
        let cfStr = foldCase str
        fun stream ->
            if stream.MatchCaseFolded(cfStr) then Reply(())
            else Reply(Error, error)

let notFollowedByString str : Parser<unit,'u> =
    checkStringContainsNoNewlineOrEOSChar str "notFollowedByString"
    let error = unexpectedString str
    if str.Length = 1 then
        let chr = str.[0]
        fun stream ->
            if not (stream.Match(chr)) then Reply(())
            else Reply(Error, error)
    else
        fun stream ->
            if not (stream.Match(str)) then Reply(())
            else Reply(Error, error)

let notFollowedByStringCI str : Parser<unit,'u> =
    checkStringContainsNoNewlineOrEOSChar str "notFollowedByStringCI"
    let error = unexpectedStringCI str
    if str.Length = 1 then
        let cfChr = Text.FoldCase(str.[0])
        fun stream ->
            if not (stream.MatchCaseFolded(cfChr)) then Reply(())
            else Reply(Error, error)
    else
        let cfStr = foldCase str
        fun stream ->
            if not (stream.MatchCaseFolded(cfStr)) then Reply(())
            else Reply(Error, error)


let inline private charDoesSatisfy f c =
    match c with
    | EOS -> Error
    | _ -> if f (if c <> '\r' then c else '\n') then Ok else Error

let inline private charDoesSatisfyNot f c =
    match c with
    | EOS -> Ok
    | _ -> if not (f (if c <> '\r' then c else '\n')) then Ok else Error

let previousCharSatisfies f : Parser<unit,'u> =
    fun stream ->
        let status = charDoesSatisfy f (stream.Peek(-1))
        Reply(status, (), NoErrorMessages)

let previousCharSatisfiesNot f : Parser<unit,'u> =
    fun stream ->
        let status = charDoesSatisfyNot f (stream.Peek(-1))
        Reply(status, (), NoErrorMessages)

let nextCharSatisfies f : Parser<unit,'u> =
    fun stream ->
        let status = charDoesSatisfy f (stream.Peek())
        Reply(status, (), NoErrorMessages)

let nextCharSatisfiesNot f : Parser<unit,'u> =
    fun stream ->
        let status = charDoesSatisfyNot f (stream.Peek())
        Reply(status, (), NoErrorMessages)

let next2CharsSatisfy f : Parser<unit,'u> =
    let optF = OptimizedClosures.FSharpFunc<char, char, bool>.Adapt(f)
    fun stream ->
        let cs = stream.Peek2()
        let status = match cs.Char0, cs.Char1 with
                     | _, EOS
                     | EOS, _ -> Error
                     | '\r', '\n' ->
                         match stream.Peek(2u) with
                         | EOS -> Error
                         | c1 -> if optF.Invoke('\n', if c1 <> '\r' then c1 else '\n')
                                 then Ok else Error
                     | c0, c1 ->
                         if optF.Invoke((if c0 <> '\r' then c0 else '\n'),
                                        (if c1 <> '\r' then c1 else '\n'))
                         then Ok else Error
        Reply(status, (), NoErrorMessages)

let next2CharsSatisfyNot f : Parser<unit,'u> =
    let optF = OptimizedClosures.FSharpFunc<char, char, bool>.Adapt(f)
    fun stream ->
        let cs = stream.Peek2()
        let status = match cs.Char0, cs.Char1 with
                     | _, EOS
                     | EOS, _ -> Ok
                     | '\r', '\n' ->
                         match stream.Peek(2u) with
                         | EOS -> Ok
                         | c1 -> if not (optF.Invoke('\n', if c1 <> '\r' then c1 else '\n'))
                                 then Ok else Error
                     | c0, c1 ->
                         if not (optF.Invoke((if c0 <> '\r' then c0 else '\n'),
                                             (if c1 <> '\r' then c1 else '\n')))
                         then Ok else Error
        Reply(status, (), NoErrorMessages)


<|start_filename|>FParsecCS/ErrorMessage.cs<|end_filename|>
﻿// Copyright (c) <NAME> 2010
// License: Simplified BSD License. See accompanying documentation.

using System;
using System.Diagnostics;
using System.Collections.Generic;

using Microsoft.FSharp.Core;

namespace FParsec {

public enum ErrorMessageType {
    Expected,
    ExpectedString,
    ExpectedCaseInsensitiveString,
    Unexpected,
    UnexpectedString,
    UnexpectedCaseInsensitiveString,
    Message,
    NestedError,
    CompoundError,
    Other
}

[DebuggerDisplay("{GetDebuggerDisplay(),nq}")]
public class ErrorMessage : IEquatable<ErrorMessage> {
    public readonly ErrorMessageType Type;

    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
    internal string String;

    internal ErrorMessage(ErrorMessageType messageType) {
        Type = messageType;
    }

    public class Expected : ErrorMessage {
        public string Label { get { return String; } }
        public Expected(string labelForExpectedInput) : base(ErrorMessageType.Expected) {
            String = labelForExpectedInput;
        }
    }

    public class ExpectedString : ErrorMessage {
        public new string String { get { return String; } }
        public ExpectedString(string expectedString) : base(ErrorMessageType.ExpectedString) {
            base.String = expectedString;
        }
    }

    public class ExpectedCaseInsensitiveString : ErrorMessage {
        public string CaseInsensitiveString { get { return String; } }
        public ExpectedCaseInsensitiveString(string expectedCaseInsensitiveString) : base(ErrorMessageType.ExpectedCaseInsensitiveString) {
            String = expectedCaseInsensitiveString;
        }
    }

    public class Unexpected : ErrorMessage {
        public string Label { get { return String; } }
        public Unexpected(string labelForUnexpectedInput) : base(ErrorMessageType.Unexpected) {
            String = labelForUnexpectedInput;
        }
    }

    public class UnexpectedString : ErrorMessage {
        public new string String { get { return String; } }
        public UnexpectedString(string unexpectedString) : base(ErrorMessageType.UnexpectedString) {
            base.String = unexpectedString;
        }
    }

    public class UnexpectedCaseInsensitiveString : ErrorMessage {
        public string CaseInsensitiveString { get { return String; } }
        public UnexpectedCaseInsensitiveString(string unexpectedCaseInsensitiveString) : base(ErrorMessageType.UnexpectedCaseInsensitiveString) {
            String = unexpectedCaseInsensitiveString;
        }
    }

    public class Message : ErrorMessage {
        public new string String { get { return String; } }
        public Message(string message) : base(ErrorMessageType.Message) {
            base.String = message;
        }
    }

    public class NestedError : ErrorMessage {
        public Position Position { get; private set; }
        public object UserState { get; private set; }
        public ErrorMessageList Messages { get; private set; }

        public NestedError(Position position, object userState, ErrorMessageList messages) : base(ErrorMessageType.NestedError) {
            Position = position;
            UserState = userState;
            Messages = messages;
        }
    }

    public class CompoundError : ErrorMessage {
        public string LabelOfCompound { get { return String; } }

        public Position NestedErrorPosition { get; private set; }
        public object NestedErrorUserState { get; private set; }
        public ErrorMessageList NestedErrorMessages { get; private set; }

        public CompoundError(string labelOfCompound,
                             Position nestedErrorPosition,
                             object nestedErrorUserState,
                             ErrorMessageList nestedErrorMessages) : base(ErrorMessageType.CompoundError)
        {
            String = labelOfCompound;
            NestedErrorPosition = nestedErrorPosition;
            NestedErrorUserState = nestedErrorUserState;
            NestedErrorMessages = nestedErrorMessages;
        }
    }

    public class Other : ErrorMessage {
        public object Data { get; private set; }
        public Other(object data) : base(ErrorMessageType.Other) {
            Data = data;
        }
    }

    public override bool Equals(object obj) { return Equals(obj as ErrorMessage); }

    public bool Equals(ErrorMessage other) {
        return    (object)this == (object)other
               || (   (object)other != null
                   && Type == other.Type
                   && (Type > ErrorMessageType.Message
                       ? EqualsHelper(other)
                       : String == other.String));
    }

    public static bool operator==(ErrorMessage left, ErrorMessage right) {
        return    (object)left == (object)right
               || (   (object)left != null
                   && (object)right != null
                   && left.Type == right.Type
                   && (left.Type > ErrorMessageType.Message
                       ? left.EqualsHelper(right)
                       : left.String == right.String));
    }
    public static bool operator!=(ErrorMessage left, ErrorMessage right) { return !(left == right); }

    private bool EqualsHelper(ErrorMessage other) {
        Debug.Assert(Type == other.Type
                     && Type > ErrorMessageType.Message);
        if (Type == ErrorMessageType.NestedError) {
            var ne1 = (NestedError)this;
            var ne2 = (NestedError)other;
            return    ne1.Position  == ne2.Position
                   && ne1.Messages  == ne2.Messages
                   && LanguagePrimitives.GenericEqualityERComparer.Equals(ne1.UserState, ne2.UserState);
        } else if (Type == ErrorMessageType.CompoundError) {
            if (String != other.String) return false;
            var ce1 = (CompoundError)this;
            var ce2 = (CompoundError)other;
            return    ce1.NestedErrorPosition  == ce2.NestedErrorPosition
                   && ce1.NestedErrorMessages  == ce2.NestedErrorMessages
                   && LanguagePrimitives.GenericEqualityERComparer.Equals(ce1.NestedErrorUserState, ce2.NestedErrorUserState);
        } else { // ErrorMessageType == ErrorMessageType.Other
            Debug.Assert(Type == ErrorMessageType.Other);
            return ((Other)this).Data == ((Other)other).Data;
        }
    }

    public override int GetHashCode() {
        return (int)Type ^ (String == null ? 0 : String.GetHashCode());
    }

    private class ErrorMessageComparer : Comparer<ErrorMessage> {
        public override int Compare(ErrorMessage x, ErrorMessage y) {
            if (x == null || y == null) {
                return x == null && y == null ? 0 : (x == null ? -1 : 1);
            }
            int d = (int)x.Type - (int)y.Type;
            if (d != 0) return d;
            var type = x.Type;
            if (type <= ErrorMessageType.Message) {
                Debug.Assert(type >= 0);
                return String.CompareOrdinal(x.String, y.String);
            } else if (type == ErrorMessageType.NestedError) {
                var ne1 = (NestedError)x;
                var ne2 = (NestedError)y;
                var c = Position.Compare(ne1.Position, ne2.Position);
                if (c != 0) return c;
                var msgs1 = ErrorMessageList.ToSortedArray(ne1.Messages);
                var msgs2 = ErrorMessageList.ToSortedArray(ne2.Messages);
                int n = Math.Min(msgs1.Length, msgs2.Length);
                for (int i = 0; i < n; ++i) {
                    c = Compare(msgs1[i], msgs2[i]);
                    if (c != 0) return c;
                }
                return msgs1.Length - msgs2.Length;
            } else if (type == ErrorMessageType.CompoundError) {
                var c = String.CompareOrdinal(x.String, y.String);
                if (c != 0) return c;
                var ce1 = (CompoundError)x;
                var ce2 = (CompoundError)y;
                c = Position.Compare(ce1.NestedErrorPosition, ce2.NestedErrorPosition);
                if (c != 0) return c;
                var msgs1 = ErrorMessageList.ToSortedArray(ce1.NestedErrorMessages);
                var msgs2 = ErrorMessageList.ToSortedArray(ce2.NestedErrorMessages);
                int n = Math.Min(msgs1.Length, msgs2.Length);
                for (int i = 0; i < n; ++i) {
                    c = Compare(msgs1[i], msgs2[i]);
                    if (c != 0) return c;
                }
                return msgs1.Length - msgs2.Length;
            } else {
                Debug.Assert(type == ErrorMessageType.Other);
                return 0;
            }
        }
	}

    internal static Comparer<ErrorMessage> Comparer = new ErrorMessageComparer();
    internal static ErrorMessage[] EmptyArray = new ErrorMessage[0];

    internal string GetDebuggerDisplay() {
        switch (Type) {
            case ErrorMessageType.Expected:
                return String == null
                       ? "Expected(null)"
                       : Text.DoubleQuote("Expected(", String, ")");
            case ErrorMessageType.ExpectedString:
                return String == null
                       ? "ExpectedString(null)"
                       : Text.DoubleQuote("ExpectedString(", String, ")");
            case ErrorMessageType.ExpectedCaseInsensitiveString:
                return String == null
                       ? "ExpectedCaseInsensitiveString(null)"
                       : Text.DoubleQuote("ExpectedCaseInsensitiveString(", String, ")");
            case ErrorMessageType.Unexpected:
                return String == null
                       ? "Unexpected(null)"
                       : Text.DoubleQuote("Unexpected(", String, ")");
            case ErrorMessageType.UnexpectedString:
                return String == null
                       ? "UnexpectedString(null)"
                       : Text.DoubleQuote("UnexpectedString(", String, ")");
            case ErrorMessageType.UnexpectedCaseInsensitiveString:
                return String == null
                       ? "UnexpectedCaseInsensitiveString(null)"
                       : Text.DoubleQuote("UnexpectedCaseInsensitiveString(", String, ")");
            case ErrorMessageType.Message:
                return String == null
                       ? "Message(null)"
                       : Text.DoubleQuote("Message(", String, ")");
            case ErrorMessageType.NestedError: {
                var ne = (NestedError)this;
                var pos = ne.Position == null ? "null" : ne.Position.ToString();
                var msgs = ErrorMessageList.GetDebuggerDisplay(ne.Messages);
                return "NestedError(" + pos + ", ..., " + msgs +  ")";
            }
            case ErrorMessageType.CompoundError: {
                var ce = (CompoundError)this;
                var label = ce.String == null ? "null" : Text.Escape(ce.String, "", "\"", "\"", "", '"');
                var pos = ce.NestedErrorPosition == null ? "" : ce.NestedErrorPosition.ToString();
                var msgs = ErrorMessageList.GetDebuggerDisplay(ce.NestedErrorMessages);
                return "CompoundError(" + label +  ", " + pos + ", ..., " + msgs +  ")";
            }
            case ErrorMessageType.Other: {
                var oe = (Other)this;
                return oe.Data == null ? "Other(null)" : "Other(" + oe.ToString() + ")";
            }
            default:
                throw new InvalidOperationException();
        }
    }
}


}

<|start_filename|>FParsecCS/FastGenericEqualityERComparer.cs<|end_filename|>
﻿// Copyright (c) <NAME> 2010
// License: Simplified BSD License. See accompanying documentation.

using System;
using System.Collections;
using System.Collections.Generic;

#if PCL
using System.Reflection;
#endif

using Microsoft.FSharp.Core;

namespace FParsec {

internal static class FastGenericEqualityERComparer<T> {
    // if T is a reference type, accessing the field requires a hash table lookup
    public static EqualityComparer<T> Instance = FastGenericEqualityERComparer.Create<T>();

    /// <summary>For reference types it's faster to call Instance.Equals directly
    /// (due to limitations of the inliner of the .NET JIT.)</summary>
    public static bool Equals(T left, T right) {
        return Instance.Equals(left, right);
    }
}

internal static class FastGenericEqualityERComparer {
    public static EqualityComparer<T> Create<T>() {
        var t = typeof(T);
        if (t.IsArray) return new ArrayStructuralEqualityERComparer<T>();
    #if PCL
        var ti = t.GetTypeInfo();
        var ise = typeof(IStructuralEquatable).GetTypeInfo();
    #else
        var ti = t;
        var ise = typeof(IStructuralEquatable);
    #endif
        if (ise.IsAssignableFrom(ti)) {
            var gct = ti.IsValueType ? typeof(StructStructuralEqualityERComparer<>)
                                     : typeof(ClassStructuralEqualityERComparer<>);
            var ct = gct.MakeGenericType(t);
        #if LOW_TRUST
            return (EqualityComparer<T>)Activator.CreateInstance(ct);
        #else
            return (EqualityComparer<T>)System.Runtime.Serialization.FormatterServices.GetUninitializedObject(ct);
        #endif
        }
        return EqualityComparer<T>.Default;
    }

    private class ClassStructuralEqualityERComparer<T> : EqualityComparer<T> where T : class, IStructuralEquatable {
        public override bool Equals(T x, T y) {
            return    (object)x == (object)y
                   || ((object)x != null && x.Equals(y, LanguagePrimitives.GenericEqualityERComparer));
        }

        public override int GetHashCode(T obj) {
            if ((object)obj == null) throw new ArgumentNullException("obj");
            return obj.GetHashCode(LanguagePrimitives.GenericEqualityERComparer);
        }
    }

    private class StructStructuralEqualityERComparer<T> : EqualityComparer<T> where T : struct, IStructuralEquatable {
        public override bool Equals(T x, T y) {
            return x.Equals(y, LanguagePrimitives.GenericEqualityERComparer);
        }

        public override int GetHashCode(T obj) {
            return obj.GetHashCode(LanguagePrimitives.GenericEqualityERComparer);
        }
    }

    /// <summary>Forwards all work to F#'s GenericEqualityERComparer.</summary>
    private class ArrayStructuralEqualityERComparer<T> : EqualityComparer<T> {
        public override bool Equals(T x, T y) {
            return (object)x == (object)y || LanguagePrimitives.GenericEqualityERComparer.Equals(x, y);
        }

        public override int GetHashCode(T obj) {
            if ((object)obj == null) throw new ArgumentNullException("obj");
            return LanguagePrimitives.GenericEqualityERComparer.GetHashCode(obj);
        }
    }
}

}

<|start_filename|>FParsecCS/Properties/AssemblyInfo.cs<|end_filename|>
﻿using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

[assembly: ComVisible(false)]

#if LOW_TRUST && CLR4
    [assembly: System.Security.AllowPartiallyTrustedCallers]
    [assembly: System.Security.SecurityTransparent]
#endif

[assembly: AssemblyTitle      ("FParsecCS.dll")]
[assembly: AssemblyDescription("FParsecCS.dll")]

[assembly: AssemblyProduct      (FParsec.CommonAssemblyInfo.Product)]
[assembly: AssemblyCopyright    (FParsec.CommonAssemblyInfo.Copyright)]
[assembly: AssemblyVersion      (FParsec.CommonAssemblyInfo.Version)]
[assembly: AssemblyFileVersion  (FParsec.CommonAssemblyInfo.FileVersion)]
[assembly: AssemblyConfiguration(FParsec.CommonAssemblyInfo.CSConfiguration)]

[assembly: InternalsVisibleTo (FParsec.CommonAssemblyInfo.FParsecAssemblyName)]
[assembly: InternalsVisibleTo (FParsec.CommonAssemblyInfo.TestAssemblyName)]


namespace FParsec {

internal static partial class CommonAssemblyInfo {
    public const string Product     = "FParsec";
    public const string Copyright   = "Copyright © <NAME> 2007-2015";
    public const string Version     = "1.0.0.0";
    public const string FileVersion = "1.0.2.0";

#if !NUGET // the NuGet package build script sets these attributes
    public const string Configuration =
    #if DEBUG
        #if LOW_TRUST
            "Debug Build (/define:LOW_TRUST)";
        #else
            "Debug Build";
        #endif
    #else
        #if LOW_TRUST
            "Release Build (/define:LOW_TRUST)";
        #else
            "Release Build";
        #endif
    #endif
    public const string CSConfiguration = Configuration;
    public const string FSConfiguration = Configuration;
    public const string FParsecAssemblyName = "FParsec";
    public const string TestAssemblyName = "test_fparsec";
#endif
};

}


<|start_filename|>ParcelTest/VisitorTests.cs<|end_filename|>
﻿using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace ParcelTest
{
    [TestClass]
    public class VisitorTests
    {
        [TestMethod]
        public void ExtractConstantsTest()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            var f = "=4/(3.2*SUM(A$4:A$10,B$4:B$10)-22)";

            var constants = Parcel.constantsFromFormula(f, e.Path, e.WorkbookName, e.WorksheetName, ignore_parse_errors: false);

            Assert.IsTrue(Array.Exists(constants, c => c.Equals(new AST.ReferenceConstant(e, 3.2))));
            Assert.IsTrue(Array.Exists(constants, c => c.Equals(new AST.ReferenceConstant(e, 4))));
            Assert.IsTrue(Array.Exists(constants, c => c.Equals(new AST.ReferenceConstant(e, 22))));
            Assert.IsTrue(constants.Length == 3);
        }
    }
}


<|start_filename|>Parcel/TrustedZoneHandler/TrustedZoneHandler.cpp<|end_filename|>
#define STRICT
#include "stdafx.h"

extern "C" {
	// more or less borrowed from <NAME>'s page on
	// reading, setting, and deleting security identifiers:
	// http://blogs.msdn.com/b/oldnewthing/archive/2013/11/04/10463035.aspx

	extern "C" __declspec(dllexport) void EraseZoneData(wchar_t *);
	extern "C" __declspec(dllexport) DWORD ReadZoneData(wchar_t *);
	extern "C" __declspec(dllexport) void SetZoneData(wchar_t *, DWORD);

	void EraseZoneData(wchar_t *filename)
	{
		// start up COM
		HRESULT hr = CoInitialize(NULL);

		// COM smart ptr to zone ID
		CComPtr<IZoneIdentifier> spzi;
		// create zone ID object
		spzi.CoCreateInstance(CLSID_PersistentZoneIdentifier);
		// remove zone information
		spzi->Remove();
		/// overwrite zone information on file
		CComQIPtr<IPersistFile>(spzi)->Save(filename, TRUE);

		// shutdown COM
		CoUninitialize();
	}

	DWORD ReadZoneData(wchar_t *filename)
	{
		// start up COM
		HRESULT hr = CoInitialize(NULL);

		// COM smart pointer to security mgr interface
		CComPtr<IInternetSecurityManager> spzi;
		spzi.CoCreateInstance(CLSID_InternetSecurityManager);

		// read security DWORD; return URLZONE_INVALID if
		// this call fails
		DWORD dwZone;
		if (SUCCEEDED(spzi->MapUrlToZone(
						    filename,
						    &dwZone,
						    MUTZ_ISFILE | MUTZ_DONT_UNESCAPE))) {
			return dwZone;
		} else {
			return URLZONE_INVALID;
		}

		// shutdown COM
		CoUninitialize();
	}

	void SetZoneData(wchar_t *filename, DWORD urlz)
	{
		// start up COM
		HRESULT hr = CoInitialize(NULL);

		// COM smart ptr to zone ID
		CComPtr<IZoneIdentifier> spzi;
		// create zone ID object
		spzi.CoCreateInstance(CLSID_PersistentZoneIdentifier);
		// set field to user-defined url zone
		spzi->SetId(urlz);
		// overwrite zone information on file
		CComQIPtr<IPersistFile>(spzi)->Save(filename, TRUE);
		
		// shutdown COM
		CoUninitialize();
	}
}

<|start_filename|>ParcelTest/jenkins.c<|end_filename|>
#include <stdio.h>
#include <stdint.h>
#include <string.h>

uint32_t jhash(char *key, size_t len)
{
    uint32_t hash, i;
    for(hash = i = 0; i < len; ++i)
    {
        hash += key[i];
        hash += (hash << 10);
        hash ^= (hash >> 6);
    }
    hash += (hash << 3);
    hash ^= (hash >> 11);
    hash += (hash << 15);
    return hash;
}

int main(int argc, char *argv[]) {
    if (argc != 2) {
      printf("Usage: jenkins <my string>\n");
    }
    printf("%s: %i\n", argv[1], jhash(argv[1], strlen(argv[1])));
    return 0;
}


<|start_filename|>FParsecCS/CharStream.cs<|end_filename|>
﻿// Copyright (c) <NAME> 2007-2012
// License: Simplified BSD License. See accompanying documentation.

#if !LOW_TRUST

using System;
using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Diagnostics;
using System.Reflection;
using System.Runtime.Serialization;
using System.Runtime.InteropServices;
using System.Runtime.CompilerServices;

using Microsoft.FSharp.Core;

using FParsec.Cloning;

namespace FParsec {

/// <summary>An opaque representation of a CharStream index.</summary>
public unsafe struct CharStreamIndexToken {
#if DEBUG
    internal readonly CharStream CharStream;
    private long Index { get { return GetIndex(CharStream); } }
#endif
    internal readonly char* Ptr;
    private readonly int BlockPlus1;
    /// <summary>Returns -1 if the IndexToken was zero-initialized.</summary>
    internal int Block { get { return unchecked(BlockPlus1 - 1); } }

    internal CharStreamIndexToken(
                              #if DEBUG
                                  CharStream charStream,
                              #endif
                                  char* ptr,
                                  int block)
    {
    #if DEBUG
        CharStream = charStream;
    #endif
        Ptr = ptr;
        BlockPlus1 = unchecked(block + 1);
    }

    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private void ThrowInvalidIndexToken() {
        throw new InvalidOperationException("The CharStreamIndexToken is invalid.");
    }

    public long GetIndex(CharStream charStreamFromWhichIndexTokenWasRetrieved) {
        int block = Block;
        if (block < 0) ThrowInvalidIndexToken(); // tests for a zero-initialized IndexToken
    #if DEBUG
        Debug.Assert(CharStream == charStreamFromWhichIndexTokenWasRetrieved);
    #endif
        return charStreamFromWhichIndexTokenWasRetrieved.GetIndex(Ptr, block);
    }
}

public struct TwoChars : IEquatable<TwoChars> {
    private uint Chars;

    internal TwoChars(uint chars) {
        Chars = chars;
    }
    public TwoChars(char char0, char char1) {
        Chars = ((uint)char1 << 16) | (uint)char0;
    }

    public char Char0 { get { return unchecked((char)Chars); } }
    public char Char1 { get { return (char)(Chars >> 16); } }

    public override bool Equals(object obj) { return (obj is TwoChars) && Chars == ((TwoChars) obj).Chars; }
    public bool Equals(TwoChars other) { return Chars == other.Chars; }
    public override int GetHashCode()  { return unchecked((int)Chars); }
    public static bool operator==(TwoChars left, TwoChars right) { return left.Chars == right.Chars; }
    public static bool operator!=(TwoChars left, TwoChars right) { return left.Chars != right.Chars; }
}

/// <summary>Provides read‐access to a sequence of UTF‐16 chars.</summary>
public unsafe class CharStream : IDisposable {

    // In order to facilitate efficient backtracking we divide the stream into overlapping
    // blocks with equal number of chars. The blocks are overlapping, so that
    // backtracking over short distances at a block boundary doesn't trigger a reread of the
    // previous block.
    //
    //              Block 0
    //
    //    -----------------|--------  Block 1
    //                       Overlap
    //                      --------|--------|--------  Block 2
    //                                        Overlap
    //                                        --------|--------|--------
    //                                                                  (...)
    //  a '-' symbolizes a char, a '|' a block boundary.
    //
    //
    // In general there's no fixed relationship between the number of input bytes and the
    // number of input chars. Worse, the encoding can be stateful, which makes it necessary
    // to persist the decoder state over block boundaries. If we later want to
    // be able to reread a certain block, we therefore need to keep record of various
    // bits of information describing the state of the input stream at the beginning of a block:

    private class BlockInfo {
        /// <summary>the byte stream index of the first char in the block after the OverhangCharsAtBlockBegin</summary>
        public long ByteIndex;
        /// <summary>the value of the CharStream's ByteBufferIndex before the block is read</summary>
        public int ByteBufferIndex;

        /// <summary>the number of bytes in the stream from ByteIndex to the first char after the OverhangCharsAfterOverlap</summary>
        public int NumberOfBytesInOverlap;

        /// <summary>the last char in the overlap with the previous block (used for integrity checking)</summary>
        public char LastCharInOverlap;

        /// <summary>chars at the block begin that were already read together with chars of the last block before the overlap</summary>
        public string OverhangCharsAtBlockBegin;
        /// <summary>chars after the overlap with the previous block that were already read together with the overlap chars</summary>
        public string OverhangCharsAfterOverlap;

        // Unfortunately the Decoder API has no explicit methods for managing the state,
        // which forces us to use the comparatively inefficient serialization API
        // (via FParsec.Cloning) for this purpose.
        // The absence of explicit state management or at least a cloning method in the
        // Decoder interface is almost as puzzling to me as the absence of such methods
        // in System.Random.

        public CloneImage DecoderImageAtBlockBegin;
        public CloneImage DecoderImageAfterOverlap;

        public BlockInfo(long byteIndex, int byteBufferIndex,
                         int nBytesInOverlapCount, char lastCharInOverlap,
                         string overhangCharsAtBlockBegin, CloneImage decoderImageAtBlockBegin,
                         string overhangCharsAfterOverlap, CloneImage decoderImageAfterOverlap)
        {
            ByteIndex = byteIndex;
            ByteBufferIndex = byteBufferIndex;
            NumberOfBytesInOverlap = nBytesInOverlapCount;
            LastCharInOverlap = lastCharInOverlap;
            OverhangCharsAtBlockBegin = overhangCharsAtBlockBegin;
            OverhangCharsAfterOverlap = overhangCharsAfterOverlap;
            DecoderImageAtBlockBegin = decoderImageAtBlockBegin;
            DecoderImageAfterOverlap = decoderImageAfterOverlap;
        }
    }

    private const int DefaultBlockSize = 3*(1 << 16); // 3*2^16 = 200k
    private const int DefaultByteBufferLength = (1 << 12);
    private static int MinimumByteBufferLength = 128; // must be larger than longest detectable preamble (we can only guess here)
    private const char EOS = '\uFFFF';

    public const char EndOfStreamChar = EOS;

    /// <summary>Points to the current char in Buffer,
    /// or is null if the end of the stream has been reached.</summary>
    internal char* Ptr;
    /// <summary>Equals Ptr == null ? null : BufferBegin.</summary>
    internal char* PtrBegin;
    /// <summary>Equals Ptr == null ? null : BufferEnd.</summary>
    internal char* PtrEnd;

    /// <summary>Begin of the used part of the char buffer. Is constant. Is null if the CharStream is empty.</summary>
    internal char* BufferBegin;
    /// <summary>End of the used part of the char buffer. Varies for a multi-block stream. Is null if the CharStream is empty.</summary>
    internal char* BufferEnd;

    /// <summary>The block currently loaded in the buffer.</summary>
    internal int Block;

    /// <summary>Any CharStream method or property setter increments this value when it changes the CharStream state.
    /// Backtracking to an old state also restores the old value of the StateTag.</summary>
    public
#if SMALL_STATETAG
           int
#else
           long
#endif
                 StateTag;

    internal long IndexOfFirstCharInBlock;

    internal long _IndexOfFirstChar;
    /// <summary>The index of the first char in the stream.</summary>
    public long IndexOfFirstChar { get { return _IndexOfFirstChar; } }

    internal long _Line;
    /// <summary>The line number for the next char. (The line count starts with 1.)</summary>
    public long Line { get { return _Line; } }
    public void SetLine_WithoutCheckAndWithoutIncrementingTheStateTag(long line) {
        _Line = line;
    }

    internal long _LineBegin;
    /// <summary>The stream index of the first char of the line that also contains the next char.</summary>
    public long LineBegin { get { return _LineBegin; } }
    public void SetLineBegin_WithoutCheckAndWithoutIncrementingTheStateTag(long lineBegin) {
        _LineBegin = lineBegin;
    }

    /// <summary>The UTF‐16 column number of the next char, i.e. Index ‐ LineBegin  + 1.</summary>
    public long Column { get { return Index - LineBegin + 1; } }

    internal string _Name;
    public string Name {
        get { return _Name; }
        set { _Name = value; ++StateTag; }
    }

    /// <summary>The Encoding that is used for decoding the underlying byte stream, or
    /// System.Text.UnicodeEncoding in case the stream was directly constructed
    /// from a string or char buffer.</summary>
    public Encoding Encoding { get; private set; }

    // If the CharStream is constructed from a binary stream, we use a managed string as the char
    // buffer. This allows us to apply regular expressions directly to the input.
    // In the case of multi-block CharStreams we thus have to mutate the buffer string through pointers.
    // This is safe as long as we use a newly constructed string and we don't pass a reference
    // to the internal buffer string to the "outside world". (The one instance where we have to pass
    // a reference to the buffer string is regex matching. See the docs for Match(regex) for more info.)
    //
    // Apart from Match(regex) we access the internal buffer only through a pinned pointer.
    // This way we avoid the overhead of redundant bounds checking and can support strings, char arrays
    // and unmanaged char buffers through the same interface.
    //
    // Pinning a string or char array makes life more difficult for the GC. However, as long as
    // the buffer is only short-lived or large enough to be allocated on the large object heap,
    // there shouldn't be a problem. Furthermore, the buffer strings for CharStreams constructed
    // from a binary stream are allocated through the StringBuffer interface and hence always live
    // on the large object heap. Thus, the only scenario to really worry about (and which the
    // documentation explicitly warns about) is when a large number of small CharStreams
    // are constructed directly from strings or char arrays and are used for an extended period of time.

    /// <summary>The string holding the char buffer, or null if the buffer is not part of a .NET string.</summary>
    internal string BufferString;
    /// <summary>A pointer to the beginning of BufferString, or null if BufferString is null.</summary>
    internal char* BufferStringPointer;

    /// <summary>Holds the GCHandle for CharStreams directly constructed from strings or char arrays.</summary>
    private GCHandle BufferHandle;
    /// <summary>Holds the StringBuffer for CharStreams constructed from a binary stream.</summary>
    private StringBuffer StringBuffer;

#if DEBUG
    internal FSharpRef<int> SubstreamCount = new FSharpRef<int>(0);
    internal FSharpRef<int> ParentSubstreamCount = null;
#endif

    private MultiBlockData BlockData;
    internal bool IsSingleBlockStream { get { return BlockData == null; } }

    /// <summary>Contains the data and methods needed in case the input byte stream
    /// is large enough to span multiple blocks of the CharStream.</summary>
    private partial class MultiBlockData {
        public CharStream CharStream;

        public long IndexOfLastCharPlus1;

        /// <summary>The index of the last block of the stream, or Int32.MaxValue if the end of stream has not yet been detected.</summary>
        public int LastBlock;

        public Stream Stream;
        // we keep a separate record of the Stream.Position, so that we don't need to require Stream.CanSeek
        public long StreamPosition;
        // we use StreamLength to avoid calling Read() again on a non-seekable stream after it returned 0 once (see ticket #23)
        public long StreamLength;
        public bool LeaveOpen;

        public int MaxCharCountForOneByte;
        public Decoder Decoder;
        public bool DecoderIsSerializable;

        public int BlockSize;
        public int BlockOverlap;
        /// <summary>BufferBegin + BlockSize - minRegexSpace</summary>
        public char* RegexSpaceThreshold;

        /// <summary>The byte stream index of the first unused byte in the ByteBuffer.</summary>
        public long ByteIndex { get { return StreamPosition - (ByteBufferCount - ByteBufferIndex); } }

        public List<BlockInfo> Blocks;

        public byte[] ByteBuffer;
        public int ByteBufferIndex;
        public int ByteBufferCount;
    }

    public long IndexOfLastCharPlus1 { get {
        return BlockData != null ? BlockData.IndexOfLastCharPlus1
                                 : IndexOfFirstChar + Buffer.PositiveDistance(BufferBegin, BufferEnd);
    } }

    public int BlockOverlap { get {
        return BlockData == null ? 0 : BlockData.BlockOverlap;
    } }

    public int MinRegexSpace {
        get {
            return BlockData == null
                   ? 0
                   : (int)Buffer.PositiveDistance(BlockData.RegexSpaceThreshold,
                                                  BufferBegin + BlockData.BlockSize);
        }
        set {
            if (BlockData != null) {
                if (value < 0 || value > BlockData.BlockOverlap) throw new ArgumentOutOfRangeException("value", "The MinRegexSpace value must be non-negative and not greater than the BlockOverlap.");
                BlockData.RegexSpaceThreshold = BufferBegin + BlockData.BlockSize - value;
            }
        }
    }

    public bool IsBeginOfStream { get { return Ptr == BufferBegin && Block == 0; } }
    public bool IsEndOfStream { get { return Ptr == null; } }


    public long Index {
    #if CLR45
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
    #endif
        get {
            if (Ptr != null) {
                Debug.Assert(BufferBegin <= Ptr && Ptr < BufferEnd);
                if (sizeof(System.IntPtr) != 8) // the JIT removes the inactive branch
                    return Buffer.PositiveDistance(PtrBegin, Ptr) + IndexOfFirstCharInBlock;
                else
                    return Buffer.PositiveDistance64(PtrBegin, Ptr) + IndexOfFirstCharInBlock;
            }
            Debug.Assert(BlockData == null || BlockData.IndexOfLastCharPlus1 != Int64.MaxValue);
            return IndexOfLastCharPlus1;
        }
    }

    internal long GetIndex(char* ptr, int block) {
        if (ptr != null) {
            if (block == Block) {
                Debug.Assert(BufferBegin <= ptr && ptr < BufferEnd);
                if (sizeof(System.IntPtr) != 8)
                    return Buffer.PositiveDistance(BufferBegin, ptr) + IndexOfFirstCharInBlock;
                else
                    return Buffer.PositiveDistance64(BufferBegin, ptr) + IndexOfFirstCharInBlock;
            } else {
                Debug.Assert(BlockData != null && BufferBegin <= ptr && ptr < BufferBegin + BlockData.BlockSize);
                int blockSizeMinusOverlap = BlockData.BlockSize - BlockData.BlockOverlap;
                long indexOfBlockBegin = IndexOfFirstChar + Math.BigMul(block, blockSizeMinusOverlap);
                if (sizeof(System.IntPtr) != 8)
                    return Buffer.PositiveDistance(BufferBegin, ptr) + indexOfBlockBegin;
                else
                    return Buffer.PositiveDistance64(BufferBegin, ptr) + indexOfBlockBegin;
            }
        }
        Debug.Assert(BlockData == null || BlockData.IndexOfLastCharPlus1 != Int64.MaxValue);
        return IndexOfLastCharPlus1;
    }

    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
    public Position Position { get {
        long index = Index;
        return new Position(_Name, index, Line, index - LineBegin + 1);
    } }

    // we don't have a public constructor that only takes a string to avoid potential confusion with a filepath constructor
    internal CharStream(string chars) {
        Debug.Assert(chars != null);
        BufferString = chars;
        BufferHandle = GCHandle.Alloc(chars, GCHandleType.Pinned);
        char* bufferBegin = (char*)BufferHandle.AddrOfPinnedObject();
        BufferStringPointer = bufferBegin;
        CharConstructorContinue(bufferBegin, chars.Length);
    }

    public CharStream(string chars, int index, int length) : this(chars, index, length, 0) {}
    public CharStream(string chars, int index, int length, long streamIndexOffset) {
        if (chars == null) throw new ArgumentNullException("chars");
        if (index < 0) throw new ArgumentOutOfRangeException("index", "index is negative.");
        if (length < 0 || length > chars.Length - index) throw new ArgumentOutOfRangeException("length", "index or length is out of range.");
        if (streamIndexOffset < 0 || streamIndexOffset >= (1L << 60)) throw new ArgumentOutOfRangeException("streamIndexOffset", "streamIndexOffset must be non-negative and less than 2^60.");
        IndexOfFirstCharInBlock = streamIndexOffset;
        _IndexOfFirstChar = streamIndexOffset;
        _LineBegin = streamIndexOffset;

        BufferString = chars;
        BufferHandle = GCHandle.Alloc(chars, GCHandleType.Pinned);
        char* pBufferString = (char*)BufferHandle.AddrOfPinnedObject();
        BufferStringPointer = pBufferString;
        CharConstructorContinue(pBufferString + index, length);
    }

    public CharStream(char[] chars, int index, int length) : this(chars, index, length, 0) { }
    public CharStream(char[] chars, int index, int length, long streamIndexOffset) {
        if (chars == null) throw new ArgumentNullException("chars");
        if (index < 0) throw new ArgumentOutOfRangeException("index", "index is negative.");
        if (length < 0 || length > chars.Length - index) throw new ArgumentOutOfRangeException("length", "index or length is out of range.");
        if (streamIndexOffset < 0 || streamIndexOffset >= (1L << 60)) throw new ArgumentOutOfRangeException("streamIndexOffset", "streamIndexOffset must be non-negative and less than 2^60.");
        IndexOfFirstCharInBlock = streamIndexOffset;
        _IndexOfFirstChar = streamIndexOffset;
        _LineBegin = streamIndexOffset;

        BufferHandle = GCHandle.Alloc(chars, GCHandleType.Pinned);
        char* bufferBegin = (char*)BufferHandle.AddrOfPinnedObject() + index;
        if (bufferBegin < unchecked(bufferBegin + length + 1)) { // a pedantic check ...
            CharConstructorContinue(bufferBegin, length);
        } else {
            // ... for a purely theoretic case
            BufferHandle.Free();
            throw new ArgumentOutOfRangeException("length", "The char array may not be allocated directly below the end of the address space.");
        }

    }

    public CharStream(char* chars, int length) : this(chars, length, 0) {}
    public CharStream(char* chars, int length, long streamIndexOffset) {
        if (chars == null) throw new ArgumentNullException("chars");
        if (length < 0) throw new ArgumentOutOfRangeException("length", "length is negative.");
        if (chars >= unchecked(chars + length + 1)) // chars + length + 1 must not overflow (the + 1 is needed for some methods below)
            throw new ArgumentOutOfRangeException("length", "length is too large.");
        if (streamIndexOffset < 0 || streamIndexOffset >= (1L << 60)) throw new ArgumentOutOfRangeException("streamIndexOffset", "streamIndexOffset must be non-negative and less than 2^60.");
        IndexOfFirstCharInBlock = streamIndexOffset;
        _IndexOfFirstChar = streamIndexOffset;
        _LineBegin = streamIndexOffset;
        CharConstructorContinue(chars, length);
    }

    private void CharConstructorContinue(char* bufferBegin, int length) {
        Debug.Assert((bufferBegin != null || length == 0) && length >= 0
                     && bufferBegin < unchecked(bufferBegin + length + 1)); // the + 1 is needed for some methods below

        if (length != 0) {
            BufferBegin = bufferBegin;
            BufferEnd = bufferBegin + length;
            Ptr = bufferBegin;
            PtrBegin = bufferBegin;
            PtrEnd = BufferEnd;
        }
        _Line = 1;
        Encoding = Encoding.Unicode;
    }

    internal CharStream(string chars, char* pChars, char* begin, int length) {
        Debug.Assert((chars == null ? pChars == null
                                    : pChars <= begin && length >= 0 && (int)Buffer.PositiveDistance(pChars, begin) <= chars.Length - length)
                     && (begin == null ? length == 0
                                       : length >= 0 && begin < unchecked(begin + length + 1)));

        BufferString = chars;
        BufferStringPointer = pChars;
        if (length != 0) {
            BufferBegin = begin;
            BufferEnd = begin + length;
            Ptr = begin;
            PtrBegin = begin;
            PtrEnd = BufferEnd;
        }
        _Line = 1;
        Encoding = Encoding.Unicode;
    }

    public CharStream(string path, Encoding encoding)
           : this(path, encoding, true,
                  DefaultBlockSize, DefaultBlockSize/3, DefaultByteBufferLength) { }

    public CharStream(string path, Encoding encoding, bool detectEncodingFromByteOrderMarks)
           : this(path, encoding, detectEncodingFromByteOrderMarks,
                  DefaultBlockSize, DefaultBlockSize/3, DefaultByteBufferLength) { }

    public CharStream(string path, Encoding encoding, bool detectEncodingFromByteOrderMarks,
                      int blockSize, int blockOverlap, int byteBufferLength)
    {
        if (encoding == null) throw new ArgumentNullException("encoding");
        var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.SequentialScan);
        try {
           StreamConstructorContinue(stream, false, encoding, detectEncodingFromByteOrderMarks,
                                     blockSize, blockOverlap, byteBufferLength);
           _Name = path;
        } catch {
            stream.Dispose();
            throw;
        }
    }

    public CharStream(Stream stream, Encoding encoding)
           : this(stream,
                  false, encoding, true,
                  DefaultBlockSize, DefaultBlockSize/3, DefaultByteBufferLength) { }

    public CharStream(Stream stream, bool leaveOpen, Encoding encoding)
           : this(stream,
                  leaveOpen, encoding, true,
                  DefaultBlockSize, DefaultBlockSize/3, DefaultByteBufferLength) { }

    public CharStream(Stream stream, bool leaveOpen, Encoding encoding, bool detectEncodingFromByteOrderMarks)
           : this(stream,
                  leaveOpen, encoding, detectEncodingFromByteOrderMarks,
                  DefaultBlockSize, DefaultBlockSize/3, DefaultByteBufferLength) { }

    public CharStream(Stream stream, bool leaveOpen,
                      Encoding encoding, bool detectEncodingFromByteOrderMarks,
                      int blockSize, int blockOverlap, int byteBufferLength)
    {
        if (stream == null) throw new ArgumentNullException("stream");
        if (!stream.CanRead) throw new ArgumentException("stream is not readable");
        if (encoding == null) throw new ArgumentNullException("encoding");
        StreamConstructorContinue(stream, leaveOpen, encoding, detectEncodingFromByteOrderMarks,
                                  blockSize, blockOverlap, byteBufferLength);
    }

    /// <summary>we modify this flag via reflection in the unit test</summary>
    private static bool DoNotRoundUpBlockSizeToSimplifyTesting = false;

    private void StreamConstructorContinue(Stream stream, bool leaveOpen,
                                           Encoding encoding, bool detectEncodingFromByteOrderMarks,
                                           int blockSize, int blockOverlap, int byteBufferLength)
    {
        if (byteBufferLength < MinimumByteBufferLength) byteBufferLength = MinimumByteBufferLength;

        int remainingBytesCount = -1;
        long streamPosition;
        long streamLength;
        if (stream.CanSeek) {
            streamPosition = stream.Position;
            streamLength = stream.Length;
            long remainingBytesCount64 = streamLength - streamPosition;
            if (remainingBytesCount64 <= Int32.MaxValue) {
                remainingBytesCount = (int)remainingBytesCount64;
                if (remainingBytesCount < byteBufferLength) byteBufferLength = remainingBytesCount;
            }
        } else {
            streamPosition = 0;
            streamLength = Int64.MaxValue;
        }

        byte[] byteBuffer = new byte[byteBufferLength];
        int byteBufferCount = 0;
        do {
            int n = stream.Read(byteBuffer, byteBufferCount, byteBufferLength - byteBufferCount);
            if (n == 0) {
                remainingBytesCount = byteBufferCount;
                Debug.Assert(!stream.CanSeek || streamPosition + byteBufferCount == streamLength);
                streamLength = streamPosition + byteBufferCount;
                break;
            }
            byteBufferCount += n;
        } while (byteBufferCount < MinimumByteBufferLength);
        streamPosition += byteBufferCount;

        int preambleLength = Text.DetectPreamble(byteBuffer, byteBufferCount, ref encoding, detectEncodingFromByteOrderMarks);
        remainingBytesCount -= preambleLength;

        _Line = 1;
        Encoding = encoding;

        // we allow such small block sizes only to simplify testing
        if (blockSize < 8) blockSize = DefaultBlockSize;

        bool allCharsFitIntoOneBlock = false;
        if (remainingBytesCount >= 0 && remainingBytesCount/4 <= blockSize) {
            if (remainingBytesCount != 0) {
                try {
                    int maxCharCount = Encoding.GetMaxCharCount(remainingBytesCount); // may throw ArgumentOutOfRangeException
                    if (blockSize >= maxCharCount) {
                        allCharsFitIntoOneBlock = true;
                        blockSize = maxCharCount;
                    }
                } catch (ArgumentOutOfRangeException) { }
            } else {
                allCharsFitIntoOneBlock = true;
                blockSize = 0;
            }
        }
        var buffer = StringBuffer.Create(blockSize);
        Debug.Assert(buffer.Length >= blockSize && (blockSize > 0 || buffer.StringPointer == null));
        StringBuffer = buffer;
        BufferString = buffer.String;
        BufferStringPointer = buffer.StringPointer;
        char* bufferBegin = buffer.StringPointer + buffer.Index;
        try {
            Decoder decoder = encoding.GetDecoder();
            if (allCharsFitIntoOneBlock) {
                int bufferCount = preambleLength == byteBufferCount
                                  ? 0
                                  : Text.ReadAllRemainingCharsFromStream(bufferBegin, buffer.Length, byteBuffer, preambleLength, byteBufferCount, stream, streamPosition, decoder, streamPosition == streamLength);
                if (!leaveOpen) stream.Close();
                if (bufferCount != 0) {
                    BufferBegin = bufferBegin;
                    Ptr = bufferBegin;
                    PtrBegin = bufferBegin;
                    BufferEnd = bufferBegin + bufferCount;
                    PtrEnd = BufferEnd;
                }
                Block = 0;
            } else {
                if (!DoNotRoundUpBlockSizeToSimplifyTesting) blockSize = buffer.Length;
                BufferBegin = bufferBegin;
                BufferEnd = bufferBegin;
                var d = new MultiBlockData();
                BlockData = d;
                d.CharStream = this;
                d.Stream = stream;
                d.StreamPosition = streamPosition;
                d.StreamLength = streamLength;
                d.LeaveOpen = leaveOpen;
                d.Decoder = decoder;
                d.DecoderIsSerializable = decoder.GetType().IsSerializable;
                d.ByteBuffer = byteBuffer;
                d.ByteBufferIndex = preambleLength;
                d.ByteBufferCount = byteBufferCount;
                d.MaxCharCountForOneByte = Math.Max(1, Encoding.GetMaxCharCount(1));
                if (d.MaxCharCountForOneByte > 1024) // an arbitrary limit low enough that a char array with this size can be allocated on the stack
                    throw new ArgumentException("The CharStream class does not support Encodings with GetMaxCharCount(1) > 1024.");
                if (blockSize < 3*d.MaxCharCountForOneByte) blockSize = 3*d.MaxCharCountForOneByte;
                // MaxCharCountForOneByte == the maximum number of overhang chars
                if(    Math.Min(blockOverlap, blockSize - 2*blockOverlap) < d.MaxCharCountForOneByte
                    || blockOverlap >= blockSize/2) blockOverlap = blockSize/3;
                d.BlockSize = blockSize;
                d.BlockOverlap = blockOverlap;
                d.RegexSpaceThreshold = bufferBegin + (blockSize - 2*blockOverlap/3);
                d.IndexOfLastCharPlus1 = Int64.MaxValue;
                Block = -2; // special value recognized by ReadBlock
                d.LastBlock = Int32.MaxValue;
                d.Blocks = new List<BlockInfo>();
                // the first block has no overlap with a previous block
                d.Blocks.Add(new BlockInfo(preambleLength, preambleLength, 0, EOS, null, null, null, null));
                d.ReadBlock(0);
                if (d.LastBlock == 0) {
                    if (!d.LeaveOpen) d.Stream.Close();
                    BlockData = null;
                }
            }
        } catch {
            buffer.Dispose();
            throw;
        }
    }

    public void Dispose() {
    #if DEBUG
        lock (SubstreamCount) {
            if (SubstreamCount.Value != 0)
                throw new InvalidOperationException("A CharStream must not be disposed before all of its Substreams have been disposed.");
        }
        if (ParentSubstreamCount != null) {
            lock (ParentSubstreamCount) --ParentSubstreamCount.Value;
        }
    #endif
        if (BufferHandle.IsAllocated) BufferHandle.Free();
        if (StringBuffer != null) StringBuffer.Dispose();
        if (BlockData != null && !BlockData.LeaveOpen) BlockData.Stream.Close();
        Ptr = null;
        PtrBegin = null;
        PtrEnd = null;
        BufferBegin = null;
        BufferEnd = null;
    }

    [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Justification="The CharStream is manually disposed.")]
    public static T ParseString<T,TUserState>(string chars, int index, int length,
                                              FSharpFunc<CharStream<TUserState>,T> parser,
                                              TUserState userState,
                                              string streamName)
    {
        if (index < 0) throw new ArgumentOutOfRangeException("index", "index is negative.");
        if (length < 0 || length > chars.Length - index) throw new ArgumentOutOfRangeException("length", "length is out of range.");
        fixed (char* pChars = chars) {
            var stream = new CharStream<TUserState>(chars, pChars, pChars + index, length);
            stream.UserState = userState;
            stream._Name = streamName;
            try {
                return parser.Invoke(stream);
            } finally {
            #if DEBUG
                stream.Dispose();
            #else
                // manually dispose stream
                stream.Ptr = null;
                stream.PtrBegin = null;
                stream.PtrEnd = null;
                stream.BufferBegin = null;
                stream.BufferEnd = null;
            #endif
            }
        }
    }

    private partial class MultiBlockData {
        /// <summary>Refills the ByteBuffer if no unused byte is remaining.
        /// Returns the number of unused bytes in the (refilled) ByteBuffer.</summary>
        private int FillByteBuffer() {
            int n = ByteBufferCount - ByteBufferIndex;
            if (n > 0) return n;
            return ClearAndRefillByteBuffer(0);
        }

        /// <summary>Refills the ByteBuffer starting at the given index. If the underlying byte
        /// stream contains enough bytes, the ByteBuffer is filled up to the ByteBuffer.Length.
        /// Returns the number of bytes available for consumption in the refilled ByteBuffer.</summary>
        private int ClearAndRefillByteBuffer(int byteBufferIndex) {
            Debug.Assert(byteBufferIndex >= 0 && byteBufferIndex <= ByteBuffer.Length);
            // Stream.Read is not guaranteed to use all the provided output buffer, so we need
            // to call it in a loop when we want to rely on the buffer being fully filled
            // (unless we reach the end of the stream). Knowing that the buffer always gets
            // completely filled allows us to calculate the buffer utilization after skipping
            // a certain number of input bytes. For most streams there will be only one loop
            // iteration anyway (or two at the end of the stream).
            int i = byteBufferIndex;
            int m = ByteBuffer.Length - byteBufferIndex;
            while (m != 0 && StreamPosition != StreamLength) { // we check the StreamPosition to avoid calling Read after it returned 0 at the end of the stream (see ticket #23)
                int c = Stream.Read(ByteBuffer, i, m);
                if (c != 0) {
                    i += c;
                    m -= c;
                    StreamPosition += c;
                } else {
                    Debug.Assert(!Stream.CanSeek || StreamPosition == StreamLength);
                    StreamLength = StreamPosition;
                    break;
                }
            }
            int n = i - byteBufferIndex;
            ByteBufferIndex = byteBufferIndex;
            ByteBufferCount = byteBufferIndex + n;
            return n;
        }

        /// <summary>Reads up to the given maximum number of chars into the given buffer.
        /// If more than the maximum number of chars have to be read from the stream in order to
        /// fill the buffer (due to	the way the Decoder API works), the overhang chars are
        /// returned through the output parameter.
        /// Returns a pointer to one char after the last char read.</summary>
        private char* ReadCharsFromStream(char* buffer, int maxCount, out string overhangChars) {
            Debug.Assert(maxCount >= 0);
            fixed (byte* byteBuffer = ByteBuffer) {
                overhangChars = null;
                try {
                    while (maxCount >= MaxCharCountForOneByte) {// if maxCount < MaxCharCountForOneByte, Convert could throw
                        int nBytesInByteBuffer = FillByteBuffer();
                        bool flush = nBytesInByteBuffer == 0;
                        int bytesUsed, charsUsed; bool completed = false;
                        Decoder.Convert(byteBuffer + ByteBufferIndex, nBytesInByteBuffer,
                                        buffer, maxCount, flush,
                                        out bytesUsed, out charsUsed, out completed);
                        ByteBufferIndex += bytesUsed; // GetChars consumed bytesUsed bytes from the byte buffer
                        buffer += charsUsed;
                        maxCount -= charsUsed;
                        if (flush && completed) return buffer;
                    }
                    if (maxCount == 0) return buffer;

                    char* cs = stackalloc char[MaxCharCountForOneByte];
                    for (;;) {
                        int nBytesInByteBuffer = FillByteBuffer();
                        bool flush = nBytesInByteBuffer == 0;
                        int bytesUsed, charsUsed; bool completed;
                        Decoder.Convert(byteBuffer + ByteBufferIndex, nBytesInByteBuffer,
                                        cs, MaxCharCountForOneByte, flush,
                                        out bytesUsed, out charsUsed, out completed);
                        ByteBufferIndex += bytesUsed;
                        if (charsUsed > 0) {
                            int i = 0;
                            do {
                                *buffer = cs[i];
                                ++buffer; ++i;
                                if (--maxCount == 0) {
                                    if (i < charsUsed) overhangChars = new string(cs, i, charsUsed - i);
                                    return buffer;
                                }
                            } while (i < charsUsed);
                        }
                        if (flush && completed) return buffer;
                    }
                } catch (DecoderFallbackException e) {
                    e.Data.Add("Stream.Position", ByteIndex + e.Index);
                    throw;
                }
            }
        }

        /// <summary> Reads a block of chars (which must be different from the current block)
        /// into the BufferString. If the current CharStream block is block - 1, this method
        /// seeks the CharStream to the first char after the overlap of the two blocks.
        /// Otherwise it seeks the CharStream to the first char in the block. It returns the
        /// CharStream.Ptr value at the new position (which can be null).</summary>
        internal char* ReadBlock(int block) {
            int prevBlock = CharStream.Block;
            if (block == prevBlock) throw new InvalidOperationException();
            if (!DecoderIsSerializable && block > 0) {
                if (prevBlock > block)
                    throw new NotSupportedException("The CharStream does not support seeking backwards over ranges longer than the block overlap because the Encoding's Decoder is not serializable. The decoder has the type: " + Decoder.GetType().FullName);
                while (prevBlock + 1 < block) ReadBlock(++prevBlock);
            }

            BlockInfo bi = Blocks[block]; // will throw if block is out of range
            int blockSizeMinusOverlap = BlockSize - BlockOverlap;
            long charIndex = Math.BigMul(block, blockSizeMinusOverlap);
            char* bufferBegin = CharStream.BufferBegin;
            char* begin, buffer;
            int nCharsToRead;

            // fill [0 ... BlockOverlap-1] if block > 0
            if (prevBlock == block - 1) {
                Buffer.Copy((byte*)bufferBegin, (byte*)(bufferBegin + blockSizeMinusOverlap),
                                  BlockOverlap*sizeof(char));
                Debug.Assert(bufferBegin[BlockOverlap - 1] == bi.LastCharInOverlap);
                begin = buffer = bufferBegin + BlockOverlap;
            } else if (prevBlock >= 0) {
                Stream.Seek(bi.ByteIndex, SeekOrigin.Begin); // will throw if Stream can't seek
                // now that there was no exception, we can change the state...
                StreamPosition = bi.ByteIndex;
                ClearAndRefillByteBuffer(bi.ByteBufferIndex);
                if (block != 0)
                    Decoder = (Decoder)bi.DecoderImageAtBlockBegin.CreateClone();
                else
                    Decoder.Reset();
                if (prevBlock == block + 1) {
                    // move the overlap into [BlockSize - BlockOverlap, BlockSize - 1] before it gets overwritten
                    Buffer.Copy((byte*)(bufferBegin + blockSizeMinusOverlap), (byte*)bufferBegin,
                                      BlockOverlap*sizeof(char));
                }
                begin = buffer = bufferBegin;
                if (block > 0) {
                    nCharsToRead = BlockOverlap;
                    if (bi.OverhangCharsAtBlockBegin != null) {
                        nCharsToRead -= bi.OverhangCharsAtBlockBegin.Length;
                        for (int i = 0; i < bi.OverhangCharsAtBlockBegin.Length; ++i)
                            *(buffer++) = bi.OverhangCharsAtBlockBegin[i];
                    }
                    string overhangCharsAfterOverlap;
                    buffer = ReadCharsFromStream(buffer, nCharsToRead, out overhangCharsAfterOverlap);
                    if (   buffer != bufferBegin + BlockOverlap
                        || ByteIndex != bi.ByteIndex + bi.NumberOfBytesInOverlap
                        || *(buffer - 1) != bi.LastCharInOverlap
                        || overhangCharsAfterOverlap != bi.OverhangCharsAfterOverlap)
                        throw new IOException("CharStream: stream integrity error");
                }
            } else { // ReadBlock was called from the constructor
                if (block != 0) throw new InvalidOperationException();
                begin = buffer = bufferBegin;
            }

            // fill [0            ... BlockSize-BlockOverlap-1] if block == 0
            // and  [BlockOverlap ... BlockSize-BlockOverlap-1] otherwise
            if (block == 0) {
                nCharsToRead = blockSizeMinusOverlap;
            } else {
                nCharsToRead = blockSizeMinusOverlap - BlockOverlap;
                if (bi.OverhangCharsAfterOverlap != null) {
                    nCharsToRead -= bi.OverhangCharsAfterOverlap.Length;
                    for (int i = 0; i < bi.OverhangCharsAfterOverlap.Length; ++i)
                        *(buffer++) = bi.OverhangCharsAfterOverlap[i];
                }
            }
            string overhangCharsAtNextBlockBegin;
            buffer = ReadCharsFromStream(buffer, nCharsToRead, out overhangCharsAtNextBlockBegin);

            long byteIndexAtNextBlockBegin = ByteIndex;
            int byteBufferIndexAtNextBlockBegin = ByteBufferIndex;

            // fill [BlockSize-BlockOverlap ... BlockSize-1]
            if (block == Blocks.Count - 1) { // next block hasn't yet been read
                Cloner cloner = null;
                CloneImage decoderImageAtNextBlockBegin = null;
                if (DecoderIsSerializable) {
                    cloner = Cloner.Create(Decoder.GetType());
                    decoderImageAtNextBlockBegin = cloner.CaptureImage(Decoder);
                }
                nCharsToRead = BlockOverlap;
                if (overhangCharsAtNextBlockBegin != null) {
                    nCharsToRead -= overhangCharsAtNextBlockBegin.Length;
                    for (int i = 0; i < overhangCharsAtNextBlockBegin.Length; ++i)
                        *(buffer++) = overhangCharsAtNextBlockBegin[i];
                }
                string overhangCharsAfterOverlapWithNextBlock;
                buffer = ReadCharsFromStream(buffer, nCharsToRead, out overhangCharsAfterOverlapWithNextBlock);
                if (LastBlock == Int32.MaxValue) { // last block hasn't yet been detected
                    if (buffer == bufferBegin + BlockSize) {
                        var decoderImageAfterOverlapWithNextBlock =
                            !DecoderIsSerializable ? null : cloner.CaptureImage(Decoder);
                        int nBytesInOverlapWithNextBlock = (int)(ByteIndex - byteIndexAtNextBlockBegin);
                        Blocks.Add(new BlockInfo(byteIndexAtNextBlockBegin, byteBufferIndexAtNextBlockBegin,
                                                 nBytesInOverlapWithNextBlock, *(buffer - 1),
                                                 overhangCharsAtNextBlockBegin, decoderImageAtNextBlockBegin,
                                                 overhangCharsAfterOverlapWithNextBlock, decoderImageAfterOverlapWithNextBlock));
                    } else { // we reached the end of the stream
                        LastBlock = block;
                        IndexOfLastCharPlus1 = CharStream.IndexOfFirstChar + charIndex + (buffer - bufferBegin);
                    }
                } else if (IndexOfLastCharPlus1 != CharStream.IndexOfFirstChar + charIndex + (buffer - bufferBegin)) {
                    throw new IOException("CharStream: stream integrity error");
                }
            } else {
                BlockInfo nbi = Blocks[block + 1];
                if (buffer != bufferBegin + blockSizeMinusOverlap
                    || byteIndexAtNextBlockBegin != nbi.ByteIndex
                    || byteBufferIndexAtNextBlockBegin != nbi.ByteBufferIndex
                    || overhangCharsAtNextBlockBegin != nbi.OverhangCharsAtBlockBegin)
                    throw new IOException("CharStream: stream integrity error");

                if (prevBlock != block + 1 || (block == 0 && !DecoderIsSerializable)) { // jumping back to block 0 is supported even if the decoder is not serializable
                    nCharsToRead = BlockOverlap;
                    if (overhangCharsAtNextBlockBegin != null) {
                        nCharsToRead -= overhangCharsAtNextBlockBegin.Length;
                        for (int i = 0; i < overhangCharsAtNextBlockBegin.Length; ++i)
                            *(buffer++) = overhangCharsAtNextBlockBegin[i];
                    }
                    string overhangCharsAfterOverlapWithNextBlock;
                    buffer = ReadCharsFromStream(buffer, nCharsToRead, out overhangCharsAfterOverlapWithNextBlock);
                    int nBytesInOverlapWithNextBlock = (int)(ByteIndex - byteIndexAtNextBlockBegin);
                    if (buffer != bufferBegin + BlockSize
                        || nBytesInOverlapWithNextBlock != nbi.NumberOfBytesInOverlap
                        || *(buffer - 1) != nbi.LastCharInOverlap
                        || overhangCharsAfterOverlapWithNextBlock != nbi.OverhangCharsAfterOverlap)
                        throw new IOException("CharStream: stream integrity error");
                } else {
                    Debug.Assert(bufferBegin[BlockSize - 1] == nbi.LastCharInOverlap);
                    buffer += BlockOverlap; // we already copied the chars at the beginning of this function
                    int off = nbi.NumberOfBytesInOverlap - (ByteBufferCount - ByteBufferIndex);
                    if (off > 0) {
                        // we wouldn't have gotten here if the Stream didn't support seeking
                        Stream.Seek(off, SeekOrigin.Current);
                        StreamPosition += off;
                        ClearAndRefillByteBuffer(off%ByteBuffer.Length);
                    } else {
                        ByteBufferIndex += nbi.NumberOfBytesInOverlap;
                    }
                    Decoder = (Decoder)nbi.DecoderImageAfterOverlap.CreateClone();
                }
            }

            CharStream.Block = block;
            //CharStream.CharIndex = charIndex;
            CharStream.IndexOfFirstCharInBlock = CharStream.IndexOfFirstChar + charIndex;
            CharStream.BufferEnd = buffer;
            if (begin != buffer) {
                CharStream.Ptr = begin;
                CharStream.PtrEnd = buffer;
                CharStream.PtrBegin = CharStream.BufferBegin;
                return begin;
            } else {
                CharStream.Ptr = null;
                CharStream.PtrEnd = null;
                CharStream.PtrBegin = null;
                return null;
            }
        }
    } // class MultiBlockData


    /// <summary>Returns an iterator pointing to the given index in the stream,
    /// or to the end of the stream if the indexed position lies beyond the last char in the stream.</summary>
    /// <exception cref="ArgumentOutOfRangeException">The index is negative or less than the BeginIndex.</exception>
    /// <exception cref="NotSupportedException">Accessing the char with the given index requires seeking in the underlying byte stream, but the byte stream does not support seeking or the Encoding's Decoder is not serializable.</exception>
    /// <exception cref="IOException">An I/O error occured.</exception>
    /// <exception cref="ArgumentException">The input stream contains invalid bytes and the encoding was constructed with the throwOnInvalidBytes option.</exception>
    /// <exception cref="DecoderFallbackException">The input stream contains invalid bytes for which the decoder fallback threw this exception.</exception>
    /// <exception cref="OutOfMemoryException">Can not allocate enough memory for the internal data structure.</exception>
    /// <exception cref="ObjectDisposedException">Method is called after the stream was disposed.</exception>
    public void Seek(long index) {
        ++StateTag;
        // The following comparison is safe in case of an overflow since
        // 0 <= IndexOfFirstCharInBlock < 2^60 + 2^31 * 2^31 and BufferEnd - BufferBegin < 2^31,
        // where 2^31 is an upper bound for both the number of blocks and the number of chars in a block.
        long off = unchecked(index - IndexOfFirstCharInBlock);
        if (0 <= off && off < Buffer.PositiveDistance(BufferBegin, BufferEnd)) {
            Ptr = BufferBegin + (uint)off;
            PtrBegin = BufferBegin;
            PtrEnd   = BufferEnd;
            return;
        }
        if (index < IndexOfFirstChar) {
            --StateTag;
            throw (new ArgumentOutOfRangeException("index", "The index is negative or less than the IndexOfFirstChar."));
        }
        if (BlockData == null || index >= BlockData.IndexOfLastCharPlus1) {
            Ptr = null;
            PtrBegin = null;
            PtrEnd = null;
            return;
        }
        // we never get here for streams with only one block
        index -= IndexOfFirstChar;
        int blockSizeMinusOverlap = BlockData.BlockSize - BlockData.BlockOverlap;
        long idx_;
        long block_ = Math.DivRem(index, blockSizeMinusOverlap, out idx_);
        int block = block_ > Int32.MaxValue ? Int32.MaxValue : (int)block_;
        int idx = (int)idx_;
        Seek(block, idx);
    }

    private void Seek(int block, int indexInBlock) {
        Debug.Assert(block >= 0 && indexInBlock >= 0 && BlockData != null);
        if (block > Block) {
            if (indexInBlock < BlockData.BlockOverlap) {
                --block;
                indexInBlock += BlockData.BlockSize - BlockData.BlockOverlap;
            }
        } else if (block < Block) {
            int blockSizeMinusOverlap = BlockData.BlockSize - BlockData.BlockOverlap;
            if (indexInBlock >= blockSizeMinusOverlap) {
                ++block;
                indexInBlock -= blockSizeMinusOverlap;
            }
        }
        if (block == Block) {
            Debug.Assert(indexInBlock < Buffer.PositiveDistance(BufferBegin, BufferEnd));
            PtrBegin = BufferBegin;
            PtrEnd = BufferEnd;
        } else {
            int last = BlockData.Blocks.Count - 1;
            if (block >= last) {
                BlockData.ReadBlock(last);
                while (Block < block && Block != BlockData.LastBlock)
                    BlockData.ReadBlock(Block + 1);
                if (block != Block || indexInBlock >= Buffer.PositiveDistance(PtrBegin, PtrEnd)) {
                    Ptr = null;
                    PtrBegin = null;
                    PtrEnd = null;
                    return;
                }
            } else {
                BlockData.ReadBlock(block);
                Debug.Assert(indexInBlock < Buffer.PositiveDistance(PtrBegin, PtrEnd));
            }
        }
        Ptr = BufferBegin + indexInBlock;
    }

    internal void Seek(char* ptr, int block) {
        if (ptr != null) {
            if (block != Block) {
                Debug.Assert(BlockData != null && ptr >= BufferBegin && ptr < BufferBegin + BlockData.BlockSize);
                int indexInBlock = (int)Buffer.PositiveDistance(BufferBegin, ptr);
                Seek(block, indexInBlock);
            } else {
                Debug.Assert(ptr >= BufferBegin && ptr < BufferEnd);
                Ptr = ptr;
                PtrBegin = BufferBegin;
                PtrEnd = BufferEnd;
            }
        } else {
            Ptr = null;
            PtrBegin = null;
            PtrEnd = null;
        }
    }

    private void SeekToFirstCharAfterLastCharOfCurrentBlock() {
        if (Ptr != null) {
            if (BlockData != null && Block != BlockData.LastBlock) BlockData.ReadBlock(Block + 1);
            else {
                Ptr = null;
                PtrBegin = null;
                PtrEnd = null;
            }
        }
    }

    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
    public CharStreamIndexToken IndexToken { get {
        return new CharStreamIndexToken(
               #if DEBUG
                   this,
               #endif
                   Ptr,
                   Block
               );
    } }

    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private void ThrowInvalidIndexToken() {
        throw new ArgumentException("The CharStreamIndexToken is invalid.");
    }

    public void Seek(CharStreamIndexToken indexToken) {
        int block = indexToken.Block;
        if (block < 0) ThrowInvalidIndexToken(); // tests for zero-initialized IndexTokens
    #if DEBUG
        Debug.Assert(this == indexToken.CharStream);
    #endif
        if (Ptr != null && indexToken.Ptr != null && block == Block) {
            Ptr = indexToken.Ptr;
            Debug.Assert(Ptr >= BufferBegin && Ptr < BufferEnd);
        } else {
            Seek(indexToken.Ptr, block);
        }
        ++StateTag;
    }

    // Below we split many methods into a default method containing the code
    // for the most frequently used branch and a "...Continue" method containing
    // the code for the remaining branches. This allows the JIT to produce
    // faster code for the main branch and in a few cases even to inline it.

    public string ReadFrom(CharStreamIndexToken indexOfFirstChar) {
        int block = indexOfFirstChar.Block;
        if (block < 0) ThrowInvalidIndexToken(); // tests for zero-initialized IndexTokens
    #if DEBUG
        Debug.Assert(this == indexOfFirstChar.CharStream);
    #endif
        return ReadFrom(indexOfFirstChar.Ptr, block);
    }

    internal string ReadFrom(char* ptr, int block) {
        if (ptr != null && ptr < Ptr && block == Block) {
            Debug.Assert(BufferBegin <= ptr && Ptr < BufferEnd);
            return new string(ptr, 0, (int)Buffer.PositiveDistance(ptr, Ptr));
        }
        return ReadFromContinue(ptr, block);
    }
    private string ReadFromContinue(char* ptr, int block) {
        ulong index1 = (ulong)GetIndex(ptr, block);
        ulong index2 = (ulong)Index;

        if (index1 < index2) {
            ulong length_ = index2 - index1;
            // The maximum theoretical string size is Int32.MaxValue,
            // though on .NET it is actually less than 2^30, since the maximum
            // object size is limited to Int32.MaxValue, even on 64-bit systems.
            if (length_ > Int32.MaxValue) {
                // OutOfMemoryException is the exception the .NET string constructor throws
                // if the the string length is larger than the maximum string length,
                // even if enough memory would be available.
                throw new OutOfMemoryException();
            }
            int length = (int)length_;
            var stateTag = StateTag;
            Seek(ptr, block);
            var str = Read((int)length);
            StateTag = stateTag;
            return str;
        } else if (index1 > index2) throw new ArgumentException("The current position of the stream must not lie before the position corresponding to the given CharStreamIndexToken/CharStreamState.");
        return "";
    }

    public void RegisterNewline() {
        var index = Index;
        Debug.Assert(index != _LineBegin);
        _LineBegin = index;
        ++_Line;
        ++StateTag;
    }

    private void RegisterNewlines(char* lineBegin, uint lineOffset) {
        Debug.Assert(BufferBegin <= lineBegin && lineBegin <= BufferEnd && lineOffset > 0);
        _Line += lineOffset;
        long newLineBegin = Buffer.PositiveDistance(BufferBegin, lineBegin) + IndexOfFirstCharInBlock;
        Debug.Assert(newLineBegin != _LineBegin);
        _LineBegin = newLineBegin;
        ++StateTag;
    }

    public void RegisterNewlines(int lineOffset, int newColumnMinus1) {
        Debug.Assert(lineOffset != 0 && newColumnMinus1 >= 0);
        _Line += lineOffset;
        Debug.Assert(_Line > 0);
        var newLineBegin = Index - newColumnMinus1;
        Debug.Assert(newLineBegin != _LineBegin);
        _LineBegin = Index - newColumnMinus1;
        ++StateTag;
    }

    public void RegisterNewlines(long lineOffset, long newColumnMinus1) {
        Debug.Assert(lineOffset != 0 && newColumnMinus1 >= 0);
        _Line += lineOffset;
        Debug.Assert(_Line > 0);
        var newLineBegin = Index - newColumnMinus1;
        Debug.Assert(newLineBegin != _LineBegin);
        _LineBegin = Index - newColumnMinus1;
        ++StateTag;
    }

    public char Peek() {
        char* ptr = Ptr;
        if (ptr != null) return *ptr;
        return EOS;
    }

#if CLR45
    [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
    public void Skip() {
        char* ptr1 = Ptr + 1;
        if (ptr1 < PtrEnd) {
            Ptr = ptr1;
            ++StateTag;
            return;
        }
        SkipContinue();
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private void SkipContinue() { SkipContinue(1u); }

#if CLR45
    [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
    public char Read() {
        char* ptr = Ptr;
        char* ptr1 = ptr + 1;
        if (ptr1 < PtrEnd) {
            char c = *ptr;
            Ptr = ptr1;
            ++StateTag;
            return c;
        }
        return ReadContinue();
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private char ReadContinue() {
        var c = Peek();
        Skip();
        return c;
    }

#if CLR45
    [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
    public char SkipAndPeek() {
        char* ptr = Ptr + 1;
        if (ptr < PtrEnd) {
            Ptr = ptr;
            ++StateTag;
            return *ptr;
        }
        return SkipAndPeekContinue();
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private char SkipAndPeekContinue() { return SkipAndPeekContinue(1u); }

    private static readonly bool IsLittleEndian = BitConverter.IsLittleEndian; // improves inlining and dead code elimination, at least with the .NET JIT

#if CLR45
    [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
    public TwoChars Peek2() {
        char* ptr = Ptr;
        if (ptr + 1 < PtrEnd) {
            #if UNALIGNED_READS
                if (IsLittleEndian) {
                    return new TwoChars(*((uint*)ptr));
                } else {
                    return new TwoChars(ptr[0], ptr[1]);
                }
            #else
                return new TwoChars(ptr[0], ptr[1]);
            #endif
        }
        return Peek2Continue();
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private TwoChars Peek2Continue() {
        return new TwoChars(Peek(), Peek(1u));
    }

#if CLR45
    [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
    public char Peek(uint utf16Offset) {
        if (utf16Offset < Buffer.PositiveDistance(Ptr, PtrEnd))
            return Ptr[utf16Offset];
        return PeekContinue(utf16Offset);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private char PeekContinue(uint utf16Offset) {
        if (Ptr == null || BlockData == null || Block == BlockData.LastBlock) return EOS;
        char* ptr = Ptr;
        int block = Block;
        var stateTag = StateTag;
        Seek(Index + utf16Offset);
        char c = Peek();
        Seek(ptr, block); // backtrack
        StateTag = stateTag;
        return c;
    }

#if CLR45
    [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
    public void Skip(uint utf16Offset) {
        if (utf16Offset < Buffer.PositiveDistance(Ptr, PtrEnd)) {
            Ptr += utf16Offset;
            ++StateTag;
            return;
        }
        SkipContinue(utf16Offset);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private void SkipContinue(uint utf16Offset) {
        if (Ptr == null || utf16Offset == 0) return;
        if (BlockData == null || Block == BlockData.LastBlock) {
            Ptr = null;
            PtrBegin = null;
            PtrEnd = null;
            ++StateTag;
            return;
        }
        Seek(Index + utf16Offset);
    }

#if CLR45
    [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
    public char SkipAndPeek(uint utf16Offset) {
        if (utf16Offset < Buffer.PositiveDistance(Ptr, PtrEnd)) {
            char* ptr = Ptr + utf16Offset;
            Ptr = ptr;
            ++StateTag;
            return *ptr;
        }
        return SkipAndPeekContinue(utf16Offset);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private char SkipAndPeekContinue(uint utf16Offset) {
        SkipContinue(utf16Offset);
        return Peek();
    }

    public char Peek(int utf16Offset) { // don't force inlining, because the .NET JIT doesn't optimize after inlining
        if (utf16Offset >= 0
            ? utf16Offset < Buffer.PositiveDistance(Ptr, PtrEnd)
            : unchecked((uint)-utf16Offset) <= Buffer.PositiveDistance(PtrBegin, Ptr))
        {
            return Ptr[utf16Offset];
        }
        return PeekContinue(utf16Offset);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private char PeekContinue(int utf16Offset) {
        if (utf16Offset >= 0) return PeekContinue((uint)utf16Offset);
        var newIndex = Index + utf16Offset;
        if (newIndex >= _IndexOfFirstChar) {
            char* ptr = Ptr;
            int block = Block;
            var stateTag = StateTag;
            Seek(Index + utf16Offset);
            char c = Peek();
            Seek(ptr, block);
            StateTag = stateTag;
            return c;
        }
        return EOS;
    }

    public void Skip(int utf16Offset) {
        if (utf16Offset >= 0
            ? utf16Offset < Buffer.PositiveDistance(Ptr, PtrEnd)
            : unchecked((uint)-utf16Offset) <= Buffer.PositiveDistance(PtrBegin, Ptr))
        {
            Ptr = unchecked(Ptr + utf16Offset); // see https://connect.microsoft.com/VisualStudio/feedback/details/522944
            ++StateTag;
            return;
        }
        SkipContinue(utf16Offset);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private void SkipContinue(int utf16Offset) {
        if (utf16Offset >= 0) {
            SkipContinue((uint)utf16Offset);
            return;
        }
        Seek(Index + utf16Offset);
    }

    public void Skip(long utf16Offset) {
        if (utf16Offset >= 0
            ? utf16Offset < Buffer.PositiveDistance(Ptr, PtrEnd)
            : unchecked((ulong)-utf16Offset) <= Buffer.PositiveDistance(PtrBegin, Ptr))
        {
            Ptr = unchecked(Ptr + utf16Offset); // see https://connect.microsoft.com/VisualStudio/feedback/details/522944
            ++StateTag;
            return;
        }
        SkipContinue(utf16Offset);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private void SkipContinue(long utf16Offset) {
        long index = Index;
        Seek(utf16Offset > Int64.MaxValue - index ? Int64.MaxValue : index + utf16Offset);
    }

    public char SkipAndPeek(int utf16Offset) {
        if (utf16Offset >= 0
            ? utf16Offset < Buffer.PositiveDistance(Ptr, PtrEnd)
            : unchecked((uint)-utf16Offset) <= Buffer.PositiveDistance(PtrBegin, Ptr))
        {
            char* ptr = unchecked(Ptr + utf16Offset); // see https://connect.microsoft.com/VisualStudio/feedback/details/522944
            Ptr = ptr;
            ++StateTag;
            return *ptr;
        }
        return SkipAndPeekContinue(utf16Offset);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private char SkipAndPeekContinue(int utf16Offset) {
        if (utf16Offset >= 0) {
            SkipContinue((uint)utf16Offset);
            return Peek();
        }
        var newIndex = Index + utf16Offset;
        if (newIndex >= IndexOfFirstChar) {
            Seek(Index + utf16Offset);
            return Peek();
        } else {
            Seek(_IndexOfFirstChar);
            return EOS;
        }
    }

    public string PeekString(int length) {
        if (unchecked((uint)length) <= Buffer.PositiveDistance(Ptr, PtrEnd))
            return new String(Ptr, 0, length);
        return PeekStringContinue(length);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private string PeekStringContinue(int length) {
        return ReadContinue(length, true);
    }

    public string Read(int length) {
        char* ptr = Ptr;
        if (unchecked((uint)length) < Buffer.PositiveDistance(ptr, PtrEnd)) {
            Ptr += length;
            ++StateTag;
            return new String(ptr, 0, length);
        }
        return ReadContinue(length);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private string ReadContinue(int length) {
        return ReadContinue(length, false);
    }

    private string ReadContinue(int length, bool backtrack) {
        if (length < 0) throw new ArgumentOutOfRangeException("length", "length is negative.");
        if (length == 0 || Ptr == null) return "";
        if (BlockData == null) {
            int maxLength = (int)Buffer.PositiveDistance(Ptr, PtrEnd);
            if (length > maxLength)
                length = maxLength;
        } else {
            long maxLength = BlockData.IndexOfLastCharPlus1 - Index;
            if (length > maxLength)
                length = (int)maxLength;
        }
        string str = new String('\u0000', length);
        fixed (char* pStr = str) {
            int cc = ReadContinue(pStr, length, backtrack);
            if (cc == length) return str;
            return new String(pStr, 0, cc);
        }
    }

    public int PeekString(char[] buffer, int bufferIndex, int length) {
        return Read(buffer, bufferIndex, length, true);
    }
    public int Read(char[] buffer, int bufferIndex, int length) {
        return Read(buffer, bufferIndex, length, false);
    }
    private int Read(char[] buffer, int bufferIndex, int length, bool backtrack) {
        if (bufferIndex < 0)
            throw new ArgumentOutOfRangeException("bufferIndex", "bufferIndex is negative.");
        if (length > buffer.Length - bufferIndex) // throws if buffer is null
            throw new ArgumentOutOfRangeException("length", "bufferIndex or length is out of range.");
        // We must exit early for length == 0, because pining an empty array
        // would invoke implementation-defined behaviour.
        if (length <= 0) {
            if (length == 0) return 0;
            throw new ArgumentOutOfRangeException("length", "length is negative.");
        }
        fixed (char* pBuffer = buffer)
        return Read(pBuffer + bufferIndex, length, backtrack);
    }

    public int PeekString(char* buffer, int length) {
        return Read(buffer, length, true);
    }
    public int Read(char* buffer, int length) {
        return Read(buffer, length, false);
    }
    private int Read(char* buffer, int length, bool backtrack) {
        if (unchecked((uint)length) < Buffer.PositiveDistance(Ptr, PtrEnd)) {
            char* ptr = Ptr;
            int len = length;
        #if UNALIGNED_READS
            if ((unchecked((int)buffer) & 2) != 0 && len != 0) { // align buffer pointer
                *buffer = *ptr;
                ++buffer; ++ptr; --len;
            }
            len -= 8;
            while (len >= 0) {
                ((int*)buffer)[0] = ((int*)ptr)[0];
                ((int*)buffer)[1] = ((int*)ptr)[1];
                ((int*)buffer)[2] = ((int*)ptr)[2];
                ((int*)buffer)[3] = ((int*)ptr)[3];
                buffer += 8; ptr += 8; len -= 8;
            }
            if ((len & 4) != 0) {
                ((int*)buffer)[0] = ((int*)ptr)[0];
                ((int*)buffer)[1] = ((int*)ptr)[1];
                buffer += 4; ptr += 4;
            }
            if ((len & 2) != 0) {
                ((int*)buffer)[0] = ((int*)ptr)[0];
                buffer += 2; ptr += 2;
            }
        #else
            len -= 2;
            while (len >= 0) {
                buffer[0] = ptr[0];
                buffer[1] = ptr[1];
                buffer += 2; ptr += 2; len -= 2;
            }
        #endif
            if ((len & 1) != 0) {
                *buffer = *ptr;
                ++ptr;
            }
            if (!backtrack) {
                Ptr = ptr;
                ++StateTag;
            }
            return length;
        }
        return ReadContinue(buffer, length, backtrack);
    }
    private int ReadContinue(char* buffer, int length, bool backtrack) {
        if (length < 0)
            throw new ArgumentOutOfRangeException("length", "length is negative.");

        if (length == 0 || Ptr == null) return 0;

        int oldLength = length;
        int oldBlock = Block;
        char* oldPtr = Ptr;
        char* ptr = Ptr;

        do {
            int len = Math.Min((int)Buffer.PositiveDistance(Ptr, PtrEnd), length);
            Debug.Assert(length > 0 && len > 0);
            length -= len;
        #if UNALIGNED_READS
            if ((unchecked((int)buffer) & 2) != 0) { // align buffer pointer
                *buffer = *ptr;
                ++buffer; ++ptr; --len;
            }
            len -= 8;
            while (len >= 0) {
                ((int*)buffer)[0] = ((int*)ptr)[0];
                ((int*)buffer)[1] = ((int*)ptr)[1];
                ((int*)buffer)[2] = ((int*)ptr)[2];
                ((int*)buffer)[3] = ((int*)ptr)[3];
                buffer += 8; ptr += 8; len -= 8;
            }
            if ((len & 4) != 0) {
                ((int*)buffer)[0] = ((int*)ptr)[0];
                ((int*)buffer)[1] = ((int*)ptr)[1];
                buffer += 4; ptr += 4;
            }
            if ((len & 2) != 0) {
                ((int*)buffer)[0] = ((int*)ptr)[0];
                buffer += 2; ptr += 2;
            }
        #else
            len -= 2;
            while (len >= 0) {
                buffer[0] = ptr[0];
                buffer[1] = ptr[1];
                buffer += 2; ptr += 2; len -= 2;
            }
        #endif
            if ((len & 1) != 0) {
                *buffer = *ptr;
                ++buffer; ++ptr;
            }
        } while (length != 0
                 && BlockData != null
                 && Block != BlockData.LastBlock
                 && (ptr = BlockData.ReadBlock(Block + 1)) != null);
        if (!backtrack) {
            ++StateTag;
            if (ptr != PtrEnd) Ptr = ptr;
            else SeekToFirstCharAfterLastCharOfCurrentBlock();
        } else {
            if (Block != oldBlock) Seek(oldPtr, oldBlock);
        }
        return oldLength - length;
    }

    public bool Match(char ch) {
        char* ptr = Ptr;
        return ptr != null && ch == *ptr;
    }

    public bool MatchCaseFolded(char caseFoldedChar) {
        char* ptr = Ptr;
        return ptr != null && caseFoldedChar == CaseFoldTable.FoldedChars[*ptr];
    }

    public bool Skip(char ch) {
        char* ptr1 = Ptr + 1;
        if (ptr1 < PtrEnd && ch == *Ptr) {
            Ptr = ptr1;
            ++StateTag;
            return true;
        }
        return SkipContinue(ch);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private bool SkipContinue(char ch) {
        if (Match(ch)) {
            Skip();
            return true;
        }
        return false;
    }

    public bool SkipCaseFolded(char caseFoldedChar) {
        char* ptr1 = Ptr + 1;
        if (ptr1 < PtrEnd && caseFoldedChar == CaseFoldTable.FoldedChars[*Ptr]) {
            Ptr = ptr1;
            ++StateTag;
            return true;
        }
        return SkipCaseFoldedContinue(caseFoldedChar);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private bool SkipCaseFoldedContinue(char caseFoldedChar) {
        if (MatchCaseFolded(caseFoldedChar)) {
            Skip();
            return true;
        }
        return false;
    }

#if CLR45
    [MethodImpl(MethodImplOptions.AggressiveInlining)]
#endif
    public bool Skip(TwoChars twoChars) {
        char* ptr2 = Ptr + 2;
        if (ptr2 < PtrEnd) {
        #if UNALIGNED_READS
            if (IsLittleEndian) {
                if (new TwoChars(*((uint*)Ptr)) == twoChars) {
                    Ptr = ptr2;
                    ++StateTag;
                    return true;
                }
            } else {
                if (twoChars.Char0 == Ptr[0] && twoChars.Char1 == Ptr[1]) {
                    Ptr = ptr2;
                    ++StateTag;
                    return true;
                }
            }
        #else
            if (twoChars.Char0 == Ptr[0] && twoChars.Char1 == Ptr[1]) {
                Ptr = ptr2;
                ++StateTag;
                return true;
            }
        #endif
            return false;
        }
        return SkipContinue(twoChars);
    }
    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private bool SkipContinue(TwoChars twoChars) {
        char* cs = stackalloc char[2];
        cs[0] = twoChars.Char0;
        cs[1] = twoChars.Char1;
        return SkipContinue(cs, 2, false);
    }

    public bool Match(string chars) {
        if (chars.Length <= Buffer.PositiveDistance(Ptr, PtrEnd)) {
            for (int i = 0; i < chars.Length; ++i) {
                if (Ptr[i] != chars[i]) goto ReturnFalse;
            }
            return true;
        ReturnFalse:
            return false;
        }
        return SkipContinue(chars, true);
    }

    public bool Skip(string chars) {
        if (chars.Length < Buffer.PositiveDistance(Ptr, PtrEnd)) {
            for (int i = 0; i < chars.Length; ++i) {
                if (Ptr[i] != chars[i]) goto ReturnFalse;
            }
            Ptr += chars.Length;
            ++StateTag;
            return true;
        ReturnFalse:
            return false;
        }
        return SkipContinue(chars, false);
    }

    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private bool SkipContinue(string chars, bool backtrackEvenIfCharsMatch) {
        fixed (char* pChars = chars)
        return SkipContinue(pChars, chars.Length, backtrackEvenIfCharsMatch);
    }

    public bool MatchCaseFolded(string caseFoldedChars) {
        if (caseFoldedChars.Length <= Buffer.PositiveDistance(Ptr, PtrEnd)) {
            for (int i = 0; i < caseFoldedChars.Length; ++i) {
                if (CaseFoldTable.FoldedChars[Ptr[i]] != caseFoldedChars[i]) goto ReturnFalse;
            }
            return true;
        ReturnFalse:
            return false;
        }
        return SkipCaseFoldedContinue(caseFoldedChars, true);
    }

    public bool SkipCaseFolded(string caseFoldedChars) {
        if (caseFoldedChars.Length < Buffer.PositiveDistance(Ptr, PtrEnd)) {
            for (int i = 0; i < caseFoldedChars.Length; ++i) {
                if (CaseFoldTable.FoldedChars[Ptr[i]] != caseFoldedChars[i]) goto ReturnFalse;
            }
            Ptr += caseFoldedChars.Length;
            ++StateTag;
            return true;
        ReturnFalse:
            return false;
        }
        return SkipCaseFoldedContinue(caseFoldedChars, false);
    }

    [MethodImplAttribute(MethodImplOptions.NoInlining)]
    private bool SkipCaseFoldedContinue(string caseFoldedChars, bool backtrackEvenIfCharsMatch) {
        fixed (char* pCaseFoldedChars = caseFoldedChars)
        return SkipCaseFoldedContinue(pCaseFoldedChars, caseFoldedChars.Length, backtrackEvenIfCharsMatch);
    }

    public bool Match(char[] chars, int charsIndex, int length) {
        return Skip(chars, charsIndex, length, true);
    }
    public bool Skip(char[] chars, int charsIndex, int length) {
        return Skip(chars, charsIndex, length, false);
    }
    private bool Skip(char[] chars, int charsIndex, int length, bool backtrack) {
        if (charsIndex < 0)
            throw new ArgumentOutOfRangeException("charsIndex", "charsIndex is negative.");
        if (length > chars.Length - charsIndex) // throws if chars is null
            throw new ArgumentOutOfRangeException("length", "length is out of range.");
        // We must exit early for length == 0, because pining an empty array
        // would invoke implementation-defined behaviour.
        if (length <= 0) {
            if (length < 0) throw new ArgumentOutOfRangeException("length", "length is negative.");
            if (!backtrack) ++StateTag;
            return true;
        }
        fixed (char* pChars = chars) return Skip(pChars + charsIndex, length, backtrack);
    }

    public bool Match(char* chars, int length) {
        return Skip(chars, length, true);
    }
    public bool Skip(char* chars, int length) {
        return Skip(chars, length, false);
    }
    private bool Skip(char* chars, int length, bool backtrackEvenIfCharsMatch) {
        if (unchecked((uint)length < Buffer.PositiveDistance(Ptr, PtrEnd))) {
            #if UNALIGNED_READS
                char* ptr = Ptr;
                int len = length - 2;
                while (len >= 0) {
                    if (*((int*)ptr) != *((int*)chars)) goto ReturnFalse;
                    ptr += 2; chars += 2; len -= 2;
                }
                if ((len & 1) != 0) {
                    if (*ptr != *chars) goto ReturnFalse;
                    ++ptr;
                }
            #else
                char* ptr = Ptr;
                int len = length;
                while (len != 0) {
                    if (*ptr != *chars) goto ReturnFalse;
                    ++ptr; ++chars; --len;
                }
            #endif
                if (!backtrackEvenIfCharsMatch) {
                    Ptr = ptr;
                    ++StateTag;
                }
                return true;
        ReturnFalse:
                return false;
        }
        return SkipContinue(chars, length, backtrackEvenIfCharsMatch);
    }

    public bool MatchCaseFolded(char* caseFoldedChars, int length) {
        return SkipCaseFolded(caseFoldedChars, length, true);
    }
    public bool SkipCaseFolded(char* caseFoldedChars, int length) {
        return SkipCaseFolded(caseFoldedChars, length, false);
    }
    private bool SkipCaseFolded(char* caseFoldedChars, int length, bool backtrackEvenIfCharsMatch) {
        if (unchecked((uint)length < Buffer.PositiveDistance(Ptr, PtrEnd))) {
            char* ptr = Ptr;
            int len = length;
            while (len != 0) {
                if (CaseFoldTable.FoldedChars[*ptr] != *caseFoldedChars) goto ReturnFalse;
                ++ptr; ++caseFoldedChars; --len;
            }
            if (!backtrackEvenIfCharsMatch) {
                Ptr = ptr;
                ++StateTag;
            }
            return true;
        ReturnFalse:
            return false;
        }
        return SkipCaseFoldedContinue(caseFoldedChars, length, backtrackEvenIfCharsMatch);
    }

    private bool SkipContinue(char* chars, int length, bool backtrackEvenIfCharsMatch) {
        if (length <= 0) {
            if (length < 0) throw new ArgumentOutOfRangeException("length", "length is negative.");
            return true;
        }

        if (Ptr == null) return false;

        int oldBlock = Block;
        char* oldPtr = Ptr;
        char* ptr = Ptr;

        for (;;) {
            Debug.Assert(length > 0);
            int len = (int)Buffer.PositiveDistance(ptr, PtrEnd);
            if (len < length) {
                if (BlockData == null || Block == BlockData.LastBlock) goto ReturnFalse;
                length -= len;
            } else {
                len = length;
                length = 0;
            }
            Debug.Assert(len > 0);
        #if UNALIGNED_READS
            len -= 2;
            while (len >= 0) {
                if (*((int*)ptr) != *((int*)chars)) goto ReturnFalse;
                ptr += 2; chars += 2; len -= 2;
            }
            if ((len & 1) != 0) {
                if (*ptr != *chars) goto ReturnFalse;
                ++ptr; ++chars;
            }
        #else
            do {
                if (*ptr != *chars) goto ReturnFalse;
                ++ptr; ++chars; --len;
            } while (len != 0);
        #endif
            if (length != 0) {
                Debug.Assert(BlockData != null && Block != BlockData.LastBlock);
                ptr = BlockData.ReadBlock(Block + 1);
            } else {
                if (backtrackEvenIfCharsMatch) {
                    if (Block != oldBlock) Seek(oldPtr, oldBlock);
                } else {
                    if (ptr != PtrEnd) Ptr = ptr;
                    else SeekToFirstCharAfterLastCharOfCurrentBlock();
                    ++StateTag;
                }
                return true;
            }
        }
    ReturnFalse:
        if (Block != oldBlock) Seek(oldPtr, oldBlock);
        return false;
    }

    private bool SkipCaseFoldedContinue(char* caseFoldedChars, int length, bool backtrackEvenIfCharsMatch) {
        if (length <= 0) {
            if (length == 0) return true;
            throw new ArgumentOutOfRangeException("length", "length is negative.");
        }

        if (Ptr == null) return false;

        int oldBlock = Block;
        char* oldPtr = Ptr;
        char* ptr = Ptr;

        for (;;) {
            Debug.Assert(length > 0);
            int len = (int)Buffer.PositiveDistance(ptr, PtrEnd);
            if (len < length) {
                if (BlockData == null || Block == BlockData.LastBlock) goto ReturnFalse;
                length -= len;
            } else {
                len = length;
                length = 0;
            }
            Debug.Assert(len > 0);
            do {
                if (CaseFoldTable.FoldedChars[*ptr] != *caseFoldedChars) goto ReturnFalse;
                ++ptr; ++caseFoldedChars; --len;
            } while (len != 0);
            if (length != 0) {
                Debug.Assert(BlockData != null && Block != BlockData.LastBlock);
                ptr = BlockData.ReadBlock(Block + 1);
            } else {
                if (backtrackEvenIfCharsMatch) {
                    if (Block != oldBlock) Seek(oldPtr, oldBlock);
                } else {
                    if (ptr != PtrEnd) Ptr = ptr;
                    else SeekToFirstCharAfterLastCharOfCurrentBlock();
                    ++StateTag;
                }
                return true;
            }
        }
    ReturnFalse:
        if (Block != oldBlock) Seek(oldPtr, oldBlock);
        return false;
    }

    public Match Match(Regex regex) {
        if (BufferString == null) throw new NotSupportedException("CharStream instances constructed from char arrays or char pointers do not support regular expression matching.");
        if (Ptr != null) {
            if (BlockData != null && Ptr > BlockData.RegexSpaceThreshold && Block != BlockData.LastBlock) {
                // BlockOverlap > MinRegexSpace
                char c = *Ptr;
                char* ptr = Ptr;
                BlockData.ReadBlock(Block + 1);
                int blockSizeMinusOverlap = BlockData.BlockSize - BlockData.BlockOverlap;
                Ptr = ptr - blockSizeMinusOverlap;
                PtrBegin = BufferBegin; // might have been set to null by ReadBlock
                PtrEnd = BufferEnd;
                Debug.Assert(*Ptr == c && BufferBegin <= Ptr && Ptr < BufferEnd);
            }
            int index  = (int)Buffer.PositiveDistance(BufferStringPointer, Ptr);
            int length = (int)Buffer.PositiveDistance(Ptr, BufferEnd);
            return regex.Match(BufferString, index, length);
        }
        return regex.Match("");
    }

    public bool SkipWhitespace() {
        char* lineBegin = null;
        uint lineOffset = 0;
        char* ptr = Ptr;
        char* end = unchecked(PtrEnd - 1); // - 1 to guarantee the lookahead for '\r',
        if (ptr + 1 < PtrEnd) { // PtrEnd might be null
            char c = *ptr;
            ++ptr;
            if (c > ' ') goto ReturnFalse;
            if (c == ' ') {
                if (*ptr > ' ') {
                    Ptr = ptr;
                    ++StateTag;
                    return true;
                }
                goto Loop;
            } else {
                if (c == '\r') {
                    if (*ptr == '\n') {
                        ++ptr;
                        if (ptr > end) goto Newline;
                    }
                } else if (c != '\n') goto CheckTab;
                if (*ptr > ' ') {
                    Ptr = ptr;
                    RegisterNewline();
                    return true;
                }
                goto Newline;
            CheckTab:
                if (c != '\t') goto ReturnFalse;
                goto Loop;
            }
        Newline:
            lineBegin = ptr;
            ++lineOffset;
        Loop:
            for (;;) {
                if (ptr >= end) break;
                c = *ptr;
                ++ptr;
                if (c != ' ') {
                    if (c != '\t') {
                        if (c == '\r') {
                            if (*ptr == '\n') ++ptr;
                            goto Newline;
                        }
                        if (c == '\n') goto Newline;
                        --ptr;
                        Ptr = ptr;
                        if (lineOffset == 0) {
                            ++StateTag;
                            return true;
                        } else {
                            RegisterNewlines(lineBegin, lineOffset);
                            return true;
                        }
                    }
                }
            }
        }
        return SkipWhitespaceContinue(ptr, lineBegin, lineOffset);
    ReturnFalse:
        return false;
    }
    private bool SkipWhitespaceContinue(char* ptr, char* lineBegin, uint lineOffset) {
        var stateTag = StateTag;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        if (index == 0) {
            c = Peek();
            if (c == ' ' || c == '\t') c = SkipAndPeek();
            else if (c != '\r' && c != '\n') return false;
        } else {
            if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
            c = SkipAndPeek(index);
        }
        for (;;) {
             if (c == ' ' || c == '\t') c = SkipAndPeek();
             else if (c != '\r' && c != '\n') {
                 StateTag = stateTag + 1;
                 return true;
             } else {
                 char c0 = c;
                 c = SkipAndPeek();
                 if (c0 == '\r' && c == '\n') c = SkipAndPeek();
                 RegisterNewline();
             }
        }
    }

    public bool SkipUnicodeWhitespace() {
        char* lineBegin = null;
        uint lineOffset = 0;
        char* end = unchecked(PtrEnd - 1); // - 1 to guarantee the lookahead for '\r'
        char* ptr = Ptr;
        if (ptr + 1 < PtrEnd) { // PtrEnd might be null
            char c = *ptr;
            ++ptr;
            if (c == ' ') goto Loop;
            if (!Text.IsWhitespace(c)) return false;
            if (c <= '\r') {
                if (c == '\r') {
                    if (*ptr == '\n') ++ptr;
                } else if (c != '\n') goto Loop;
            } else {
                if (c < '\u2028' ? c != '\u0085' : c > '\u2029') goto Loop;
            }
        Newline:
            lineBegin = ptr;
            ++lineOffset;
        Loop:
            for (;;) {
                if (ptr >= end) break;
                c = *ptr;
                ++ptr;
                if (c != ' ') {
                    if (Text.IsWhitespace(c)) {
                        if (c <= '\r') {
                            if (c == '\r') {
                                if (*ptr == '\n') ++ptr;
                                goto Newline;
                            }
                            if (c == '\n') goto Newline;
                        } else if (c < '\u2028' ? c == '\u0085' : c <= '\u2029') goto Newline;
                    } else {
                        --ptr;
                        Ptr = ptr;
                        if (lineOffset == 0) {
                            ++StateTag;
                            return true;
                        } else {
                            RegisterNewlines(lineBegin, lineOffset);
                            return true;
                        }
                    }
                }
            }
        }
        return SkipUnicodeWhitespaceContinue(ptr, lineBegin, lineOffset);
    }
    private bool SkipUnicodeWhitespaceContinue(char* ptr, char* lineBegin, uint lineOffset) {
        var stateTag = StateTag;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        if (index == 0) {
            c = Peek();
            if (!Text.IsWhitespace(c)) return false;
            if (c == ' ' || c == '\t') c = SkipAndPeek();
        } else {
            if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
            c = SkipAndPeek(index);
        }
        for (;;) {
            if (c == ' ') c = SkipAndPeek();
            else {
                if (!Text.IsWhitespace(c)) break;
                char c0 = c;
                c = SkipAndPeek();
                if (c0 <= '\r') {
                    if (c0 == '\r') {
                        if (c == '\n') c = SkipAndPeek();
                    } else if (c0 != '\n') continue;
                } else if (c0 < '\u2028' ? c0 != '\u0085' : c0 > '\u2029') continue;
                RegisterNewline();
            }
        }
        StateTag = stateTag + 1;
        return true;
    }

    public bool SkipNewline() {
        var ptr = Ptr;
        if (ptr + 2 < PtrEnd) {
            char c = *ptr;
            ++ptr;
            if (c == '\r') {
                if (*ptr == '\n') ++ptr;
            } else if (c != '\n') return false;
            Ptr = ptr;
            RegisterNewline();
            return true;
        } else {
            var stateTag = StateTag;
            char c = Peek();
            if (c == '\r') {
                c = SkipAndPeek();
                if (c == '\n') Skip();
            } else {
                if (c != '\n') return false;
                Skip();
            }
            RegisterNewline();
            StateTag = stateTag + 1;
            return true;
        }
    }

    public bool SkipUnicodeNewline() {
        var ptr = Ptr;
        if (ptr + 2 < PtrEnd) {
            char c = *ptr;
            ++ptr;
            if (c <= '\r') {
                if (c == '\r') {
                    if (*ptr == '\n') ++ptr;
                } else if (c != '\n') goto ReturnFalse;
            } else if (c >= '\u2028' ? c > '\u2029' : c != '\u0085') goto ReturnFalse;
            Ptr = ptr;
            RegisterNewline();
            return true;
        } else {
            char c = Peek();
            uint n = 1;
            if (c <= '\r') {
                if (c == '\r') {
                    if (Peek(1u) == '\n') n = 2;
                } else if (c != '\n') goto ReturnFalse;
            } else if (c >= '\u2028' ? c > '\u2029' : c != '\u0085') goto ReturnFalse;
            Skip(n);
            var stateTag = StateTag;
            RegisterNewline();
            StateTag = stateTag;
            return true;
        }
    ReturnFalse:
        return false;
    }

    public int SkipNewlineThenWhitespace(int powerOf2TabStopDistance, bool allowFormFeed) {
        int tabStopDistanceMinus1 = unchecked(powerOf2TabStopDistance - 1);
        if (powerOf2TabStopDistance <= 0 || (powerOf2TabStopDistance & tabStopDistanceMinus1) != 0)
            throw new ArgumentOutOfRangeException("powerOf2TabStopDistance", "powerOf2TabStopDistance must be a positive power of 2.");

        char* lineBegin = null;
        uint lineOffset = 0;
        int ind = -1;
        char* end = unchecked(PtrEnd - 1); // - 1 to guarantee the lookahead for '\r'
        char* ptr = Ptr;
        if (ptr + 1 < PtrEnd) { // PtrEnd might be null
            char c = *ptr;
            ++ptr;
            if (c == '\r') {
                if (*ptr == '\n') ++ptr;
            } else if (c != '\n') {
                return -1;
            }
        Newline:
            lineBegin = ptr;
            ++lineOffset;
            ind = 0;
            for (;;) {
                if (ptr >= end) break;
                c = *ptr;
                ++ptr;
                if (c == ' ') {
                    ind = unchecked(ind + 1);
                    if (ind >= 0) continue;
                    // indentation has overflown, so put back ' ' and return
                    ind = unchecked(ind - 1);
                } else if (c <= '\r') {
                    if (c == '\r') {
                        if (*ptr == '\n') ++ptr;
                        goto Newline;
                    }
                    if (c == '\n') goto Newline;
                    if (c == '\t') {
                        // ind = ind + tabStopDistance - ind%tabStopDistance
                        int d = tabStopDistanceMinus1 + 1 - (ind & tabStopDistanceMinus1);
                        ind = unchecked(ind + d);
                        if (ind >= 0) continue;
                        // indentation has overflown, so put back '\t' and return
                        ind = unchecked(ind - d);
                    } else if (c == '\f' && allowFormFeed) {
                        ind = 0;
                        continue;
                    }
                }
                --ptr;
                Ptr = ptr;
                RegisterNewlines(lineBegin, lineOffset);
                return ind;
            }
            // end of block
        }
        return SkipNewlineWhitespaceContinue(ptr, lineBegin, lineOffset, ind, tabStopDistanceMinus1, allowFormFeed);
    }
    private int SkipNewlineWhitespaceContinue(char* ptr, char* lineBegin, uint lineOffset, int ind_,
                                              int tabStopDistanceMinus1, bool allowFormFeed)
    {
        var stateTag = StateTag;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        if (index == 0) {
            c = Peek();
            if (!(c == '\r' || c == '\n')) return -1;
        } else {
            RegisterNewlines(lineBegin, lineOffset);
            c = SkipAndPeek(index);
        }
        int ind = ind_;
        for (;;) {
            if (c == ' ') {
                ind = unchecked(ind + 1);
                if (ind >= 0) c = SkipAndPeek();
                else {
                    // indentation has overflown, so put back ' ' and return
                    ind = unchecked(ind - 1);
                    break;
                }
            } else if (c == '\r' || c == '\n') {
                ind = 0;
                char c0 = c;
                c = SkipAndPeek();
                if (c0 == '\r' && c == '\n') c = SkipAndPeek();
                RegisterNewline();
            } else if (c == '\t') {
                // ind = ind + tabStopDistance - ind%tabStopDistance
                int d = tabStopDistanceMinus1 + 1 - (ind & tabStopDistanceMinus1);
                ind = unchecked(ind + d);
                if (ind >= 0) c = SkipAndPeek();
                else {
                    // indentation has overflown, so put back '\t' and return
                    ind = unchecked(ind - d);
                    break;
                }
            } else if (c == '\f' && allowFormFeed) {
                ind = 0;
                c = SkipAndPeek();
            } else break;
        }
        StateTag = stateTag + 1;
        return ind;
    }

    public void SkipRestOfLine(bool skipNewline) {
        char* ptr = Ptr;
        char* end = unchecked(PtrEnd - 2); // - 2, so that we can do (*) without further checking
        if (ptr + 2 < PtrEnd) { // PtrEnd might be null
            for (;;) {
                char c = *ptr;
                if (c > '\r') {
                    if (++ptr == end) break;
                } else if (c != '\r' && c != '\n') {
                    if (++ptr == end) break;
                } else {
                    if (!skipNewline) {
                        if (ptr != Ptr) {
                            Ptr = ptr;
                            ++StateTag;
                        }
                        return;
                    } else {
                        ++ptr;
                        if (c == '\r' && *ptr == '\n') ++ptr;
                        Ptr = ptr; // (*)
                        RegisterNewline();
                        return;
                    }
                }
            }
        }
        SkipRestOfLineContinue(ptr, skipNewline);
    }
    private void SkipRestOfLineContinue(char* ptr, bool skipNewline) {
        var stateTag = StateTag;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        if (index == 0) {
            c = Peek();
            if (c == EOS || (!skipNewline && (c == '\r' || c == '\n'))) return;
        } else {
            c = SkipAndPeek(index);
        }
        while (c != EOS) {
            if (c == '\r' || c == '\n') {
                if (skipNewline) SkipNewline();
                break;
            }
            c = SkipAndPeek();
        }
        StateTag = stateTag + 1;
        return;
    }

    public string ReadRestOfLine(bool skipNewline) {
        char* ptr = Ptr;
        char* end = unchecked(PtrEnd - 2); // - 2, so that we can do (*) without further checking
        if (ptr + 2 < PtrEnd) { // PtrEnd might be null
            for (;;) {
                char c = *ptr;
                if (c > '\r') {
                    if (++ptr == end) break;
                } else if (c != '\r' && c != '\n') {
                    if (++ptr == end) break;
                } else {
                    char* ptr0 = Ptr;
                    if (!skipNewline) {
                        if (ptr != ptr0) {
                            Ptr = ptr;
                            ++StateTag;
                            return new string(ptr0, 0, (int)Buffer.PositiveDistance(ptr0, ptr));
                        } else {
                            return "";
                        }
                    } else {
                        var skippedString = ptr == ptr0 ? "" : new string(ptr0, 0, (int)Buffer.PositiveDistance(ptr0, ptr));
                        ++ptr;
                        if (c == '\r' && *ptr == '\n') ++ptr;
                        Ptr = ptr; // (*)
                        RegisterNewline();
                        return skippedString;
                    }
                }
            }
        }
        return ReadRestOfLineContinue(ptr, skipNewline);
    }
    private string ReadRestOfLineContinue(char* ptr, bool skipNewline) {
        var stateTag = StateTag;
        var indexToken = IndexToken;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        if (index == 0) {
            c = Peek();
            if (c == EOS || (!skipNewline && (c == '\r' || c == '\n'))) return "";
        } else {
            c = SkipAndPeek(index);
        }
        while (c != EOS) {
            if (c == '\r' || c == '\n') {
                var skippedString = ReadFrom(indexToken);
                if (skipNewline) SkipNewline();
                StateTag = stateTag + 1;
                return skippedString;
            }
            c = SkipAndPeek();
        }
        StateTag = stateTag + 1;
        return ReadFrom(indexToken);
    }

    public char ReadCharOrNewline() {
        var ptr = Ptr;
        if (ptr + 2 < PtrEnd) {
            char c = *ptr;
            ++ptr;
            if (c != '\r') {
                if (c != '\n') {
                    Ptr = ptr;
                    ++StateTag;
                    return c;
                }
            } else if (*ptr == '\n') ++ptr;
            Ptr = ptr;
            RegisterNewline();
            return '\n';
        } else {
            char c0 = Peek();
            if (c0 != EOS) {
                char c = SkipAndPeek();
                var stateTag = StateTag;
                if (c0 != '\r') {
                    if (c0 != '\n') return c0;
                } else if (c == '\n') Skip();
                RegisterNewline();
                StateTag = stateTag;
                return '\n';
            }
            return EOS;
        }
    }

    public int SkipCharsOrNewlines(int maxCount) {
        if (maxCount < 0) throw new ArgumentOutOfRangeException("maxCount", "maxCount is negative.");
        char* lineBegin = null;
        uint lineOffset = 0;
        int nCRLF = 0;
        char* ptr = Ptr;
        if (ptr != null) {
            char* bufferEnd1 = PtrEnd - 1; // - 1 to guarantee the lookahead for '\r'
            char* end2 = unchecked(ptr + maxCount);
            char* end = end2 >= ptr && end2 <= bufferEnd1 ? end2 : bufferEnd1;
            if (ptr < end) {
                for (;;) {
                    char c = *ptr;
                    ++ptr;
                    if (c > '\r') {
                        if (ptr == end) break;
                    } else {
                        if (c == '\r') {
                            if (*ptr == '\n') {
                                ++ptr;
                                ++nCRLF;
                                if (end < bufferEnd1) ++end;
                            }
                        } else if (c != '\n') goto CheckBound;
                        lineBegin = ptr;
                        ++lineOffset;
                    CheckBound:
                        if (ptr >= end) break;
                    }
                }
                if (end < bufferEnd1) {
                    int count = (int)Buffer.PositiveDistance(Ptr, ptr) - nCRLF;
                    Ptr = ptr;
                    if (lineOffset == 0) {
                        ++StateTag;
                        return count;
                    } else {
                        RegisterNewlines(lineBegin, lineOffset);
                        return count;
                    }
                }
            }
        }
        return SkipCharsOrNewlinesContinue(ptr, lineBegin, lineOffset, nCRLF, maxCount);
    }
    private int SkipCharsOrNewlinesContinue(
                    char* ptr, char* lineBegin, uint lineOffset, int nCRLF,
                    int maxCount)
    {
        var stateTag = StateTag;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        int count;
        if (index == 0) {
            if (maxCount == 0 || (c = Peek()) == EOS) return 0;
            count = 0;
        } else {
            if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
            c = SkipAndPeek(index);
            count = (int)index - nCRLF;
        }
        for (;;) {
             if (c == EOS || count == maxCount) break;
             ++count;
             char c0 = c;
             c = SkipAndPeek();
             if (c0 <= '\r') {
                 if (c0 == '\r') {
                     if (c == '\n') c = SkipAndPeek();
                 } else if (c0 != '\n') continue;
                 RegisterNewline();
             }
        }
        StateTag = unchecked(stateTag + 1);
        return count;
    }

    public string ReadCharsOrNewlines(int maxCount, bool normalizeNewlines) {
        if (maxCount < 0) throw new ArgumentOutOfRangeException("maxCount", "maxCount is negative.");
        char* lineBegin = null;
        uint lineOffset = 0;
        int nCRLF = 0;
        int nCR = 0;
        char* ptr = Ptr;
        if (ptr != null) {
            char* PtrEnd1 = PtrEnd - 1; // - 1 to guarantee the lookahead for '\r'
            char* end2 = unchecked(ptr + maxCount);
            char* end = end2 >= ptr && end2 <= PtrEnd1 ? end2 : PtrEnd1;
            if (ptr < end) {
                for (;;) {
                    char c = *ptr;
                    ++ptr;
                    if (c > '\r') {
                        if (ptr == end) break;
                    } else {
                        if (c == '\r') {
                            if (*ptr == '\n') {
                                ++ptr;
                                ++nCRLF;
                                if (end < PtrEnd1) ++end;
                            } else {
                                ++nCR;
                            }
                        } else if (c != '\n') goto CheckBound;
                        lineBegin = ptr;
                        ++lineOffset;
                    CheckBound:
                        if (ptr >= end) break;
                    }
                }
                if (end < PtrEnd1) {
                    char* ptr0 = Ptr;
                    Ptr = ptr;
                    int length = (int)Buffer.PositiveDistance(ptr0, ptr);
                    if (lineOffset == 0) {
                        ++StateTag;
                        return new string(ptr0, 0, length);
                    }
                    RegisterNewlines(lineBegin, lineOffset);
                    return   !normalizeNewlines || (nCR | nCRLF) == 0
                           ? new string(ptr0, 0, length)
                           : Text.CopyWithNormalizedNewlines(ptr0, length, nCRLF, nCR);
                }
            }
        }
        return ReadCharsOrNewlinesContinue(ptr, lineBegin, lineOffset, nCRLF, nCR, maxCount, normalizeNewlines);
    }
    private string ReadCharsOrNewlinesContinue(
                       char* ptr, char* lineBegin, uint lineOffset, int nCRLF, int nCR,
                       int maxCount, bool normalizeNewlines)
    {
        var stateTag = StateTag;
        var indexToken = IndexToken;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        int count;
        if (index == 0) {
            if (maxCount == 0 || (c = Peek()) == EOS) return "";
            count = 0;
        } else {
            if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
            c = SkipAndPeek(index);
            count = (int)index - nCRLF;
        }
        for (;;) {
             if (c == EOS || count == maxCount) break;
             ++count;
             char c0 = c;
             c = SkipAndPeek();
             if (c0 <= '\r') {
                 if (c0 == '\r') {
                     if (c == '\n') {
                        ++nCRLF;
                        c = SkipAndPeek();
                     } else {
                         ++nCR;
                     }
                 } else if (c0 != '\n') continue;
                 RegisterNewline();
             }
        }
        StateTag = unchecked(stateTag + 1);
        string str = ReadFrom(indexToken);
        if ((nCR | nCRLF) == 0 || !normalizeNewlines) return str;
        fixed (char* pStr = str)
        return Text.CopyWithNormalizedNewlines(pStr, str.Length, nCRLF, nCR);
    }

    public int SkipCharsOrNewlinesWhile(Microsoft.FSharp.Core.FSharpFunc<char,bool> predicate) {
        return SkipCharsOrNewlinesWhile(predicate, predicate);
    }
    public int SkipCharsOrNewlinesWhile(FSharpFunc<char,bool> predicateForFirstChar,
                                        FSharpFunc<char,bool> predicate)
    {
        char* lineBegin = null;
        uint lineOffset = 0;
        int nCRLF = 0;
        char* ptr = Ptr;
        char* end = unchecked(PtrEnd - 1); // - 1 to guarantee the lookahead for '\r'
        if (ptr + 1 < PtrEnd) { // PtrEnd might be null
            char c = *ptr;
            ++ptr;
            if (c > '\r') {
                if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
            } else if (c == '\r') {
                if (!predicateForFirstChar.Invoke('\n')) goto ReturnEmpty;
                if (*ptr == '\n') {
                    ++ptr;
                    ++nCRLF;
                }
                lineBegin = ptr;
                ++lineOffset;
            } else {
                if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
                if (c == '\n') {
                    lineBegin = ptr;
                    lineOffset = 1;
                }
            }
            for (;;) {
                if (ptr >= end) goto EndOfBlock;
                c = *ptr;
                ++ptr;
                if (c > '\r') {
                    if (!predicate.Invoke(c)) break;
                } else if (c == '\r') {
                    if (!predicate.Invoke('\n')) break;
                    if (*ptr == '\n') {
                        ++ptr;
                        ++nCRLF;
                    }
                    lineBegin = ptr;
                    ++lineOffset;
                } else {
                    if (!predicate.Invoke(c)) break;
                    if (c == '\n') {
                        lineBegin = ptr;
                        ++lineOffset;
                    }
                }
            }
            --ptr;
            int count = (int)Buffer.PositiveDistance(Ptr, ptr) - nCRLF;
            Ptr = ptr;
            if (lineOffset == 0) {
                ++StateTag;
                return count;
            }
            RegisterNewlines(lineBegin, lineOffset);
            return count;
        ReturnEmpty:
            return 0;
        }
    EndOfBlock:
        return SkipCharsOrNewlinesWhileContinue(ptr, lineBegin, lineOffset, nCRLF, predicateForFirstChar, predicate);
    }
    private int SkipCharsOrNewlinesWhileContinue(
                    char* ptr, char* lineBegin, uint lineOffset, int nCRLF,
                    FSharpFunc<char,bool> predicateForFirstChar, FSharpFunc<char,bool> predicate)
    {
        var stateTag = StateTag;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        int count;
        if (index == 0) {
            c = Peek();
            char cc = c == '\r' ? '\n' : c;
            if (c == EOS || !predicateForFirstChar.Invoke(cc)) return 0;
            count = 1;
            char c0 = c;
            c = SkipAndPeek();
            if (cc == '\n') {
                if (c0 == '\r' && c == '\n') c = SkipAndPeek();
                RegisterNewline();
            }
        } else {
            if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
            c = SkipAndPeek(index);
            count = (int)index - nCRLF;
        }
        for (;;) {
            if (c == EOS) break;
            if (c != '\r' && c != '\n') {
                if (!predicate.Invoke(c)) break;
                count = unchecked(count + 1);
                if (count >= 0) c = SkipAndPeek();
                else { // overflow
                    count = unchecked(count - 1);
                    break;
                }
            } else {
                if (!predicate.Invoke('\n')) break;
                count = unchecked(count + 1);
                if (count >= 0) {
                    char c0 = c;
                    c = SkipAndPeek();
                    if (c0 == '\r' && c == '\n') c = SkipAndPeek();
                    RegisterNewline();
                } else {
                    count = unchecked(count - 1);
                    break;
                }
            }
        }
        StateTag = unchecked(stateTag + 1);
        return count;
    }

    public string ReadCharsOrNewlinesWhile(FSharpFunc<char,bool> predicate, bool normalizeNewlines) {
        return ReadCharsOrNewlinesWhile(predicate, predicate, normalizeNewlines);
    }
    public string ReadCharsOrNewlinesWhile(
                      FSharpFunc<char,bool> predicateForFirstChar, FSharpFunc<char,bool> predicate,
                      bool normalizeNewlines)
    {
        char* lineBegin = null;
        uint lineOffset = 0;
        int nCRLF = 0;
        int nCR = 0;
        char* ptr = Ptr;
        char* end = unchecked(PtrEnd - 1); // - 1 to guarantee the lookahead for '\r'
        if (ptr + 1 < PtrEnd) { // PtrEnd might be null
            char c = *ptr;
            ++ptr;
            if (c > '\r') {
                if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
            } else if (c == '\r') {
                if (!predicateForFirstChar.Invoke('\n')) goto ReturnEmpty;
                if (*ptr == '\n') {
                    ++ptr;
                    ++nCRLF;
                } else {
                    ++nCR;
                }
                lineBegin = ptr;
                ++lineOffset;
            } else {
                if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
                if (c == '\n') {
                    lineBegin = ptr;
                    lineOffset = 1;
                }
            }
            for (;;) {
                if (ptr >= end) goto EndOfBlock;
                c = *ptr;
                ++ptr;
                if (c > '\r') {
                    if (!predicate.Invoke(c)) break;
                } else if (c == '\r') {
                    if (!predicate.Invoke('\n')) break;
                    if (*ptr == '\n') {
                        ++ptr;
                        ++nCRLF;
                    } else {
                        ++nCR;
                    }
                    lineBegin = ptr;
                    ++lineOffset;
                } else {
                    if (!predicate.Invoke(c)) break;
                    if (c == '\n') {
                        lineBegin = ptr;
                        ++lineOffset;
                    }
                }
            }
            --ptr;
            char* ptr0 = Ptr;
            Ptr = ptr;
            int length = (int)Buffer.PositiveDistance(ptr0, ptr);
            if (lineOffset == 0) {
                ++StateTag;
                return new string(ptr0, 0, length);
            }
            RegisterNewlines(lineBegin, lineOffset);
            return !normalizeNewlines || (nCR | nCRLF) == 0
                   ? new string(ptr0, 0, length)
                   : Text.CopyWithNormalizedNewlines(ptr0, length, nCRLF, nCR);
        ReturnEmpty:
            return "";
        }
    EndOfBlock:
        return ReadCharsOrNewlinesWhileContinue(ptr, lineBegin, lineOffset, nCRLF, nCR, predicateForFirstChar, predicate, normalizeNewlines);
    }
    private string ReadCharsOrNewlinesWhileContinue(
                       char* ptr, char* lineBegin, uint lineOffset, int nCRLF, int nCR,
                       FSharpFunc<char,bool> predicateForFirstChar, FSharpFunc<char,bool> predicate,
                       bool normalizeNewlines)
    {
        var stateTag = StateTag;
        var indexToken = IndexToken;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        int count;
        if (index == 0) {
            c = Peek();
            char cc = c == '\r' ? '\n' : c;
            if (c == EOS || !predicateForFirstChar.Invoke(cc)) return "";
            count = 1;
            char c0 = c;
            c = SkipAndPeek();
            if (cc == '\n') {
                if (c0 == '\r') {
                    if (c == '\n') {
                        ++nCRLF;
                        c = SkipAndPeek();
                    } else {
                        ++nCR;
                    }
                }
                RegisterNewline();
            }
        } else {
            if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
            c = SkipAndPeek(index);
            count = (int)index - nCRLF;
        }
        for (;;) {
            if (c == EOS) break;
            if (c != '\r' && c != '\n') {
                if (!predicate.Invoke(c)) break;
                count = unchecked(count + 1);
                if (count < 0) break;
                c = SkipAndPeek();
            } else {
                if (!predicate.Invoke('\n')) break;
                count = unchecked(count + 1);
                if (count < 0) break;
                char c0 = c;
                c = SkipAndPeek();
                if (c0 == '\r') {
                    if (c == '\n') {
                        ++nCRLF;
                        c = SkipAndPeek();
                    } else {
                        ++nCR;
                    }
                }
                RegisterNewline();
            }
        }
        StateTag = unchecked(stateTag + 1);
        string str = ReadFrom(indexToken);
        if ((nCR | nCRLF) == 0 || !normalizeNewlines) return str;
        fixed (char* pStr = str)
        return Text.CopyWithNormalizedNewlines(pStr, str.Length, nCRLF, nCR);
    }

    public int SkipCharsOrNewlinesWhile(FSharpFunc<char,bool> predicate, int minCount, int maxCount) {
        return SkipCharsOrNewlinesWhile(predicate, predicate, minCount, maxCount);
    }
    public int SkipCharsOrNewlinesWhile(
                   FSharpFunc<char,bool> predicateForFirstChar, FSharpFunc<char,bool> predicate,
                   int minCount, int maxCount)
    {
        if (maxCount < 0) throw new ArgumentOutOfRangeException("maxCount", "maxCount is negative.");
        char* lineBegin = null;
        uint lineOffset = 0;
        int nCRLF = 0;
        char* ptr = Ptr;
        if (ptr != null) {
            char* bufferEnd1 = unchecked(PtrEnd - 1); // - 1 to guarantee the lookahead for '\r'
            char* end2 = unchecked(ptr + maxCount);
            char* end = end2 >= ptr && end2 <= bufferEnd1 ? end2 : bufferEnd1;
            if (ptr < end) {
                char c = *ptr;
                ++ptr;
                if (c > '\r') {
                    if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
                } else if (c == '\r') {
                    if (!predicateForFirstChar.Invoke('\n')) goto ReturnEmpty;
                    if (*ptr == '\n') {
                        ++ptr;
                        ++nCRLF;
                        if (end < bufferEnd1) ++end;
                    }
                    lineBegin = ptr;
                    ++lineOffset;
                } else {
                    if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
                    if (c == '\n') {
                        lineBegin = ptr;
                        ++lineOffset;
                    }
                }
                for (;;) {
                    if (ptr < end) {
                        c = *ptr;
                        ++ptr;
                        if (c > '\r') {
                            if (!predicate.Invoke(c)) break;
                        } else if (c == '\r') {
                            if (!predicate.Invoke('\n')) break;
                            if (*ptr == '\n') {
                                ++ptr;
                                ++nCRLF;
                                if (end < bufferEnd1) ++end;
                            }
                            lineBegin = ptr;
                            ++lineOffset;
                        } else {
                            if (!predicate.Invoke(c)) break;
                            if (c == '\n') {
                                lineBegin = ptr;
                                ++lineOffset;
                            }
                        }
                    } else {
                        if (end >= bufferEnd1) goto EndOfBlock;
                        goto ReturnCount;
                    }
                }
                --ptr;
            ReturnCount:
                int count = (int)Buffer.PositiveDistance(Ptr, ptr) - nCRLF;
                if (count >= minCount) {
                    Ptr = ptr;
                    if (lineOffset == 0) {
                        ++StateTag;
                        return count;
                    } else {
                        RegisterNewlines(lineBegin, lineOffset);
                        return count;
                    }
                }
            ReturnEmpty:
                return 0;
            }
        }
    EndOfBlock:
        return SkipCharsOrNewlinesWhileContinue(ptr, lineBegin, lineOffset, nCRLF, predicateForFirstChar, predicate, minCount, maxCount);
    }
    private int SkipCharsOrNewlinesWhileContinue(
                    char* ptr, char* lineBegin, uint lineOffset, int nCRLF,
                    FSharpFunc<char,bool> predicateForFirstChar, FSharpFunc<char,bool> predicate,
                    int minCount, int maxCount)
    {
        var ptr0 = Ptr;
        var block0 = Block;
        var tag0 = StateTag;
        var line0 = _Line;
        var lineBegin0 = _LineBegin;
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        int count;
        if (index == 0) {
            c = Peek();
            if (c == EOS || maxCount == 0) goto ReturnEmpty;
            if (c != '\r' && c != '\n') {
                if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
                count = 1;
                c = SkipAndPeek();
            } else {
                if (!predicateForFirstChar.Invoke('\n')) goto ReturnEmpty;
                count = 1;
                char c0 = c;
                c = SkipAndPeek();
                if (c0 == '\r' && c == '\n') c = SkipAndPeek();
                RegisterNewline();
            }
        } else {
            if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
            c = SkipAndPeek(index);
            count = (int)index - nCRLF;
        }
        for (;;) {
            if (c == EOS || count == maxCount) break;
            if (c != '\r' && c != '\n') {
                if (!predicate.Invoke(c)) break;
                ++count;
                c = SkipAndPeek();
            } else {
                if (!predicate.Invoke('\n')) break;
                ++count;
                char c0 = c;
                c = SkipAndPeek();
                if (c0 == '\r' && c == '\n') c = SkipAndPeek();
                RegisterNewline();
            }
        }
        if (count >= minCount) {
            StateTag = unchecked(tag0 + 1);
            return count;
        }
    ReturnEmpty:
        // backtrack
        Seek(ptr0, block0);
        _Line = line0;
        _LineBegin = lineBegin0;
        StateTag = tag0;
        return 0;
    }

    public string ReadCharsOrNewlinesWhile(FSharpFunc<char,bool> predicate, int minCount, int maxCount, bool normalizeNewlines) {
        return ReadCharsOrNewlinesWhile(predicate, predicate, minCount, maxCount, normalizeNewlines);
    }
    public string ReadCharsOrNewlinesWhile(
                      FSharpFunc<char,bool> predicateForFirstChar, FSharpFunc<char,bool> predicate,
                      int minCount, int maxCount, bool normalizeNewlines)
    {
        if (maxCount < 0) throw new ArgumentOutOfRangeException("maxCount", "maxCount is negative.");
        char* lineBegin = null;
        uint lineOffset = 0;
        int nCRLF = 0;
        int nCR = 0;
        char* ptr = Ptr;
        if (ptr != null) {
            char* bufferEnd1 = PtrEnd - 1; // - 1 to guarantee the lookahead for '\r'
            char* end2 = unchecked(ptr + maxCount);
            char* end = end2 >= ptr && end2 <= bufferEnd1 ? end2 : bufferEnd1;
            if (ptr < end) {
                char c = *ptr;
                ++ptr;
                if (c > '\r') {
                    if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
                } else if (c == '\r') {
                    if (!predicateForFirstChar.Invoke('\n')) goto ReturnEmpty;
                    if (*ptr == '\n') {
                        ++ptr;
                        ++nCRLF;
                        if (end < bufferEnd1) ++end;
                    } else {
                        ++nCR;
                    }
                    lineBegin = ptr;
                    lineOffset = 1;
                } else {
                    if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
                    if (c == '\n') {
                        lineBegin = ptr;
                        lineOffset = 1;
                    }
                }
                for (;;) {
                    if (ptr < end) {
                        c = *ptr;
                        ++ptr;
                        if (c > '\r') {
                            if (!predicate.Invoke(c)) break;
                        } else if (c == '\r') {
                            if (!predicate.Invoke('\n')) break;
                            if (*ptr == '\n') {
                                ++ptr;
                                ++nCRLF;
                                if (end < bufferEnd1) ++end;
                            } else {
                                ++nCR;
                            }
                            lineBegin = ptr;
                            ++lineOffset;
                        } else {
                            if (!predicate.Invoke(c)) break;
                            if (c == '\n') {
                                lineBegin = ptr;
                                ++lineOffset;
                            }
                        }
                    } else {
                        if (end >= bufferEnd1) goto EndOfBlock;
                        goto ReturnStringInBlock;
                    }
                }
                --ptr;
            ReturnStringInBlock:
                {
                    char* ptr0 = Ptr;
                    int length = (int)Buffer.PositiveDistance(ptr0, ptr);
                    if (length - nCRLF >= minCount) {
                        Ptr = ptr;
                        if (lineOffset == 0) {
                            ++StateTag;
                            return new string(ptr0, 0, length);
                        }
                        RegisterNewlines(lineBegin, lineOffset);
                        return !normalizeNewlines || (nCR | nCRLF) == 0
                               ? new string(ptr0, 0, length)
                               : Text.CopyWithNormalizedNewlines(ptr0, length, nCRLF, nCR);
                    }
                }
            ReturnEmpty:
                return "";
            }
        }
    EndOfBlock:
        return ReadCharsOrNewlinesWhileContinue(ptr, lineBegin, lineOffset, nCRLF, nCR, predicateForFirstChar, predicate, minCount, maxCount, normalizeNewlines);
    }
    private string ReadCharsOrNewlinesWhileContinue(
                       char* ptr, char* lineBegin, uint lineOffset, int nCRLF, int nCR,
                       FSharpFunc<char,bool> predicateForFirstChar, FSharpFunc<char,bool> predicate,
                       int minCount, int maxCount, bool normalizeNewlines)
    {
        var ptr0 = Ptr;
        var block0 = Block;
        var tag0 = StateTag;
        var line0 = _Line;
        var lineBegin0 = _LineBegin;

        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c;
        int count;
        if (index == 0) {
            c = Peek();
            if (c == EOS || maxCount == 0) goto ReturnEmpty;
            if (c != '\r' && c != '\n') {
                count = 1;
                if (!predicateForFirstChar.Invoke(c)) goto ReturnEmpty;
                c = SkipAndPeek();
            } else {
                if (!predicateForFirstChar.Invoke('\n')) goto ReturnEmpty;
                count = 1;
                char c0 = c;
                c = SkipAndPeek();
                if (c0 == '\r') {
                    if (c == '\n') {
                        ++nCRLF;
                        c = SkipAndPeek();
                    } else {
                        ++nCR;
                    }
                }
                RegisterNewline();
            }
        } else {
            if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
            c = SkipAndPeek(index);
            count = (int)index - nCRLF;
        }
        for (;;) {
            if (c == EOS || count == maxCount) break;
            if (c != '\r' && c != '\n') {
                if (!predicate.Invoke(c)) break;
                ++count;
                c = SkipAndPeek();
            } else {
                if (!predicate.Invoke('\n')) break;
                ++count;
                char c0 = c;
                c = SkipAndPeek();
                if (c0 == '\r') {
                    if (c == '\n') {
                        ++nCRLF;
                        c = SkipAndPeek();
                    } else {
                        ++nCR;
                    }
                }
                RegisterNewline();
            }
        }
        if (count >= minCount) {
            StateTag = unchecked(tag0 + 1);
            string str = ReadFrom(ptr0, block0);
            if ((nCR | nCRLF) == 0 || !normalizeNewlines) return str;
            fixed (char* pStr = str)
            return Text.CopyWithNormalizedNewlines(pStr, str.Length, nCRLF, nCR);
        }
    ReturnEmpty:
         // backtrack
        Seek(ptr0, block0);
        _Line = line0;
        _LineBegin = lineBegin0;
        StateTag = tag0;
        return "";
    }

    private static bool Rest3OfStringEquals(char* str1, char* str2, int length) {
        for (int i = 3; i < length; ++i) {
            if (str1[i] != str2[i]) goto ReturnFalse;
        }
        return true;
    ReturnFalse:
        return false;
    }

    private static bool Rest3OfStringEqualsCaseFolded(char* str1, char* cfStr2, int length) {
        char* cftable = CaseFoldTable.FoldedChars;
        for (int i = 3; i < length; ++i) {
            if (cftable[str1[i]] != cfStr2[i]) goto ReturnFalse;
        }
        return true;
    ReturnFalse:
        return false;
    }

    public int SkipCharsOrNewlinesUntilString(string str, int maxCount, out bool foundString) {
        int strLength = str.Length; // throws if str is null
        if (strLength == 0) throw new ArgumentException("The string argument is empty.");
        if (maxCount < 0) throw new ArgumentOutOfRangeException("maxCount", "maxCount is negative.");
        char* lineBegin = null;
        fixed (char* pStr = str) {
            uint lineOffset = 0;
            int nCRLF = 0;
            char* ptr = Ptr;
            if (ptr != null) {
                char* bufferEnd = PtrEnd;
                char* end1 = unchecked(bufferEnd - strLength);
                if (end1 >= ptr && end1 < bufferEnd) {
                    char* end2 = unchecked(ptr + maxCount);
                    char* end = end2 < ptr || end1 <= end2 ? end1 : end2;
                    for (;;) {
                        char c = *ptr;
                        if (c != pStr[0]) {
                            if (ptr == end) break;
                            ++ptr;
                            if (c > '\r' || c == '\t') continue;
                        } else {
                            Debug.Assert(ptr + strLength <= PtrEnd);
                            if (strLength == 1 || (ptr[1] == pStr[1] &&
                                (strLength == 2 || (ptr[2] == pStr[2] &&
                                 (strLength == 3 || Rest3OfStringEquals(ptr, pStr, strLength))))))
                            {
                                foundString = true;
                                int count = (int)Buffer.PositiveDistance(Ptr, ptr) - nCRLF;
                                Ptr = ptr;
                                if (lineOffset == 0) {
                                    if (count != 0) ++StateTag;
                                    return count;
                                } else {
                                    RegisterNewlines(lineBegin, lineOffset);
                                    return count;
                                }
                            }
                            c = *ptr;
                            if (ptr == end) break;
                            ++ptr;
                            if (c > '\r' || c == '\t') continue;
                        }
                        if (c == '\r') {
                            if (*ptr == '\n') {
                                ++ptr;
                                lineBegin = ptr;
                                ++lineOffset;
                                ++nCRLF;
                                if (end < end1) ++end;
                                else if (ptr > end) break;
                                continue;
                            }
                        } else if (c != '\n') continue;
                        lineBegin = ptr;
                        ++lineOffset;
                    } // for
                    if (ptr < end1) {
                        foundString = false;
                        int count = (int)Buffer.PositiveDistance(Ptr, ptr) - nCRLF;
                        Ptr = ptr;
                        if (lineOffset == 0) {
                            if (count != 0) ++StateTag;
                            return count;
                        } else {
                            RegisterNewlines(lineBegin, lineOffset);
                            return count;
                        }
                    }
                }
            }
            return SkipCharsOrNewlinesUntilStringContinue(ptr, lineBegin, lineOffset, nCRLF, pStr, strLength, maxCount, out foundString);
        }
    }
    private int SkipCharsOrNewlinesUntilStringContinue(
                    char* ptr, char* lineBegin, uint lineOffset, int nCRLF,
                    char* pStr, int strLength, int maxCount, out bool foundString)
    {
        var stateTag = StateTag;
        foundString = false;
        if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char c = SkipAndPeek((uint)index);
        int count = (int)index - nCRLF;
        for (;;) {
            if (c != pStr[0] || !Match(pStr, strLength)) {
                if (c == EOS || count == maxCount) break;
                ++count;
                char c0 = c;
                c = SkipAndPeek();
                if (c0 <= '\r') {
                    if (c0 == '\r') {
                        if (c == '\n') {
                            c = SkipAndPeek();
                        }
                    } else if (c0 != '\n') continue;
                    RegisterNewline();
                }
            } else {
                foundString = true;
                break;
            }
        }
        StateTag = count == 0 ? stateTag : unchecked(stateTag + 1);
        return count;
    }

    public int SkipCharsOrNewlinesUntilString(
                   string str, int maxCount, bool normalizeNewlines,
                   out string skippedCharsIfStringFoundOtherwiseNull)
    {
        int strLength = str.Length; // throws if str is null
        if (strLength == 0) throw new ArgumentException("The string argument is empty.");
        if (maxCount < 0) throw new ArgumentOutOfRangeException("maxCount", "maxCount is negative.");
        fixed (char* pStr = str) {
            char* lineBegin = null;
            uint lineOffset = 0;
            int nCRLF = 0;
            int nCR = 0;
            char* ptr = Ptr;
            if (ptr != null) {
                char* end1 = unchecked(PtrEnd - strLength);
                if (end1 >= ptr && end1 < PtrEnd) {
                    char* end2 = unchecked(ptr + maxCount);
                    char* end = end2 < ptr || end1 <= end2 ? end1 : end2;
                    for (;;) {
                        char c = *ptr;
                        if (c != pStr[0]) {
                            if (ptr == end) break;
                            ++ptr;
                            if (c > '\r' || c == '\t') continue;
                        } else {
                            Debug.Assert(ptr + strLength <= PtrEnd);
                            if (strLength == 1 || (ptr[1] == pStr[1] &&
                                (strLength == 2 || (ptr[2] == pStr[2] &&
                                 (strLength == 3 || Rest3OfStringEquals(ptr, pStr, strLength))))))
                            {
                                char* ptr0 = Ptr;
                                if (ptr != ptr0) {
                                    Ptr = ptr;
                                    int length = (int)Buffer.PositiveDistance(ptr0, ptr);
                                    if (lineOffset == 0) {
                                        if (length != 0) ++StateTag;
                                        skippedCharsIfStringFoundOtherwiseNull = new string(ptr0, 0, length);
                                        return length;
                                    } else {
                                        RegisterNewlines(lineBegin, lineOffset);
                                        skippedCharsIfStringFoundOtherwiseNull = !normalizeNewlines || (nCR | nCRLF) == 0
                                                        ? new string(ptr0, 0, length)
                                                        : Text.CopyWithNormalizedNewlines(ptr0, length, nCRLF, nCR);
                                        return length - nCRLF;
                                    }
                                } else {
                                    skippedCharsIfStringFoundOtherwiseNull = "";
                                    return 0;
                                }
                            }
                            c = *ptr;
                            if (ptr == end) break;
                            ++ptr;
                            if (c > '\r' || c == '\t') continue;
                        }
                        if (c == '\r') {
                            if (*ptr == '\n') {
                                ++ptr;
                                lineBegin = ptr;
                                ++lineOffset;
                                ++nCRLF;
                                if (end < end1) ++end;
                                else if (ptr > end) break;
                                continue;
                            } else {
                                ++nCR;
                            }
                        } else if (c != '\n') continue;
                        lineBegin = ptr;
                        ++lineOffset;
                    } // for
                    if (ptr < end1) {
                        skippedCharsIfStringFoundOtherwiseNull = null;
                        int count = (int)Buffer.PositiveDistance(Ptr, ptr) - nCRLF;
                        Ptr = ptr;
                        if (lineOffset == 0) {
                            if (count != 0) ++StateTag;
                            return count;
                        } else {
                            RegisterNewlines(lineBegin, lineOffset);
                            return count;
                        }
                    }
                }
            }
            return SkipCharsOrNewlinesUntilStringContinue(ptr, lineBegin, lineOffset, nCRLF, nCR, pStr, strLength, maxCount, normalizeNewlines, out skippedCharsIfStringFoundOtherwiseNull);
        }
    }
    private int SkipCharsOrNewlinesUntilStringContinue(
                    char* ptr, char* lineBegin, uint lineOffset, int nCRLF, int nCR,
                    char* pStr, int strLength, int maxCount, bool normalizeNewlines, out string skippedCharsIfStringFoundOtherwiseNull)
    {
        var stateTag = StateTag;
        var indexToken = IndexToken;
        if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        int count = (int)index - nCRLF;
        char c = SkipAndPeek(index);
        for (;;) {
            if (c != pStr[0] || !Match(pStr, strLength)) {
                if (c == EOS || count == maxCount) break;
                ++count;
                char c0 = c;
                c = SkipAndPeek();
                if (c0 <= '\r') {
                    if (c0 == '\r') {
                        if (c == '\n') {
                            c = SkipAndPeek();
                            ++nCRLF;
                        } else {
                            ++nCR;
                        }
                    } else if (c0 != '\n') continue;
                    RegisterNewline();
                }
            } else { // found string
                if (count != 0) {
                    StateTag = unchecked(stateTag + 1);
                    var s = ReadFrom(indexToken);
                    if (!normalizeNewlines || (nCR | nCRLF) == 0) {
                        skippedCharsIfStringFoundOtherwiseNull = s;
                        return count;
                    } else {
                        fixed (char* ps = s)
                        skippedCharsIfStringFoundOtherwiseNull = Text.CopyWithNormalizedNewlines(ps, s.Length, nCRLF, nCR);
                        return count;
                    }
                } else {
                    StateTag = stateTag;
                    skippedCharsIfStringFoundOtherwiseNull = "";
                    return 0;
                }
            }
        }
        StateTag = count == 0 ? stateTag : unchecked(stateTag + 1);
        skippedCharsIfStringFoundOtherwiseNull = null;
        return count;
    }

    public int SkipCharsOrNewlinesUntilCaseFoldedString(
                   string caseFoldedString, int maxCount,
                   out bool foundString)
    {
        int strLength = caseFoldedString.Length; // throws if str is null
        if (strLength == 0) throw new ArgumentException("The string argument is empty.");
        if (maxCount < 0) throw new ArgumentOutOfRangeException("maxCount", "maxCount is negative.");
        char* lineBegin = null;
        fixed (char* pStr = caseFoldedString) {
            uint lineOffset = 0;
            int nCRLF = 0;
            char* ptr = Ptr;
            if (ptr != null) {
                char* bufferEnd = PtrEnd;
                char* end1 = unchecked(bufferEnd - strLength);
                if (end1 >= ptr && end1 < bufferEnd) {
                    char* end2 = unchecked(ptr + maxCount);
                    char* end = end2 < ptr || end1 <= end2 ? end1 : end2;

                    char* cftable = CaseFoldTable.FoldedChars;
                    for (;;) {
                        char c = cftable[*ptr];
                        if (c != pStr[0]) {
                            if (ptr == end) break;
                            ++ptr;
                            if (c > '\r' || c == '\t') continue;
                        } else {
                            Debug.Assert(ptr + strLength <= PtrEnd);
                            if (strLength == 1 || (cftable[ptr[1]] == pStr[1] &&
                                (strLength == 2 || (cftable[ptr[2]] == pStr[2] &&
                                 (strLength == 3 || Rest3OfStringEqualsCaseFolded(ptr, pStr, strLength))))))
                            {
                                foundString = true;
                                int count = (int)Buffer.PositiveDistance(Ptr, ptr) - nCRLF;
                                Ptr = ptr;
                                if (lineOffset == 0) {
                                    if (count != 0) ++StateTag;
                                    return count;
                                } else {
                                    RegisterNewlines(lineBegin, lineOffset);
                                    return count;
                                }
                            }
                            c = *ptr; // we don't need to casefold here
                            if (ptr == end) break;
                            ++ptr;
                            if (c > '\r' || c == '\t') continue;
                        }
                        if (c == '\r') {
                            if (*ptr == '\n') {
                                ++ptr;
                                lineBegin = ptr;
                                ++lineOffset;
                                ++nCRLF;
                                if (end < end1) ++end;
                                else if (ptr > end) break;
                                continue;
                            }
                        } else if (c != '\n') continue;
                        lineBegin = ptr;
                        ++lineOffset;
                    } // for
                    if (ptr < end1) {
                        foundString = false;
                        int count = (int)Buffer.PositiveDistance(Ptr, ptr) - nCRLF;
                        Ptr = ptr;
                        if (lineOffset == 0) {
                            if (count != 0) ++StateTag;
                            return count;
                        } else {
                            RegisterNewlines(lineBegin, lineOffset);
                            return count;
                        }
                    }
                }
            }
            return SkipCharsOrNewlinesUntilCaseFoldedStringContinue(ptr, lineBegin, lineOffset, nCRLF, pStr, strLength, maxCount, out foundString);
        }
    }
    private int SkipCharsOrNewlinesUntilCaseFoldedStringContinue(
                    char* ptr, char* lineBegin, uint lineOffset, int nCRLF,
                    char* pStr, int strLength, int maxCount, out bool foundString)
    {
        var stateTag = StateTag;
        foundString = false;
        if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        char* cftable = CaseFoldTable.FoldedChars;
        char c = cftable[SkipAndPeek((uint)index)];
        int count = (int)index - nCRLF;
        for (;;) {
            if (c != pStr[0] || !MatchCaseFolded(pStr, strLength)) {
                if (c == EOS || count == maxCount) break;
                ++count;
                char c0 = c;
                c = cftable[SkipAndPeek()];
                if (c0 <= '\r') {
                    if (c0 == '\r') {
                        if (c == '\n') {
                            c = cftable[SkipAndPeek()];
                            ++nCRLF;
                        }
                    } else if (c0 != '\n') continue;
                    RegisterNewline();
                }
            } else {
                foundString = true;
                break;
            }
        }
        StateTag = count == 0 ? stateTag : unchecked(stateTag + 1);
        return count;
    }

    public int SkipCharsOrNewlinesUntilCaseFoldedString(
                   string caseFoldedString, int maxCount, bool normalizeNewlines,
                   out string skippedCharsIfStringFoundOtherwiseNull)
    {
        int strLength = caseFoldedString.Length; // throws if str is null
        if (strLength == 0) throw new ArgumentException("The string argument is empty.");
        if (maxCount < 0) throw new ArgumentOutOfRangeException("maxCount", "maxCount is negative.");
        fixed (char* pStr = caseFoldedString) {
            char* lineBegin = null;
            uint lineOffset = 0;
            int nCRLF = 0;
            int nCR = 0;
            char* ptr = Ptr;
            if (ptr != null) {
                char* bufferEnd = PtrEnd;
                char* end1 = unchecked(bufferEnd - strLength);
                if (end1 >= ptr && end1 < bufferEnd) {
                    char* end2 = unchecked(ptr + maxCount);
                    char* end = end2 < ptr || end1 <= end2 ? end1 : end2;
                    char* cftable = CaseFoldTable.FoldedChars;
                    for (;;) {
                        char c = cftable[*ptr];
                        if (c != pStr[0]) {
                            if (ptr == end) break;
                            ++ptr;
                            if (c > '\r' || c == '\t') continue;
                        } else {
                            Debug.Assert(ptr + strLength <= PtrEnd);
                            if (strLength == 1 || (cftable[ptr[1]] == pStr[1] &&
                                (strLength == 2 || (cftable[ptr[2]] == pStr[2] &&
                                 (strLength == 3 || Rest3OfStringEqualsCaseFolded(ptr, pStr, strLength))))))
                            {
                                char* ptr0 = Ptr;
                                if (ptr != ptr0) {
                                    Ptr = ptr;
                                    int length = (int)Buffer.PositiveDistance(ptr0, ptr);
                                    if (lineOffset == 0) {
                                        if (length != 0) ++StateTag;
                                        skippedCharsIfStringFoundOtherwiseNull = new string(ptr0, 0, length);
                                        return length;
                                    } else {
                                        RegisterNewlines(lineBegin, lineOffset);
                                        skippedCharsIfStringFoundOtherwiseNull = !normalizeNewlines || (nCR | nCRLF) == 0
                                                        ? new string(ptr0, 0, length)
                                                        : Text.CopyWithNormalizedNewlines(ptr0, length, nCRLF, nCR);
                                        return length - nCRLF;
                                    }
                                } else {
                                    skippedCharsIfStringFoundOtherwiseNull = "";
                                    return 0;
                                }
                            }
                            c = *ptr; // we don't need to casefold here
                            if (ptr == end) break;
                            ++ptr;
                            if (c > '\r' || c == '\t') continue;
                        }
                        if (c == '\r') {
                            if (*ptr == '\n') {
                                ++ptr;
                                lineBegin = ptr;
                                ++lineOffset;
                                ++nCRLF;
                                if (end < end1) ++end;
                                else if (ptr > end) break;
                                continue;
                            } else {
                                ++nCR;
                            }
                        } else if (c != '\n') continue;
                        lineBegin = ptr;
                        ++lineOffset;
                    } // for
                    if (ptr < end1) {
                        skippedCharsIfStringFoundOtherwiseNull = null;
                        int count = (int)Buffer.PositiveDistance(Ptr, ptr) - nCRLF;
                        Ptr = ptr;
                        if (lineOffset == 0) {
                            if (count != 0) ++StateTag;
                            return count;
                        } else {
                            RegisterNewlines(lineBegin, lineOffset);
                            return count;
                        }
                    }
                }
            }
            return SkipCharsOrNewlinesUntilCaseFoldedStringContinue(ptr, lineBegin, lineOffset, nCRLF, nCR, pStr, strLength, maxCount, normalizeNewlines, out skippedCharsIfStringFoundOtherwiseNull);
        }
    }
    private int SkipCharsOrNewlinesUntilCaseFoldedStringContinue(
                    char* ptr, char* lineBegin, uint lineOffset, int nCRLF, int nCR,
                    char* pStr, int strLength, int maxCount, bool normalizeNewlines, out string skippedCharsIfStringFoundOtherwiseNull)
    {
        var stateTag = StateTag;
        var indexToken = IndexToken;
        if (lineOffset != 0) RegisterNewlines(lineBegin, lineOffset);
        uint index = Buffer.PositiveDistance(Ptr, ptr);
        int count = (int)index - nCRLF;
        char* cftable = CaseFoldTable.FoldedChars;
        char c = cftable[SkipAndPeek(index)];
        for (;;) {
            if (c != pStr[0] || !MatchCaseFolded(pStr, strLength)) {
                if (c == EOS || count == maxCount) break;
                ++count;
                char c0 = c;
                c = cftable[SkipAndPeek()];
                if (c0 <= '\r') {
                    if (c0 == '\r') {
                        if (c == '\n') {
                            c = cftable[SkipAndPeek()];
                            ++nCRLF;
                        } else {
                            ++nCR;
                        }
                    } else if (c0 != '\n') continue;
                    RegisterNewline();
                }
            } else { // found string
                if (count != 0) {
                    StateTag = unchecked(stateTag + 1);
                    var s = ReadFrom(indexToken);
                    if ((nCR | nCRLF) == 0 || !normalizeNewlines) {
                        skippedCharsIfStringFoundOtherwiseNull = s;
                        return count;
                    } else {
                        fixed (char* ps = s)
                        skippedCharsIfStringFoundOtherwiseNull = Text.CopyWithNormalizedNewlines(ps, s.Length, nCRLF, nCR);
                        return count;
                    }
                } else {
                    StateTag = stateTag;
                    skippedCharsIfStringFoundOtherwiseNull = "";
                    return 0;
                }
            }
        }
        StateTag = count == 0 ? stateTag : unchecked(stateTag + 1);
        skippedCharsIfStringFoundOtherwiseNull = null;
        return count;
    }
}

public unsafe struct CharStreamState<TUserState> {
#if DEBUG
    internal readonly CharStream<TUserState> CharStream;
    private long Index { get { return GetIndex(CharStream); } }
#endif
    internal readonly char* Ptr;
    internal readonly int   Block;
#if SMALL_STATETAG
    public   readonly int Tag;
#else
    public   readonly long Tag;
#endif
    public   readonly long  Line;
    public   readonly long  LineBegin;
    public   readonly TUserState UserState;
    public   readonly string Name;

    // Public (though undocumented) as long as the .NET JIT doesn't
    // always inline CharStream<TUserState>.State
    public CharStreamState(CharStream<TUserState> charStream) {
     #if DEBUG
        CharStream = charStream;
     #endif
        Ptr       = charStream.Ptr;
        Block     = charStream.Block;
        Tag       = charStream.StateTag;
        Line      = charStream._Line;
        LineBegin = charStream._LineBegin;
        UserState = charStream._UserState;
        Name      = charStream._Name;
    }

    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
    public CharStreamIndexToken IndexToken { get {
        if (Line <= 0) // tests for a zero-initialized state
            throw new InvalidOperationException("The CharStreamState is invalid.");

        return new CharStreamIndexToken(
               #if DEBUG
                   CharStream,
               #endif
                   Ptr,
                   Block);
    } }

    // On .NET calling an instance method of a generic struct can be more
    // expensive than calling an instance method of a generic class
    // (when the type parameter value is not statically known at the call
    // site and isn't a value type that makes the .NET JIT specialize
    // the code).
    //
    // Moving the actual implementations of the following methods into
    // the CharStream<TUserState> class allows the .NET JIT to inline them,
    // so that we effectively replace struct method calls with cheaper
    // class method calls.

    public long GetIndex(CharStream<TUserState> charStreamFromWhichStateWasRetrieved) {
        return charStreamFromWhichStateWasRetrieved.GetIndex(ref this);
    }

    public Position GetPosition(CharStream<TUserState> charStreamFromWhichStateWasRetrieved) {
        return charStreamFromWhichStateWasRetrieved.GetPosition(ref this);
    }
}

/// <summary>Provides read‐access to a sequence of UTF‐16 chars.</summary>
public unsafe sealed class CharStream<TUserState> : CharStream {
    // we don't have a public constructor that only takes a string to avoid potential confusion with a filepath constructor
    internal CharStream(string chars) : base(chars) { }

    public CharStream(string chars, int index, int length) : base(chars, index, length) {}

    public CharStream(string chars, int index, int length, long streamIndexOffset)
           : base(chars, index, length, streamIndexOffset) { }

    public CharStream(char[] chars, int index, int length) : base(chars, index, length) { }

    public CharStream(char[] chars, int index, int length, long streamIndexOffset)
           : base(chars, index, length, streamIndexOffset) { }

    public CharStream(char* chars, int length) : base(chars, length) { }

    public CharStream(char* chars, int length, long streamIndexOffset)
           : base(chars, length, streamIndexOffset) { }

    internal CharStream(string chars, char* pChars, char* begin, int length)
             : base(chars, pChars, begin, length) { }

    public CharStream(string path, Encoding encoding)
           : base(path, encoding) { }

    public CharStream(string path, Encoding encoding, bool detectEncodingFromByteOrderMarks)
           : base(path, encoding, detectEncodingFromByteOrderMarks) { }

    public CharStream(string path, Encoding encoding, bool detectEncodingFromByteOrderMarks,
                      int blockSize, int blockOverlap, int byteBufferLength)
           : base(path, encoding, detectEncodingFromByteOrderMarks, blockSize, blockOverlap, byteBufferLength) { }

    public CharStream(Stream stream, Encoding encoding)
           : base(stream, encoding) { }

    public CharStream(Stream stream, bool leaveOpen, Encoding encoding)
           : base(stream, leaveOpen, encoding) { }

    public CharStream(Stream stream, bool leaveOpen, Encoding encoding, bool detectEncodingFromByteOrderMarks)
           : base(stream, leaveOpen, encoding, detectEncodingFromByteOrderMarks) { }

    public CharStream(Stream stream, bool leaveOpen,
                      Encoding encoding, bool detectEncodingFromByteOrderMarks,
                      int blockSize, int blockOverlap, int byteBufferLength)
           : base(stream, leaveOpen, encoding, detectEncodingFromByteOrderMarks,
                  blockSize, blockOverlap, byteBufferLength) {}

    internal TUserState _UserState;
    public TUserState UserState {
        get { return _UserState; }
        set { _UserState = value; ++StateTag; }
    }

    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
    public CharStreamState<TUserState> State { get {
        return new CharStreamState<TUserState>(this);
    } }


    // GetIndex and GetPosition are helper methods for CharStreamState

    internal long GetIndex(ref CharStreamState<TUserState> state) {
        if (state.Line <= 0) // tests for a zero-initialized state
            throw new InvalidOperationException("The CharStreamState is invalid.");
    #if DEBUG
        Debug.Assert(this == state.CharStream);
    #endif
        return GetIndex(state.Ptr, state.Block);
    }

    internal Position GetPosition(ref CharStreamState<TUserState> state) {
        if (state.Line <= 0) // tests for a zero-initialized state
            throw new InvalidOperationException("The CharStreamState is invalid.");
    #if DEBUG
        Debug.Assert(this == state.CharStream);
    #endif
        long index = GetIndex(state.Ptr, state.Block);
        return new Position(state.Name, index, state.Line, index - state.LineBegin + 1);
    }

    // Passing a large struct by value is suboptimal, so for optimization purposes
    // we define internal overloads that take ref arguments. Unfortunately, C#/F#
    // doesn't have const-refs, so we can't make these overloads public (at least,
    // not without risking heart attacks within certain user demographics of this library).
    // An alternative would be to move the following methods into the CharStreamState class,
    // but IMHO the resulting API would feel less intuitive and be somewhat less disoverable.

    public void BacktrackTo(CharStreamState<TUserState> state) {
        BacktrackTo(ref state);
    }
    internal void BacktrackTo(ref CharStreamState<TUserState> state) {
        if (state.Line <= 0) // tests for zero-initialized states
            throw new ArgumentException("The CharStreamState is invalid.");
    #if DEBUG
        Debug.Assert(this == state.CharStream);
    #endif
        Seek(state.Ptr, state.Block);
        StateTag   = state.Tag;
        _Line      = state.Line;
        _LineBegin = state.LineBegin;
        _UserState = state.UserState;
        _Name      = state.Name;
    }

    public string ReadFrom(CharStreamState<TUserState> stateWhereStringBegins, bool normalizeNewlines) {
        return ReadFrom(ref stateWhereStringBegins, normalizeNewlines);
    }
    internal string ReadFrom(ref CharStreamState<TUserState> stateWhereStringBegins, bool normalizeNewlines) {
        if (stateWhereStringBegins.Line <= 0) // tests for zero-initialized states
            throw new ArgumentException("The CharStreamState is invalid.");
    #if DEBUG
        Debug.Assert(this == stateWhereStringBegins.CharStream);
    #endif
        string str = ReadFrom(stateWhereStringBegins.Ptr, stateWhereStringBegins.Block);
        if (!normalizeNewlines || _Line == stateWhereStringBegins.Line) return str;
        return Text.NormalizeNewlines(str);
    }

    public CharStream<TSubStreamUserState> CreateSubstream<TSubStreamUserState>(CharStreamState<TUserState> stateWhereSubstreamBegins) {
        return CreateSubstream<TSubStreamUserState>(ref stateWhereSubstreamBegins);
    }
    internal CharStream<TSubStreamUserState> CreateSubstream<TSubStreamUserState>(ref CharStreamState<TUserState> stateWhereSubstreamBegins) {
        if (stateWhereSubstreamBegins.Line <= 0) // tests for zero-initialized states
            throw new ArgumentException("The CharStreamState is invalid.");
    #if DEBUG
        Debug.Assert(this == stateWhereSubstreamBegins.CharStream);
    #endif
        CharStream<TSubStreamUserState> subStream;
        if (IsSingleBlockStream) {
            // the CharStream has only one block, so its safe to
            // construct a new CharStream from a pointer into the original buffer
            char* ptr0 = stateWhereSubstreamBegins.Ptr;
            if (ptr0 == null) ptr0 = BufferEnd;
            char* end = Ptr;
            if (end == null) end = BufferEnd;
            if (end < ptr0) throw new ArgumentException("The current position of the stream must not lie before the position corresponding to the given CharStreamState.");
            int length = (int)Buffer.PositiveDistance(ptr0, end);
            subStream = new CharStream<TSubStreamUserState>(BufferString, BufferStringPointer, ptr0, length);
            var indexOfFirstChar = Buffer.PositiveDistance(BufferBegin, ptr0) + _IndexOfFirstChar;
            subStream.IndexOfFirstCharInBlock = indexOfFirstChar;
            subStream._IndexOfFirstChar = indexOfFirstChar;
        } else if (Block == stateWhereSubstreamBegins.Block && Ptr != null && stateWhereSubstreamBegins.Ptr != null) {
            char* ptr0 = stateWhereSubstreamBegins.Ptr;
            char* end = Ptr;
            if (end < ptr0) throw new ArgumentException("The current position of the stream must not lie before the position corresponding to the given CharStreamState.");
            int length = (int)Buffer.PositiveDistance(ptr0, end);
            string subString = new String(ptr0, 0, length);
            subStream = new CharStream<TSubStreamUserState>(subString);
            var indexOfFirstChar = Buffer.PositiveDistance(BufferBegin, ptr0) + _IndexOfFirstChar;
            subStream.IndexOfFirstCharInBlock = indexOfFirstChar;
            subStream._IndexOfFirstChar = indexOfFirstChar;
        } else {
            var subString = ReadFrom(ref stateWhereSubstreamBegins, false);
            subStream = new CharStream<TSubStreamUserState>(subString);
            var indexOfFirstChar = GetIndex(stateWhereSubstreamBegins.Ptr, stateWhereSubstreamBegins.Block);
            subStream.IndexOfFirstCharInBlock = indexOfFirstChar;
            subStream._IndexOfFirstChar = indexOfFirstChar;
        }
        subStream.StateTag = stateWhereSubstreamBegins.Tag;
        subStream._Line = stateWhereSubstreamBegins.Line;
        subStream._LineBegin = stateWhereSubstreamBegins.LineBegin;
        subStream._Name = stateWhereSubstreamBegins.Name;
    #if DEBUG
        ++SubstreamCount.Value;
        subStream.ParentSubstreamCount = SubstreamCount;
    #endif
        return subStream;
    }
}

}

#endif // !LOW_TRUST


<|start_filename|>ParcelTest/RangeTests.cs<|end_filename|>
﻿using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using PairList = Microsoft.FSharp.Collections.FSharpList<System.Tuple<AST.Address, AST.Address>>;
using AddrPair = System.Tuple<AST.Address, AST.Address>;

namespace ParcelTest
{
    [TestClass]
    public class RangeTests
    {
        [TestMethod]
        public void RangeCase1Test()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            String s = "A1:A2";

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct =
                new AST.ReferenceRange(
                    e,
                    new AST.Range(
                        AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                        AST.Address.fromA1withMode(2, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                    )
                );
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void RangeCase2Test()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            String s = "A1,A2";

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct =
                new AST.ReferenceRange(
                    e,
                    new AST.Range(
                        new AST.Range(
                            AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                            AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                        ),
                        new AST.Range(
                            AST.Address.fromA1withMode(2, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                            AST.Address.fromA1withMode(2, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                        )
                    )
                );
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void RangeCase3Test()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            String s = "A1,A2:A3";

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct =
                new AST.ReferenceRange(
                    e,
                    new AST.Range(
                        new AST.Range(
                            AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                            AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                        ),
                        new AST.Range(
                            AST.Address.fromA1withMode(2, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                            AST.Address.fromA1withMode(3, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                        )
                    )
                );
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void RangeCase4Test()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            String s = "A1:A2,A3";

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct =
                new AST.ReferenceRange(
                    e,
                    new AST.Range(
                        new AST.Range(
                            AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                            AST.Address.fromA1withMode(2, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                        ),
                        new AST.Range(
                            AST.Address.fromA1withMode(3, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                            AST.Address.fromA1withMode(3, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                        )
                    )
                );
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void RangeCase5Test()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            String s = "A1:A2,A3:A4";

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct =
                new AST.ReferenceRange(
                    e,
                    new AST.Range(
                        new AST.Range(
                            AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                            AST.Address.fromA1withMode(2, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                        ),
                        new AST.Range(
                            AST.Address.fromA1withMode(3, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                            AST.Address.fromA1withMode(4, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                        )
                    )
                );
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void RangeCase6Test()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            String s = "A1,A2,A3,A4,A5";

            AddrPair[] addrpairs = {
                                    new AddrPair(
                                        AST.Address.fromA1withMode(5, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                                        AST.Address.fromA1withMode(5, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                                    ),
                                    new AddrPair(
                                        AST.Address.fromA1withMode(4, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                                        AST.Address.fromA1withMode(4, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                                    ),
                                    new AddrPair(
                                        AST.Address.fromA1withMode(3, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                                        AST.Address.fromA1withMode(3, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                                    ),
                                    new AddrPair(
                                        AST.Address.fromA1withMode(2, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                                        AST.Address.fromA1withMode(2, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                                    ),
                                    new AddrPair(
                                        AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                                        AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                                    )
                                   };

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct = new AST.ReferenceRange(e, new AST.Range(addrpairs));
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void RangeCase7Test()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            String s = "A1,A2:A3,A4";

            AddrPair[] addrpairs = {
                                    new AddrPair(
                                        AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                                        AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                                    ),
                                    new AddrPair(
                                        AST.Address.fromA1withMode(2, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                                        AST.Address.fromA1withMode(3, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                                    ),
                                    new AddrPair(
                                        AST.Address.fromA1withMode(4, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path),
                                        AST.Address.fromA1withMode(4, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path)
                                    )
                                   };

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct = new AST.ReferenceRange(e, new AST.Range(addrpairs));
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void rangeCase8Test()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);

            String s = "A1:B1";

            AST.Range range = new AST.Range(
                AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorkbookName, e.WorkbookName, e.Path),
                AST.Address.fromA1withMode(1, "B", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorkbookName, e.WorkbookName, e.Path)
                );

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct = new AST.ReferenceRange(e, range);
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void absoluteRangeTest()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);

            String s = "$A$1:$B$1";

            AST.Range range = new AST.Range(
                AST.Address.fromA1withMode(1, "A", AST.AddressMode.Absolute, AST.AddressMode.Absolute, e.WorkbookName, e.WorkbookName, e.Path),
                AST.Address.fromA1withMode(1, "B", AST.AddressMode.Absolute, AST.AddressMode.Absolute, e.WorkbookName, e.WorkbookName, e.Path)
                );

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct = new AST.ReferenceRange(e, range);
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void mixedRangeTest1()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);

            String s = "$A1:$B1";

            AST.Range range = new AST.Range(
                AST.Address.fromA1withMode(1, "A", AST.AddressMode.Relative, AST.AddressMode.Absolute, e.WorkbookName, e.WorkbookName, e.Path),
                AST.Address.fromA1withMode(1, "B", AST.AddressMode.Relative, AST.AddressMode.Absolute, e.WorkbookName, e.WorkbookName, e.Path)
                );

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct = new AST.ReferenceRange(e, range);
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void mixedRangeTest2()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);

            String s = "A$1:B$1";

            AST.Range range = new AST.Range(
                AST.Address.fromA1withMode(1, "A", AST.AddressMode.Absolute, AST.AddressMode.Relative, e.WorkbookName, e.WorkbookName, e.Path),
                AST.Address.fromA1withMode(1, "B", AST.AddressMode.Absolute, AST.AddressMode.Relative, e.WorkbookName, e.WorkbookName, e.Path)
                );

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct = new AST.ReferenceRange(e, range);
            Assert.AreEqual(r, correct);
        }
    }
}


<|start_filename|>Parcel/AST.fs<|end_filename|>
﻿module AST
    open System

    type ParseException(formula: string, reason: string) =
        inherit System.Exception(formula)
        new(formula: string) = ParseException(formula, "")

    type IndirectAddressingNotSupportedException(expression: string) =
        inherit Exception(expression)

    // The Env object is threaded through grammar combinators (via UserState)
    // so that parsers always have the path, workbook name, and worksheet name
    // from the calling workbook environment in order to construct fully-
    // qualified AST nodes.
    [<Serializable>]
    type Env(path: string, wbname: string, wsname: string) =
        member self.Path = if path.EndsWith @"\" then path else path + @"\"
        member self.WorkbookName = wbname
        member self.WorksheetName = wsname
        override self.GetHashCode() : int =
            path.GetHashCode() |||
            wbname.GetHashCode() |||
            wsname.GetHashCode()
        override self.Equals(obj: obj) : bool =
            let d2 = obj :?> Env
            path = d2.Path &&
            wbname = d2.WorkbookName &&
            wsname = d2.WorksheetName

    [<Serializable>]
    type AddressMode =
    | Absolute
    | Relative

    [<Serializable>]
    type Address(R: int, C: int, Rmode: AddressMode, Cmode: AddressMode, env: Env) =
        let mutable _cachedHC : int option = None

        new(R: int, C: int, env: Env) = Address(R, C, Relative, Relative, env)

        interface IComparable with
            member self.CompareTo(obj) =
                let addr = obj :?> Address
                match env.WorksheetName.CompareTo(addr.WorksheetName) with
                | -1 -> -1
                | 1  -> 1
                | _  ->
                    // this guarantees a consistent total order, but if used in user-facing
                    // contexts, it might not be very intuitive to end-users.
                    let c = Hash.cantorPair (System.Convert.ToUInt32 R) (System.Convert.ToUInt32 R) System.UInt32.MaxValue
                    let c' = Hash.cantorPair (System.Convert.ToUInt32 addr.Row) (System.Convert.ToUInt32 addr.Col) System.UInt32.MaxValue
                    c.CompareTo c'

        static member fromR1C1withMode(R: int, C: int, rmode: AddressMode, cmode: AddressMode, wsname: string, wbname: string, path: string) : Address =
            Address(R, C, rmode, cmode, Env(path, wbname, wsname))
        static member fromA1withMode(row: int, col: string, rmode: AddressMode, cmode: AddressMode, wsname: string, wbname: string, path: string) : Address =
            Address(row, Address.CharColToInt(col), rmode, cmode, Env(path, wbname, wsname))
        member self.copyWithNewEnv(envnew: Env) =
            Address(R, C, Rmode, Cmode, envnew)
        member self.A1Local() : string =
            let cmode =
                match self.ColMode with
                | AddressMode.Absolute -> "$"
                | AddressMode.Relative -> ""
            let rmode =
                match self.RowMode with
                | AddressMode.Absolute -> "$"
                | AddressMode.Relative -> ""
            cmode + Address.IntToColChars(self.X) + rmode + self.Y.ToString()
        member self.A1Path() : string = env.Path
        member self.A1Worksheet() : string = env.WorksheetName
        member self.A1Workbook() : string = env.WorkbookName
        member self.A1FullyQualified() : string =
            "'" + self.A1Path() + "[" + self.A1Workbook() + "]" + self.A1Worksheet() + "'!" + self.A1Local()
        member self.R1C1 =
            let wsstr = env.WorksheetName + "!"
            let wbstr = "[" + env.WorkbookName + "]"
            let pstr = env.Path
            pstr + wbstr + wsstr + "R" + R.ToString() + "C" + C.ToString()
        member self.XMode = Cmode
        member self.YMode = Rmode
        member self.X: int = C
        member self.Y: int = R
        member self.RowMode = Rmode
        member self.ColMode = Cmode
        member self.Row = R
        member self.Col = C
        member self.Path = env.Path
        member self.WorksheetName = env.WorksheetName
        member self.WorkbookName = env.WorkbookName
        static member addressHash(R: int)(C: int)(sheetname: string) : int =
            // 2^m = w = expected maximum number of worksheets
            // the following is Cantor's pairing function mod w;
            // for signed int, that gives us 2^(32 - m) cells per
            // sheet without collisions.
            assert(R >= 0)
            assert(C >= 0)
            let m = 4u
            let w = 2u <<< int32 (m - 1u) // there are rarely more than 16 worksheets
            let r = (2u <<< int32 (32u - m - 1u)) // the total number of distinct integers allowed from pi
            let k_1 = System.Convert.ToUInt32(R)
            let k_2 = System.Convert.ToUInt32(C)
            // get uint from pairing function
            let pi = Hash.cantorPair k_1 k_2 r
            // shift pi depending on the worksheet hashcode
            // (we don't have access to the real index here; getting the
            //  sheetname's hashcode mod w is an approximation)
            let sheet_hc = uint32 (sheetname.GetHashCode()) % w
            let hashcode = pi + sheet_hc * r
            int32 hashcode // this casts; it does not convert
        // necessary because Address is used as a Dictionary key
        override self.GetHashCode() : int =
            match _cachedHC with
            | Some(hc) -> hc
            | None ->
                let hc = Address.addressHash R C self.WorksheetName
                _cachedHC <- Some(hc)
                hc
        override self.Equals(obj: obj) : bool =
            let addr = obj :?> Address
            self.SameAs addr
        member self.SameAs(addr: Address) : bool =
            // odd construction is for breakpoint-friendliness
            let a = self.X = addr.X
            let b = self.Y = addr.Y
            let c = self.WorksheetName = addr.WorksheetName
            let d = self.WorkbookName = addr.WorkbookName
            a && b && c && d
        override self.ToString() =
            "(" + self.X.ToString() + "," + self.Y.ToString() + ")"
        member self.ToFormula = self.A1FullyQualified()
        static member CharColToInt(col: string) : int =
            let rec ccti(idx: int) : int =
                let ltr = (int col.[idx]) - 64
                let num = (int (Math.Pow(26.0, float (col.Length - idx - 1)))) * ltr
                if idx = 0 then
                    num
                else
                    num + ccti(idx - 1)
            ccti(col.Length - 1)
        static member FromA1String(addr: string, wsname: string, wbname: string, path: string) : Address =
            let reg = System.Text.RegularExpressions.Regex("(?<cmode>\$?)(?<column>[A-Z]+)(?<rmode>\$?)(?<row>[0-9]+)")
            let m = reg.Match(addr)
            let r = System.Convert.ToInt32(m.Groups.["row"].Value)
            let c = Address.CharColToInt(m.Groups.["column"].Value)
            let rmode = if String.IsNullOrEmpty(m.Groups.["rmode"].Value) then AddressMode.Relative else AddressMode.Absolute
            let cmode = if String.IsNullOrEmpty(m.Groups.["cmode"].Value) then AddressMode.Relative else AddressMode.Absolute
            Address.fromR1C1withMode(r, c, rmode, cmode, wsname, wbname, path)
        static member FromA1StringForceMode(addr: string, rmode: AddressMode, cmode: AddressMode, wsname: string, wbname: string, path: string) : Address =
            let reg = System.Text.RegularExpressions.Regex("\$?(?<column>[A-Z]+)\$?(?<row>[0-9]+)")
            let m = reg.Match(addr)
            let r = System.Convert.ToInt32(m.Groups.["row"].Value)
            let c = Address.CharColToInt(m.Groups.["column"].Value)
            Address.fromR1C1withMode(r, c, rmode, cmode, wsname, wbname, path)
        static member FromString(addr: string, wsname: string, wbname: string, path: string) : Address =
            let reg = System.Text.RegularExpressions.Regex("R(?<row>[0-9]+)C(?<column>[0-9]+)")
            let m = reg.Match(addr)
            let r = System.Convert.ToInt32(m.Groups.["row"].Value)
            let c = System.Convert.ToInt32(m.Groups.["column"].Value)
            Address.fromR1C1withMode(r, c, AddressMode.Relative, AddressMode.Relative, wsname, wbname, path)
        static member IntToColChars(dividend: int) : string =
            let mutable quot = dividend / 26
            let rem = dividend % 26
            if rem = 0 then
                quot <- quot - 1
            let ltr = if rem = 0 then
                        'Z'
                      else
                        char (64 + rem)
            if quot = 0 then
                ltr.ToString()
            else
                Address.IntToColChars(quot) + ltr.ToString()

    and IndirectAddress(expr: string, env: Env) =
        inherit Address(0,0,env)
        do
            // indirect references are essentially lambdas for
            // constructing addresses
            raise(IndirectAddressingNotSupportedException(expr))

    // note that regions may overlap; be careful not to double-count cells!
    // Default constructor -- each address-address pair corresponds to top-left
    // and bottom right of a region
    and Range(regions: (Address * Address) list) =
        let _regions = regions
        new(regions: (Address * Address)[]) = Range(List.ofArray regions)
        new(range1: Range, range2: Range) = Range(range1.Ranges() @ range2.Ranges())
     
        new(addr1: Address, addr2: Address) = Range([(addr1,addr2)])
        override self.ToString() =
            let sregs = List.map (fun (tl: Address, br: Address) ->
                tl.ToString() + ":" + br.ToString()) _regions
            "List(" + String.Join(",", sregs) + ")"
        member self.copyWithNewEnv(envnew: Env) =
            Range(List.map (fun (tl: Address, br: Address) ->
                    tl.copyWithNewEnv(envnew), br.copyWithNewEnv(envnew)) _regions
                 )
        member self.A1Local() : string =
            let sregs = List.map (fun (tl: Address, br: Address) ->
                            tl.A1Local() + ":" + br.A1Local()
                        ) _regions
            String.Join(",", sregs)
        member self.GetWorksheetName() : string =
            let names = List.fold (fun wss (tl: Address, br: Address) ->
                                    tl.WorksheetName :: br.WorksheetName :: wss
                                  ) [] _regions |>
                        List.toSeq |>
                        Seq.distinct
            assert(Seq.length names = 1)
            Seq.head names
        member self.GetWorkbookName() : string =
            // I am fairly certain that cross-workbook / cross-worksheet
            // ranges are not possible; just return the first
            // path but check that this is true
            let names = List.fold (fun wss (tl: Address, br: Address) ->
                                    tl.WorkbookName :: br.WorkbookName :: wss
                                  ) [] _regions |>
                        List.toSeq |>
                        Seq.distinct
            assert(Seq.length names = 1)
            Seq.head names
        member self.GetPathName() : string =
            // I am fairly certain that cross-workbook / cross-worksheet
            // ranges are not possible; just return the first
            // path but check that this is true
            let paths = List.fold (fun wss (tl: Address, br: Address) ->
                                    tl.Path :: br.Path :: wss
                                  ) [] _regions |>
                        List.toSeq |>
                        Seq.distinct
            assert(Seq.length paths = 1)
            Seq.head paths
        member self.Ranges() : (Address*Address) list = _regions
        // regions may overlap; thus we only return distinct cell addresses
        member self.Addresses() : Address[] =
            // for every contigious region
            let xs =
                List.map (fun (tl: Address, br: Address) ->
                    // for every column in that region
                    Array.map (fun c ->
                        // and every row in that region
                        Array.map (fun r ->
                            // get the address of the cell contained
                            Address.fromR1C1withMode(r, c, tl.RowMode, tl.ColMode, tl.WorksheetName, tl.WorkbookName, tl.Path)
                        ) [|tl.Y..br.Y|] |>
                        Array.toList
                    ) [|tl.X..br.X|] |>
                    Array.toList |>
                    List.concat
                ) _regions |>
                List.concat
            
            // only enumerate overlapping cells once
            let dist = xs |> List.distinct
            let xsa = dist |> Array.ofList

            // sort by leftmost-topmost
            let sorted =
                xsa |>
                Array.sortWith (fun a b ->
                    if a.Row < b.Row then
                        -1
                    else if a.Row = b.Row && a.Col < b.Col then
                        -1
                    else if a.Row = b.Row && a.Col = b.Col then
                        0
                    else
                        1
                   )
            sorted

        static member addrsInRegion(lt: int*int)(rb: int*int) : (int*int)[] =
            let lt_x = fst lt
            let lt_y = snd lt
            let rb_x = fst rb
            let rb_y = snd rb

            // for every contigious region
            // for every column in that region
            Array.map (fun c ->
                // and every row in that region
                Array.map (fun r -> (c, r)
                ) [|lt_y..rb_y|]
            ) [|lt_x..rb_x|] |>
            Array.concat
            
        override self.GetHashCode() : int =
            Hash.jenkinsOneAtATimeHash _regions
        override self.Equals(obj: obj) : bool =
            let r = obj :?> Range
            let r_set = Set.ofArray(r.Addresses())
            let self_set = Set.ofArray(self.Addresses())
            self_set = r_set
        member self.Equals2(obj: obj) : bool =
            let r = obj :?> Range
            failwith "no"
        member self.ToFormula : string =
            let subrngs = List.map (fun (a1: Address,a2: Address) ->
                              a1.A1FullyQualified() + ":" + a2.A1Local()
                          ) _regions
            String.Join(",", subrngs)

    type ReferenceType =
    | ReferenceAddress  = 0
    | ReferenceRange    = 1
    | ReferenceFunction = 2
    | ReferenceConstant = 3
    | ReferenceString   = 4
    | ReferenceNamed    = 5
    | ReferenceBoolean  = 6

    type Arity =
    | Fixed of int
    | LowBound of int
    | VarArgs

    [<AbstractClass>]
    type Reference(env: Env) =
        abstract member Type: ReferenceType
        abstract member ToFormula: string
        member self.Path = env.Path
        member self.WorkbookName = env.WorkbookName
        member self.WorksheetName = env.WorksheetName
        member self.Environment = env

    and ReferenceRange(env: Env, rng: Range) =
        inherit Reference(env)
        let _rng = rng.copyWithNewEnv(env)

        override self.Type = ReferenceType.ReferenceRange
        override self.ToString() =
            "ReferenceRange(" + env.Path + ",[" + env.WorkbookName + "]," + env.WorksheetName + "," + _rng.ToString() + ")"
        member self.Range = _rng
        override self.Equals(obj: obj) : bool =
            let rr = obj :?> ReferenceRange
            self.Path = rr.Path &&
            self.WorkbookName = rr.WorkbookName &&
            self.WorksheetName = rr.WorksheetName &&
            self.Range = rr.Range
        override self.GetHashCode() : int =
            env.GetHashCode() ||| rng.GetHashCode()
        override self.ToFormula : string = rng.ToFormula

    and ReferenceAddress(env: Env, addr: Address) =
        inherit Reference(env)
        let _addr = addr.copyWithNewEnv(env)
        override self.Type = ReferenceType.ReferenceAddress
        override self.ToString() =
            "ReferenceAddress(" + env.Path + ",[" + env.WorkbookName + "]," + env.WorksheetName + "," + _addr.ToString() + ")"
        member self.Address = _addr
        override self.Equals(obj: obj) : bool =
            let ra = obj :?> ReferenceAddress
            self.Path = ra.Path &&
            self.WorkbookName = ra.WorkbookName &&
            self.WorksheetName = ra.WorksheetName &&
            self.Address = ra.Address
        override self.GetHashCode() : int =
            env.GetHashCode() ||| addr.GetHashCode()
        override self.ToFormula = addr.ToFormula

    and ReferenceFunction(env: Env, fnname: string, arglist: Expression list, arity: Arity) =
        inherit Reference(env)
        override self.Type = ReferenceType.ReferenceFunction
        member self.ArgumentList = arglist
        member self.FunctionName = fnname.ToUpper()
        member self.Arity = arity
        override self.ToString() =
            match arity with
            | Fixed a -> self.FunctionName + "[function" + a.ToString() + "](" + String.Join(",", (List.map (fun arg -> arg.ToString()) arglist)) + ")"
            | LowBound aplus -> self.FunctionName + "[function" + aplus.ToString() + "+](" + String.Join(",", (List.map (fun arg -> arg.ToString()) arglist)) + ")"
            | VarArgs -> self.FunctionName + "[functionVarArgs](" + String.Join(",", (List.map (fun arg -> arg.ToString()) arglist)) + ")"
        override self.Equals(obj: obj) : bool =
            let rf = obj :?> ReferenceFunction
            let arglists = List.zip arglist rf.ArgumentList
            self.Path = rf.Path &&
            self.WorkbookName = rf.WorkbookName &&
            self.WorksheetName = rf.WorksheetName &&
            self.FunctionName = rf.FunctionName &&
            // and recursively compare arglists
            List.fold (fun acc (ethis, ethat) -> acc && ethis = ethat) true arglists
        override self.GetHashCode() : int =
            env.GetHashCode() ||| fnname.GetHashCode() ||| arglist.GetHashCode()
        override self.ToFormula = self.FunctionName +
                                  "(" + String.Join(",", (List.map (fun (arg: Expression) -> arg.ToFormula) arglist)) + ")"

    and ReferenceConstant(env: Env, value: double) =
        inherit Reference(env)
        override self.Type = ReferenceType.ReferenceConstant
        member self.Value = value
        override self.ToString() = "Constant(" + value.ToString() + ")"
        override self.Equals(obj: obj) : bool =
            let rc = obj :?> ReferenceConstant
            self.Path = rc.Path &&
            self.WorkbookName = rc.WorkbookName &&
            self.WorksheetName = rc.WorksheetName &&
            self.Value = rc.Value
        override self.GetHashCode() : int =
            env.GetHashCode() ||| value.GetHashCode()
        override self.ToFormula = value.ToString()

    and ReferenceBoolean(env: Env, value: bool) =
        inherit Reference(env)
        override self.Type = ReferenceType.ReferenceBoolean
        member self.Value = value
        override self.ToString() = "Boolean(" + value.ToString() + ")"
        override self.Equals(obj: obj) : bool =
            let rc = obj :?> ReferenceBoolean
            self.Path = rc.Path &&
            self.WorkbookName = rc.WorkbookName &&
            self.WorksheetName = rc.WorksheetName &&
            self.Value = rc.Value
        override self.GetHashCode() : int =
            env.GetHashCode() ||| value.GetHashCode()
        override self.ToFormula = value.ToString()

    and ReferenceString(env: Env, value: string) =
        inherit Reference(env)
        override self.Type = ReferenceType.ReferenceString
        member self.Value = value
        override self.ToString() = "String(" + value + ")"
        override self.Equals(obj: obj) : bool =
            let rs = obj :?> ReferenceString
            self.Path = rs.Path &&
            self.WorkbookName = rs.WorkbookName &&
            self.WorksheetName = rs.WorksheetName &&
            self.Value = rs.Value
        override self.GetHashCode() : int =
            env.GetHashCode() ||| value.GetHashCode()
        override self.ToFormula = value

    and ReferenceNamed(env: Env, varname: string) =
        inherit Reference(env)
        override self.Type = ReferenceType.ReferenceNamed
        member self.Name = varname
        override self.ToString() = "ReferenceName(" + varname + ")"
        override self.Equals(obj: obj) : bool =
            let rn = obj :?> ReferenceNamed
            self.Path = rn.Path &&
            self.WorkbookName = rn.WorkbookName &&
            self.WorksheetName = rn.WorksheetName &&
            self.Name = rn.Name
        override self.GetHashCode() : int =
            env.GetHashCode() ||| varname.GetHashCode()
        override self.ToFormula = varname

    and ReferenceUnion(env: Env, refs: Expression list) =
        inherit Reference(env)
        override self.Type = ReferenceType.ReferenceNamed
        member self.Name = "union"
        member self.References = refs
        override self.ToString() = "ReferenceUnion(" + String.Join(",", refs) + ")"
        override self.Equals(obj: obj) : bool =
            match obj with
            | :? ReferenceUnion as ru ->
                List.zip refs ru.References
                |> List.fold (fun acc (r1,r2) ->
                      acc && r1 = r2
                   ) true
            | _ -> false
        override self.GetHashCode() : int =
            refs |> List.fold (fun acc r -> acc ||| r.GetHashCode()) (env.GetHashCode())
        override self.ToFormula = "(" + String.Join(",", refs) + ")"

    // TODO: implement .Equals!
    and Expression =
    | ReferenceExpr of Reference
    | BinOpExpr of string * Expression * Expression
    | UnaryOpExpr of char * Expression
    | ParensExpr of Expression
        override self.ToString() =
            match self with
            | ReferenceExpr(r) -> "ReferenceExpr(" + r.ToString() + ")"
            | BinOpExpr(op,e1,e2) -> "BinOpExpr(" + op.ToString() + "," + e1.ToString() + "," + e2.ToString() + ")"
            | UnaryOpExpr(op, e) -> "UnaryOpExpr(" + op.ToString() + "," + e.ToString() + ")"
            | ParensExpr(e) -> "ParensExpr(" + e.ToString() + ")"
        member self.ToFormula =
            match self with
            | ReferenceExpr(r) -> r.ToFormula
            | BinOpExpr(op,e1,e2) -> e1.ToFormula + op + e2.ToFormula
            | UnaryOpExpr(op, e) -> op.ToString() + e.ToFormula
            | ParensExpr(e) -> "(" + e.ToFormula + ")"
        member self.WellFormedFormula = "=" + self.ToFormula

<|start_filename|>ParcelTest/MockWorkbook.cs<|end_filename|>
﻿using System;
using System.Linq;
using FSCore = Microsoft.FSharp.Core;

namespace ParcelTest
{
    public class MockWorkbook
    {
        private string _path;
        private string _workbook_name;
        private string[] _sheet_names;

        public MockWorkbook(string path, string workbook_name, string[] sheet_names)
        {
            _path = path;
            _workbook_name = workbook_name;
            // mimic an Excel one-based array
            _sheet_names = (new string[] { "inaccessible" }).Concat(sheet_names).ToArray();
        }
        public AST.Env envForSheet(int idx)
        {
            return new AST.Env(_path, _workbook_name, _sheet_names[idx]);
        }
        public string Path { 
            get { return _path; }
        }
        public string WorkbookName
        {
            get { return _workbook_name; }
        }
        public string worksheetName(int idx) {
            return _sheet_names[idx];
        }
        public static int testGetRanges(string formula)
        {
            // mock workbook object
            var mwb = standardMockWorkbook();
            var ranges = Parcel.rangeReferencesFromFormula(
                formula,
                mwb.Path,
                mwb.WorkbookName,
                mwb.worksheetName(1),
                false
                );

            return ranges.Count();
        }

        public static MockWorkbook standardMockWorkbook()
        {
            return new MockWorkbook(
                System.IO.Directory.GetCurrentDirectory(),
                "MockWorkbook.xls",
                new [] { "sheet1", "sheet2", "sheet3" }
                );
        }
    }
}


<|start_filename|>Parcel/Grammar.fs<|end_filename|>
﻿module Grammar
    open FParsec
    open AST
    open System.Text.RegularExpressions

    let DEBUG_MODE = false
    let NO_OPTIMIZATIONS = false

    //#region OPTIMIZED_IMPLEMENTATIONS
    // this is an optimized "choice" parser that disables
    // error message logging when not in debug mode
    let (<||>) (p1: Parser<'a,'u>) (p2: Parser<'a,'u>) : Parser<'a,'u> =
        if NO_OPTIMIZATIONS then
            fun stream ->
                let mutable stateTag = stream.StateTag
                let mutable reply = p1 stream
                if reply.Status = Error && stateTag = stream.StateTag then
                    let error = reply.Error
                    reply <- p2 stream
                    if stateTag = stream.StateTag then
                        reply.Error <- mergeErrors reply.Error error
                reply
        else
            fun stream ->
                let mutable stateTag = stream.StateTag
                let mutable reply = p1 stream
                if reply.Status = Error && stateTag = stream.StateTag then
                    reply <- p2 stream
                reply

    // this is an optimized "attempt" parser that disables
    // error message logging when not in debug mode
    let attempt_opt (p: Parser<'a,'u>) : Parser<'a,'u> =
        if NO_OPTIMIZATIONS then
            fun stream ->
                // state is only declared mutable so it can be passed by ref, it won't be mutated
                let mutable state = CharStreamState(stream) // = stream.State (manually inlined)
                let mutable reply = p stream
                if reply.Status <> Ok then
                    if state.Tag <> stream.StateTag then
                        reply.Error  <- nestedError stream reply.Error
                        reply.Status <- Error // turns FatalErrors into Errors
                        stream.BacktrackTo(&state) // passed by ref as a (slight) optimization
                    elif reply.Status = FatalError then
                        reply.Status <- Error
                reply
        else
            fun stream ->
                // state is only declared mutable so it can be passed by ref, it won't be mutated
                let mutable state = CharStreamState(stream) // = stream.State (manually inlined)
                let mutable reply = p stream
                if reply.Status <> Ok then
                    if state.Tag <> stream.StateTag then
                        reply.Status <- Error // turns FatalErrors into Errors
                        stream.BacktrackTo(&state) // passed by ref as a (slight) optimization
                    elif reply.Status = FatalError then
                        reply.Status <- Error
                reply

    // this is an optimized "choice" parser that disables
    // error message logging when not in debug mode
    let choice_opt (ps: seq<Parser<'a,'u>>)  =
        if NO_OPTIMIZATIONS then
            match ps with
            | :? (Parser<'a,'u>[]) as ps ->
                if ps.Length = 0 then pzero
                else
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable error = NoErrorMessages
                        let mutable reply = ps.[0] stream
                        let mutable i = 1
                        while reply.Status = Error && stateTag = stream.StateTag && i < ps.Length do
                            error <- mergeErrors error reply.Error
                            reply <- ps.[i] stream
                            i <- i + 1
                        if stateTag = stream.StateTag then
                            error <- mergeErrors error reply.Error
                            reply.Error <- error
                        reply
            | :? (Parser<'a,'u> list) as ps ->
                match ps with
                | [] -> pzero
                | hd::tl ->
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable error = NoErrorMessages
                        let mutable hd, tl = hd, tl
                        let mutable reply = hd stream
                        while reply.Status = Error && stateTag = stream.StateTag
                              && (match tl with
                                  | h::t -> hd <- h; tl <- t; true
                                  | _ -> false)
                           do
                            error <- mergeErrors error reply.Error
                            reply <- hd stream
                        if stateTag = stream.StateTag then
                            error <- mergeErrors error reply.Error
                            reply.Error <- error
                        reply
            | _ -> fun stream ->
                       use iter = ps.GetEnumerator()
                       if iter.MoveNext() then
                           let stateTag = stream.StateTag
                           let mutable error = NoErrorMessages
                           let mutable reply = iter.Current stream
                           while reply.Status = Error && stateTag = stream.StateTag && iter.MoveNext() do
                               error <- mergeErrors error reply.Error
                               reply <- iter.Current stream
                           if stateTag = stream.StateTag then
                               error <- mergeErrors error reply.Error
                               reply.Error <- error
                           reply
                       else
                           Reply()
        else
            match ps with
            | :? (Parser<'a,'u>[]) as ps ->
                if ps.Length = 0 then pzero
                else
                    fun stream ->
                        let stateTag = stream.StateTag
                        let mutable reply = ps.[0] stream
                        let mutable i = 1
                        while reply.Status = Error && stateTag = stream.StateTag && i < ps.Length do
                            reply <- ps.[i] stream
                            i <- i + 1
                        reply
            | :? (Parser<'a,'u> list) as ps ->
                match ps with
                | [] -> pzero
                | hd::tl ->
                    fun stream ->
                        let mutable hd, tl = hd, tl
                        let mutable reply = hd stream
                        while reply.Status = Error && stream.StateTag = stream.StateTag
                              && (match tl with
                                  | h::t -> hd <- h; tl <- t; true
                                  | _ -> false)
                           do
                            reply <- hd stream
                        reply
            | _ -> fun stream ->
                       use iter = ps.GetEnumerator()
                       if iter.MoveNext() then
                           let mutable reply = iter.Current stream
                           while reply.Status = Error && stream.StateTag = stream.StateTag && iter.MoveNext() do
                               reply <- iter.Current stream
                           reply
                       else
                           Reply()

    //#endregion OPTIMIZED_IMPLEMENTATIONS

    let (<!>) (p: Parser<_,_>) label : Parser<_,_> =
        if DEBUG_MODE then
            fun stream ->
                let s = String.replicate (int (stream.Position.Column)) " "
                printfn "%d%sTrying %s(\"%s\")" (stream.Index) s label (stream.PeekString 1000)
//                printfn "At index: %d, string remaining: %s" (stream.Index) (stream.PeekString 1000)
                let reply = p stream    // set a breakpoint here when debugging
                printfn "%d%s%s(\"%s\") (%A)" (stream.Index) s label (stream.PeekString 1000) reply.Status
                reply
        else
            p

    // This is a version of the forwarding parser that
    // also takes a generic argument.
    let createParserForwardedToRefWithArguments() =
        let dummyParser = fun x y -> failwith "a parser created with createParserForwardedToRefWithArguments was not initialized"
        let r = ref dummyParser
        (fun arg stream -> !r arg stream), r : ('t -> Parser<'a, 'u>) * ('t -> Parser<'a, 'u>) ref

    (*
     * FPARSEC CHEAT SHEET
     * -------------------
     *
     * <||>      The infix "choice combinator" <||> applies the parser on the right side if the parser on the left side fails.
     * |>>      The infix "pipeline combinator" |>> applies the function on the right side to the result of the parser on the left side.
     * >>.      p1 >>. p2 parses p1 and p2 in sequence and returns the result of p2. 
     * .>>      p1 .>> p2 also parses p1 and p2 in sequence, but it returns the result of p1 instead of p2.
     * .>>.     An infix synonym for tuple2
     * >>=      The "bind combinator" takes a parser and a function producing a parser as arguments. p >>= f first applies
     *          the parser p to the input, then applies the function f to the result returned by p and finally applies
     *          the parser returned by f to the input.
     * tuple2   tuple2 p1 p2 returns a tuple consisting of the result of p1 (first) and the result of p2 (second)
     * pipe2    pipe2 p1 p2 f sequentially applies the two parsers p1 and p2 and then returns the result of the function
     *          application f x1 x2, where x1 and x2 are the results returned by p1 and p2.
     * sepBy    sepBy p1 p2 takes an "element" parser (p1) and a "separator" parser (p2) as the arguments and returns
     *          a parser for a list of elements separated by separators. 
     * createParserForwardedToRef
     *          let p, pRef = createParserForwardedToRef() creates a parser p that forwards all calls to the parser in
     *          the reference cell pRef. Initially, pRef holds a reference to a dummy parser that raises an exception on
     *          any invocation.
     * do ... :=
     *          do pRef :=, instead of let p =, lets us overwrite the implementation of parser p using the reference to p, pRef.
     *)

    // a simple typedef
    type P<'t> = Parser<'t, Env>   
    
    // custom character classes
    let isWSChar(c: char) : bool =
        isDigit(c) || isLetter(c) || c = '-' || c = ' '

    // Grammar forward references
    let (ArgumentList: P<Range> -> P<Expression list>, ArgumentListImpl) = createParserForwardedToRefWithArguments()
    let (ExpressionDecl: P<Range> -> P<Expression>, ExpressionDeclImpl) = createParserForwardedToRefWithArguments()
    let (RangeA1Union: P<Range>, RangeA1UnionImpl) = createParserForwardedToRef()
    let (RangeA1NoUnion: P<Range>, RangeA1NoUnionImpl) = createParserForwardedToRef()
    let (RangeR1C1: P<Range>, RangeR1C1Impl) = createParserForwardedToRef()

    // space-insensitive commas
    let Comma = spaces >>. pstring "," .>> spaces

    // Addresses
    let AddrR = pstring "R" >>. pint32
    let AddrC = pstring "C" >>. pint32
    let AddrIndirect = getUserState >>=
                        fun us ->
                            between
                                (pstring "INDIRECT(")
                                (pstring ")")
                                (manySatisfy ((<>) ')'))
                            |>> (fun expr -> IndirectAddress(expr, us) :> Address)
    let AddrR1C1 = getUserState >>=
                    fun (us: Env) ->
                        (attempt_opt AddrIndirect)
                        <||> (pipe2
                                AddrR
                                AddrC
                                (fun row col ->
                                    // TODO: R1C1 absolute/relative
                                    Address.fromR1C1withMode(row, col, AddressMode.Relative, AddressMode.Relative, us.WorksheetName, us.WorkbookName, us.Path)))
                    <!> "AddrR1C1"
    let AbsOrNot : P<AddressMode> = (attempt_opt ((pstring "$") >>% Absolute) <||> (pstring "" >>% Relative)) 
    let AddrA = many1Satisfy isAsciiUpper
    let AddrAAbs = pipe2 AbsOrNot AddrA (fun mode col -> (mode, col))
    let Addr1 = pint32
    let Addr1Abs = pipe2 AbsOrNot Addr1 (fun mode row -> (mode, row))
    let AddrA1 = getUserState >>=
                    fun (us: Env) ->
                        (attempt_opt AddrIndirect)
                        <||> (pipe2
                            AddrAAbs
                            Addr1Abs
                            (fun col row ->
                                let (colMode,colA) = col
                                let (rowMode,row1) = row
                                Address.fromA1withMode(row1, colA, rowMode, colMode, us.WorksheetName, us.WorkbookName, us.Path)
                                ))
                    
    let AnyAddr = ((attempt_opt AddrIndirect)
                  <||> (attempt_opt AddrR1C1)
                  <||> AddrA1)
                  <!> "AnyAddr"

    // Ranges
    let RA1TO_1 = (pstring ":" >>. AddrA1 .>> Comma) <!> "RA1TO_1"
    let RA1TO_2 = (pstring ":" >>. AddrA1) <!> "RA1TO_2"
    let RTO = ((attempt_opt RA1TO_1) <||> RA1TO_2) <!> "RTO"

    let RA1TC_1 = (Comma >>. AddrA1 .>> Comma) <!> "RA1TC_1"
    let RA1TC_2 = (Comma >>. AddrA1) <!> "RA1TC_2"
    let RTC = ((attempt_opt RA1TC_1) <||> RA1TC_2) <!> "RTC"

    let RA1_1 = (* AddrA1 RTC (1)*) (pipe2 AddrA1 RTC (fun a1 a2 -> Range([(a2,a2); (a1,a1)]))) <!> "RA1_1"
    let RA1_2 = (* AddrA1 RTO (2)*) (pipe2 AddrA1 RTO (fun a1 a2 -> Range(a1,a2))) <!> "RA1_2"

    let rat_fun = fun a1 a2 a3 -> Range([(a1,a2);(a3,a3)])
    let RA1_3a = (pipe3 AddrA1 RA1TO_1 RA1TC_1 rat_fun) <!> "RA1_3a"
    let RA1_3b = (pipe3 AddrA1 RA1TO_1 RA1TC_2 rat_fun) <!> "RA1_3b"
    let RA1_3c = (pipe3 AddrA1 RA1TO_2 RA1TC_1 rat_fun) <!> "RA1_3c"
    let RA1_3d = (pipe3 AddrA1 RA1TO_2 RA1TC_2 rat_fun) <!> "RA1_3d"
    let RA1_3 = (* AddrA1 RTO RTC (3)*) 
                ((attempt_opt RA1_3a) <||> (attempt_opt RA1_3b) <||> (attempt_opt RA1_3c) <||> RA1_3d) <!> "RA1_3"

    let RA1_4_UNION = (* AddrA1 "," R (4)*) (pipe2 (AddrA1 .>> Comma) RangeA1Union (fun a1 r1 -> Range(r1.Ranges() @ [(a1, a1)]))) <!> "RA1_4_UNION"
    let RA1_5_UNION = (* AddrA1 RTO R (5)*) (pipe3 AddrA1 RTO RangeA1Union (fun a1 a2 r2 -> Range ((a1,a2) :: r2.Ranges()))) <!> "RA1_5_UNION"
    do RangeA1UnionImpl :=   (attempt_opt RA1_5_UNION)
                        <||> (attempt_opt RA1_4_UNION)
                        <||> (attempt_opt RA1_3)
                        <||> (attempt_opt RA1_1)
                        <||> RA1_2
    do RangeA1NoUnionImpl := pipe2 AddrA1 RA1TO_2 (fun a1 a2 -> Range(a1, a2))
                        
    do RangeR1C1Impl := RangeA1Union

    let RangeWithUnion = (attempt_opt RangeA1Union) <!> "Union Range"
    let RangeNoUnion = (attempt_opt RangeA1NoUnion) <!> "No-Union Range"

    let RangeAny = ((attempt_opt RangeWithUnion) <||> RangeWithUnion) <!> "RangeAny"

    // Worksheet Names
    let WorksheetNameQuoted =
        let NormalChar = satisfy ((<>) '\'')
        let EscapedChar = pstring "''" |>> (fun s -> ''')
        between (pstring "'") (pstring "'")
                (many1Chars (NormalChar <||> EscapedChar))
    let WorksheetNameUnquoted = (many1Satisfy (fun c -> isWSChar(c)))
    let WorksheetName = (WorksheetNameQuoted <||> WorksheetNameUnquoted)
    // Workbook Names (this may be too restrictive)
    let Path = many1Satisfy ((<>) '[')
    let WorkbookName = between
                        (pstring "[")
                        (pstring "]")
                        (many1Satisfy (fun c -> c <> '[' && c <> ']'))
    let Workbook = (
                        (Path |>> Some)
                        <||> ((pstring "") >>% None)
                   )
                   .>>. WorkbookName

    // References
    // References consist of the following parts:
    //   A workbook name prefix
    //   A worksheet name prefix
    //   A single-cell address ("Address") or multi-cell address ("Range")
    let RRWQuoted = (between (pstring "'") (pstring "'") (Workbook .>>. WorksheetNameUnquoted)) <!> "RRWQuoted"
    let RangeReferenceWorkbook R = getUserState >>=
                                    fun (us: Env) ->
                                        (pipe2
                                            (RRWQuoted .>> pstring "!")
                                            R
                                            (fun ((wbpath, wbname), wsname) rng ->
                                                match wbpath with
                                                | Some(pth) -> ReferenceRange(Env(pth, wbname, wsname), rng) :> Reference
                                                | None      -> ReferenceRange(Env(us.Path, wbname, wsname), rng) :> Reference
                                            )
                                        )
                                        <!> "RangeReferenceWorkbook"
    let RangeReferenceWorksheet R = getUserState >>=
                                    fun (us: Env) ->
                                        pipe2
                                            (WorksheetName .>> pstring "!")
                                            R
                                            (fun wsname rng -> ReferenceRange(Env(us.Path, us.WorkbookName, wsname), rng) :> Reference)
                                    <!> "RangeReferenceWorksheet"
    let RangeReferenceNoWorksheet R = getUserState >>=
                                        fun (us: Env) ->
                                            R
                                            |>> (fun rng -> ReferenceRange(us, rng) :> Reference)
                                    <!> "RangeReferenceNOWorksheet"
    let RangeReference R = (attempt_opt (RangeReferenceWorkbook R))
                         <||> (attempt_opt (RangeReferenceWorksheet R))
                         <||> RangeReferenceNoWorksheet R
                         <!> "RangeReference"

    let ARWQuoted = (between
                        (pstring "'")
                        (pstring "'")
                        (Workbook .>>. WorksheetNameUnquoted))
                    <!> "ARWQuoted"
    let AddressReferenceWorkbook = getUserState >>=
                                    fun us ->
                                        (pipe2
                                            (ARWQuoted .>> pstring "!")
                                            AnyAddr
                                            (fun ((wbpath, wbname), wsname) addr ->
                                                match wbpath with
                                                | Some(pth) -> ReferenceAddress(Env(pth, wbname, wsname), addr) :> Reference
                                                | None      -> ReferenceAddress(Env(us.Path, wbname, wsname), addr) :> Reference
                                            )
                                        )
                                    <!> "AddressReferenceWorkbook"
    let AddressReferenceWorksheet = getUserState >>=
                                        fun us ->
                                            pipe2
                                                (WorksheetName .>> pstring "!")
                                                AnyAddr
                                                (fun wsname addr -> ReferenceAddress(Env(us.Path, us.WorkbookName, wsname), addr) :> Reference)
                                    <!> "AddressReferenceWorksheet"
    let AddressReferenceNoWorksheet = getUserState >>=
                                        fun us ->
                                            AnyAddr
                                            |>> (fun addr -> ReferenceAddress(us, addr) :> Reference)
                                      <!> "AddressReferenceNOWorksheet"
    let AddressReference = (attempt_opt AddressReferenceWorkbook)
                           <||> (attempt_opt AddressReferenceWorksheet)
                           <||> AddressReferenceNoWorksheet
                           <!> "AddressReference"

    let NamedReferenceFirstChar = satisfy (fun c -> c = '_' || isLetter(c))
    let NamedReferenceLastChars = manySatisfy (fun c -> c = '_' || isLetter(c) || isDigit(c))
    let NamedReference = getUserState >>=
                            fun us ->
                                pipe2
                                    NamedReferenceFirstChar
                                    NamedReferenceLastChars
                                    (fun c s -> ReferenceNamed(us, c.ToString() + s) :> Reference)
                         <!> "NamedReference"
    let StringReference = getUserState >>=
                            fun us ->
                                between
                                    (pstring "\"")
                                    (pstring "\"")
                                    (manySatisfy ((<>) '"'))
                                |>> (fun s -> ReferenceString(us, s) :> Reference)
                          <!> "StringReference"
    let ConstantReference = getUserState >>= 
                                fun us ->
                                    (attempt_opt
                                        (pfloat .>> pstring "%"
                                        |>> fun r -> ReferenceConstant(us, r / 100.0) :> Reference))
                                    <||> (pfloat |>> (fun r -> ReferenceConstant(us, r) :> Reference))
                            <!> "ConstantReference"
    let BooleanReference = getUserState >>=
                             fun us ->
                                 Array.map (fun b -> attempt_opt (pstring b)) [| "TRUE"; "FALSE" |]
                                 |> choice_opt
                                 |>> fun r -> ReferenceBoolean(us, System.Boolean.Parse(r)) :> Reference
                           <!> "BooleanReference"

    // This helper function sorts strings from
    // longest length to shortest length; length
    // ties are sorted lexicographically; ensures
    // that a substring is never matched, consumed,
    // and then returned to the calling parser which
    // then fails.
    let lmf(sa: string[]) : string[] =
        sa |>
        Array.sortWith
            (fun e1 e2 ->
                if e1.Length < e2.Length then
                    1
                elif e1.Length = e2.Length then
                    e1.CompareTo e2
                else
                    -1
            )

    // Functions
    let ArityNFunctionNameMaker n xs = xs |> Array.map (fun name -> attempt_opt (pstring name)) |> choice_opt <!> ("Arity" + n.ToString() + "FunctionName")
    let ArityAtLeastNFunctionNameMaker nplus xs = xs |> Array.map (fun name -> attempt_opt (pstring name)) |> choice_opt <!> ("Arity" + nplus.ToString() + "+FunctionName")

    let Arity0Names: string[] = [|"COLUMN"; "NA"; "NOW"; "PI"; "RAND"; "ROW"; "SHEET"; "SHEETS"; "TODAY"|]
    let Arity0FunctionName: P<string> = ArityNFunctionNameMaker 0 (lmf Arity0Names)

    let Arity1Names: string[] = [|"ABS"; "ACOS"; "ACOSH"; "ACOT"; "ACOTH"; "ARABIC"; "AREAS"; "ASC"; "ASIN"; "ASINH"; "ATAN"; "ATANH";
        "BAHTTEXT"; "BIN2DEC"; "BIN2HEX"; "BIN2OCT"; "CEILING.PRECISE"; "CELL"; "CHAR"; "CLEAN"; "CODE"; "COLUMN"; "COLUMNS"; "COS"; "COSH"; "COT"; "COTH"; "COUNTBLANK";
        "CSC"; "CSCH"; "CUBESETCOUNT"; "DAY"; "DBCS"; "DEC2BIN"; "DEC2HEX"; "DEC2OCT"; "DEGREES"; "DELTA"; "DOLLAR"; "ENCODEURL"; "ERF"; "ERF.PRECISE"; "ERFC"; "ERFC.PRECISE";
        "ERROR.TYPE"; "EVEN"; "EXP"; "FACT"; "FACTDOUBLE"; "FISHER"; "FISHERINV"; "FIXED"; "FLOOR.PRECISE"; "FORMULATEXT"; "GAMMA"; "GAMMALN"; "GAMMALN.PRECISE"; "GAUSS";
        "GESTEP"; "GROWTH"; "HEX2BIN"; "HEX2DEC"; "HEX2OCT"; "HOUR"; "HYPERLINK"; "IMABS"; "IMAGINARY"; "IMARGUMENT"; "IMCONJUGATE"; "IMCOS"; "IMCOSH"; "IMCOT"; "IMCSC";
        "IMCSCH"; "IMEXP"; "IMLN"; "IMLOG10"; "IMLOG2"; "IMREAL"; "IMSEC"; "IMSECH"; "IMSIN"; "IMSINH"; "IMSQRT"; "IMTAN"; "INDEX"; "INFO"; "INT";  "IRR";
        "ISBLANK"; "ISERR"; "ISERROR"; "ISEVEN"; "ISFORMULA"; "ISLOGICAL"; "ISNA"; "ISNONTEXT"; "ISNUMBER"; "ISODD"; "ISREF"; "ISTEXT"; "ISO.CEILING"; "ISOWEEKNUM";  "JIS";
        "LEFT"; "LEFTB"; "LEN"; "LENB"; "LINEST"; "LN"; "LOG"; "LOG10"; "LOGEST"; "LOWER"; "MDETERM"; 
        "MEDIAN"; "MINUTE"; "MDETERM"; "MDETERM"; "MINVERSE"; "MONTH"; "MUNIT";
        "N"; "NORMSDIST"; "NORM.S.INV"; "NORMSINV"; "NOT"; "NUMBERVALUE"; 
        "OCT2BIN"; "OCT2DEC"; "OCT2HEX"; "ODD"; "OR"; 
        "PHI"; "PHONETIC"; "PRODUCT"; "PROPER";
        "RADIANS"; "RIGHT"; "RIGHTB"; "ROMAN"; "ROW"; "ROWS";
        "SEC"; "SECH"; "SECOND"; "SHEET"; "SHEETS"; "SIGN"; "SIN"; "SINH"; "SKEW"; "SKEW.P"; "SQL.REQUEST"; "SQRT"; "SQRTPI"; "STDEV"; "STDEV.P"; "STDEV.S"; "STDEVA"; "STDEVP"; "STDEVPA"; "SUMPRODUCT"; "SUMSQ"; 
        "T"; "TAN"; "TANH"; "TIMEVALUE"; "TRANSPOSE"; "TREND"; "TRIM"; "TRUNC"; "TYPE";
        "UNICHAR"; "UNICODE"; "UPPER";
        "VALUE"; "VAR"; "VAR.P"; "VAR.S"; "VARA"; "VARP"; "VARPA"; 
        "WEBSERVICE"; "WEEKDAY"; "WEEKNUM"; 
        "XOR";  
        "YEAR"|]
    let Arity1FunctionName: P<string> = ArityNFunctionNameMaker 1 (lmf Arity1Names)

    let Arity2Names: string[] = [|"ADDRESS"; "ATAN2"; "AVERAGEIF"; "BASE"; "BESSELI"; "BESSELJ"; "BESSELK"; "BESSELY";
        "BIN2HEX"; "BIN2OCT"; "BITAND"; "BITLSHIFT"; "BITOR"; "BITRSHIFT"; "BITXOR"; "CEILING"; "CEILING.PRECISE"; "CELL"; "CHIDIST"; "CHIINV"; "CHITEST";
        "CHISQ.DIST.RT"; "CHISQ.TEST"; "COMBIN"; "COMBINA"; "COMPLEX"; "COUNTIF"; "COVAR"; "COVARIANCE.P"; "COVARIANCE.S"; "CUBEMEMBER"; "CUBERANKEDMEMBER";
        "CUBESET"; "DATEVALUE"; "DAYS"; "DAYS360"; "DEC2BIN"; "DEC2HEX"; "DEC2OCT"; "DECIMAL"; "DELTA"; "DOLLAR"; "DOLLARDE"; "DOLLARFR"; "EDATE"; "EFFECT";
        "EOMONTH"; "ERF"; "EXACT"; "F.DIST"; "FILTERXML"; "FIND"; "FINDB"; "FIXED"; "FLOOR"; "FLOOR.PRECISE"; "FORECAST.ETS.SEASONALITY"; "FREQUENCY"; "F.TEST";
        "FTEST"; "FVSCHEDULE"; "GESTEP"; "GROWTH"; "HEX2BIN"; "HEX2OCT"; "HYPERLINK"; "IF"; "IFERROR"; "IFNA"; "IMDIV"; "IMPOWER"; "IMSUB"; "INDEX";
        "INTERCEPT"; "IRR"; "ISO.CEILING"; "LARGE"; "LEFT"; "LEFTB"; "LINEST"; "LOG"; "LOGEST"; "LOOKUP";
        "MATCH";
        "MEDIAN"; "MMULT"; "MOD"; "MROUND";    
        "NETWORKDAYS.INTL"; "NETWORKDAYS"; "NOMINAL"; "NORM.S.DIST"; "NPV"; "NUMBERVALUE"; 
        "OCT2BIN"; "OCT2DEC"; "OCT2HEX"; "ODDFPRICE"; "OR";
        "PEARSON"; "PERCENTILE.EXC"; "PERCENTILE.INC"; "PERCENTILE"; "PERCENTRANK.EXC"; "PERCENTRANK.INC"; "PERCENTRANK"; "PERMUT"; "PERMUTATIONA"; "POWER"; "PROB"; "PRODUCT"; 
        "QUARTILE"; "QUARTILE.EXC"; "QUARTILE.INC"; "QUOTIENT";
        "RANDBETWEEN"; "RANK.AVG"; "RANK.EQ"; "RANK"; "REGISTER.ID"; "REPT"; "RIGHT"; "RIGHTB"; "ROMAN"; "ROUND"; "ROUNDDOWN"; "ROUNDUP"; "RSQ";
        "SEARCH"; "SEARCHB"; "SKEW"; "SKEW.P"; "SLOPE"; "SMALL"; "SQL.REQUEST"; "STDEV"; "STDEV.P"; "STDEV.S"; "STDEVA"; "STDEVP"; "STDEVPA"; "STEYX"; "SUBTOTAL"; "SUMIF"; "SUMPRODUCT"; "SUMSQ"; "SUMX2MY2"; "SUMX2PY2"; "SUMXMY2"; 
        "TBILLYIELD"; "T.DIST.RT"; "TEXT"; "T.INV"; "TINV"; "T.INV.2T"; "TREND"; "TRIMMEAN"; "TRUNC";
        "VAR"; "VAR.P"; "VAR.S"; "VARA"; "VARP"; "VARPA";
        "WEEKDAY"; "WEEKNUM"; "WORKDAY"; "WORKDAY.INTL";
        "XIRR"; "XOR";
        "YEARFRAC"; 
        "ZTEST"; "Z.TEST"|]
    let Arity2FunctionName: P<string> = ArityNFunctionNameMaker 2 (lmf Arity2Names)

    let Arity3Names: string[] = [|"ADDRESS"; "AVERAGEIF"; "BASE"; "BETADIST"; "BETAINV"; "BETA.INV"; "BINOM.DIST.RANGE"; "BINOM.INV";
        "CEILING.MATH"; "CHISQ.DIST"; "COMPLEX"; "CONFIDENCE"; "CONFIDENCE.NORM"; "CONFIDENCE.T"; "CONVERT"; "CORREL"; "COUPDAYBS"; "COUPDAYS"; "COUPDAYSNC"; "COUPNCD";
        "COUPNUM"; "COUPPCD"; "CRITBINOM"; "CUBEKPIMEMBER"; "CUBEMEMBER"; "CUBEMEMBERPROPERTY"; "CUBERANKEDMEMBER"; "CUBESET";
        "DATE"; "DATEDIF"; "DAVERAGE"; "DAYS360"; "DCOUNT"; "DCOUNTA"; "DGET"; "DMAX"; "DMIN"; "DPRODUCT"; "DSTDEV"; "DSTDEVP"; "DSUM"; "DVAR"; "DVARP"; "EXPON.DIST";
        "EXPONDIST"; "FDIST"; "F.DIST.RT"; "FIND"; "FINDB";"F.INV"; "F.INV.RT"; "FINV"; "FIXED"; "FLOOR.MATH"; "FORECAST"; "FORECAST.ETS"; "FORECAST.ETS.SEASONALITY";
        "FORECAST.LINEAR"; "FORECAST.ETS.CONFINT"; "FORECAST.ETS.STAT"; "FV"; "GAMMA.INV"; "GAMMAINV"; "GROWTH"; "HLOOKUP";
        "IF"; "INDEX"; "LINEST"; "LOGEST"; "LOGINV"; "LOGNORMDIST"; "LOGNORM.INV"; "LOOKUP";
        "MATCH";
        "MEDIAN"; "MIRR"; "MID"; "MIDB"; "MULTINOMIAL";
        "NEGBINOMDIST"; "NETWORKDAYS"; "NETWORKDAYS.INTL"; "NORM.INV"; "NORMINV"; "NPER"; "NPV"; "NUMBERVALUE"; 
        "OFFSET"; "OR";
        "PDURATION"; "PERCENTRANK.EXC"; "PERCENTRANK.INC"; "PERCENTRANK"; "PMT"; "POISSON.DIST"; "POISSON"; "PROB"; "PRODUCT"; "PV"; 
        "RANK.AVG"; "RANK.EQ"; "RANK"; "RATE"; "REGISTER.ID"; "RRI"; "RTD"; 
        "SEARCH"; "SEARCHB"; "SKEW"; "SKEW.P"; "SLN"; "SQL.REQUEST"; "STANDARDIZE"; "STDEV"; "STDEV.P"; "STDEV.S"; "STDEVA"; "STDEVP"; "STDEVPA"; "SUBSTITUTE"; "SUBTOTAL"; "SUMIF"; "SUMIFS"; "SUMPRODUCT"; "SUMSQ"; "SWITCH";
        "TBILLEQ"; "TBILLPRICE"; "T.DIST"; "TDIST"; "T.DIST.2T"; "TEXTJOIN"; "TIME"; "TREND";
        "VAR"; "VAR.P"; "VAR.S"; "VARA"; "VARP"; "VARPA"; "VLOOKUP"; 
        "WORKDAY"; "WORKDAY.INTL";
        "XIRR"; "XNPV"; "XOR";
        "YEARFRAC"; 
        "ZTEST"; "Z.TEST"|]
    let Arity3FunctionName: P<string> = ArityNFunctionNameMaker 3 (lmf Arity3Names)

    let Arity4Names: string[] = [|"ADDRESS"; "ACCRINTM"; "BETADIST"; "BETA.DIST"; "BETAINV"; "BETA.INV"; "BINOMDIST"; "BINOM.DIST";
        "BINOM.DIST.RANGE"; "COUPDAYBS"; "COUPDAYS"; "COUPDAYSNC"; "COUPNCD"; "COUPNUM"; "COUPPCD"; "CUBESET"; "DB"; "DDB"; "DISC"; "F.DIST"; "FORECAST.ETS";
        "FORECAST.ETS.SEASONALITY"; "FORECAST.ETS.CONFINT"; "FORECAST.ETS.STAT"; "FV"; "GAMMA.DIST"; "GAMMADIST"; "GROWTH"; "HLOOKUP"; "HYPGEOMDIST"; "INTRATE";
        "LINEST"; "LOGEST"; "LOGNORM.DIST"; 
        "MEDIAN"; "IPMT"; "ISPMT";
        "NEGBINOM.DIST"; "NETWORKDAYS.INTL"; "NORM.DIST"; "NORMDIST"; "NPER"; "NPV"; 
        "OFFSET"; "OR";
        "PMT"; "PPMT"; "PRICEDISC"; "PRICEMAT"; "PROB"; "PRODUCT"; "PV"; 
        "RATE"; "RECEIVED"; "REPLACE"; "REPLACEB"; "RTD"; 
        "SERIESSUM"; "SKEW"; "SKEW.P"; "SQL.REQUEST"; "STDEV"; "STDEV.P"; "STDEV.S"; "STDEVA"; "STDEVP"; "STDEVPA"; "SUBSTITUTE"; "SUBTOTAL"; "SUMPRODUCT"; "SUMSQ"; "SWITCH"; "SYD";
        "TEXTJOIN"; "TREND";"T.TEST"; "TTEST";
        "VAR"; "VAR.P"; "VAR.S"; "VARA"; "VARP"; "VARPA"; "VLOOKUP"; 
        "WEIBULL"; "WEIBULL.DIST"; "WORKDAY.INTL";
        "XOR";
        "YIELDDISC"|]
    let Arity4FunctionName: P<string> = ArityNFunctionNameMaker 4 (lmf Arity4Names)

    let Arity5Names = [|"ADDRESS"; "ACCRINTM"; "BETADIST"; "BETA.DIST"; "BETAINV"; "BETA.INV"; "CUBESET"; "DB"; "DDB";
        "DISC"; "DURATION"; "EUROCONVERT"; "FORECAST.ETS"; "FORECAST.ETS.CONFINT"; "FORECAST.ETS.STAT"; "FV"; "HYPGEOM.DIST"; "INTRATE"; "IPMT"; 
        "MEDIAN"; "MDURATION"; "MDURATION";
        "NPER"; "NPV";
        "OFFSET"; "OR";
        "PMT"; "PPMT"; "PRICEDISC"; "PRICEMAT"; "PRODUCT"; "PV"; 
        "RATE"; "RECEIVED"; "RTD"; 
        "SKEW"; "SKEW.P"; "SQL.REQUEST"; "STDEV"; "STDEV.P"; "STDEV.S"; "STDEVA"; "STDEVP"; "STDEVPA"; "SUBTOTAL"; "SUMIFS"; "SUMPRODUCT"; "SUMSQ"; "SWITCH";
        "TEXTJOIN"; 
        "VAR"; "VAR.P"; "VARA"; "VARP";"VARPA"; "VDB"; 
        "XOR";
        "YIELDDISC"; "YIELDMAT"|]
    let Arity5FunctionName: P<string> = ArityNFunctionNameMaker 5 (lmf Arity5Names)

    let Arity6Names: string[] = [|"ACCRINT"; "AMORDEGRC"; "AMORLINC"; "BETA.DIST";
        "CUMIPMT"; "CUMPRINC"; "DURATION"; "FORECAST.ETS"; "FORECAST.ETS.CONFINT"; "FORECAST.ETS.STAT"; "IPMT"; 
        "MEDIAN"; "MDURATION"; "MDURATION"; 
        "NPV"; 
        "OR";
        "PPMT"; "PRICE"; "PRODUCT"; 
        "RATE"; "RTD"; 
        "SKEW"; "SKEW.P"; "STDEV"; "STDEV.P"; "STDEV.S"; "STDEVA"; "STDEVP"; "STDEVPA"; "SUBTOTAL"; "SUMPRODUCT"; "SUMSQ"; "SWITCH";
        "TEXTJOIN"; 
        "VAR"; "VAR.P"; "VAR.S"; "VARA"; "VARP"; "VARPA"; "VDB";
        "XOR";
        "YIELD"; "YIELDMAT"|]
    let Arity6FunctionName: P<string> = ArityNFunctionNameMaker 6 (lmf Arity6Names)

    let Arity7Names: string[] = [|"ACCRINT"; "AMORDEGRC"; "AMORLINC"; "FORECAST.ETS.CONFINT";
        "MEDIAN"; "MULTINOMIAL";
        "NPV"; 
        "ODDLPRICE"; "ODDLYIELD"; "OR";
        "PRICE"; "PRODUCT"; 
        "RTD"; 
        "STDEV"; "STDEV.P"; "STDEV.S"; "STDEVA"; "STDEVP"; "STDEVPA"; "SUBTOTAL"; "SUMIFS"; "SUMPRODUCT"; "SUMSQ"; "SWITCH";
        "TEXTJOIN"; 
        "VAR"; "VAR.P"; "VAR.S"; "VARA"; "VARP"; "VARPA"; "VDB";
        "XOR";
        "YIELD"|]
    let Arity7FunctionName: P<string> = ArityNFunctionNameMaker 7 (lmf Arity7Names)

    let Arity8Names: string[] = [|"ACCRINT";
        "MEDIAN"; "MULTINOMIAL";
        "NPV"; 
        "ODDFPRICE"; "ODDFYIELD"; "ODDLPRICE"; "ODDLYIELD"; "OR";
        "PRODUCT"; 
        "RTD"; 
        "STDEV"; "STDEV.P"; "STDEV.S"; "STDEVA"; "STDEVP"; "STDEVPA"; "SUBTOTAL"; "SUMPRODUCT"; "SUMSQ"; "SWITCH";
        "TEXTJOIN"; 
        "VAR"; "VAR.P"; "VAR.S"; "VARA"; "VARP"; "VARPA";
        "XOR"|]
    let Arity8FunctionName: P<string> = ArityNFunctionNameMaker 8 (lmf Arity8Names)

    let Arity9Names: string[] = [|"ACCRINT";
        "MEDIAN"; "MULTINOMIAL";
        "NPV"; 
        "ODDFPRICE"; "ODDFYIELD"; "OR";
        "PRODUCT"; 
        "RTD";
        "STDEV"; "STDEV.P"; "STDEV.S"; "STDEVA"; "STDEVP"; "STDEVPA"; "SUBTOTAL"; "SUMPRODUCT"; "SUMSQ"; "SWITCH";
        "VAR"; "VAR.P"; "VAR.S"; "VARA"; "VARP"; "VARPA";
        "XOR"|]
    let Arity9FunctionName: P<string> = ArityNFunctionNameMaker 9 (lmf Arity9Names)

    let ArityAtLeast1Names: string[] = [|"AND"; "AVEDEV"; "AVERAGEA"; "AVERAGE";
        "CALL"; "CONCATENATE"; "CONCAT"; "COUNTA"; "COUNT"; "CUBEVALUE"; "DEVSQ"; "GCD"; "GEOMEAN"; "HARMEAN"; "IMPRODUCT";
        "IMSUM"; "KURT"; "LCM"; "MAXA"; "MAX"; "MINA"; "MIN"; "MODE.MULT"; "MODE.SNGL"; "MODE"; "MULTINOMIAL"; |]
    let ArityAtLeast1FunctionName: P<string> = ArityAtLeastNFunctionNameMaker 1 (lmf ArityAtLeast1Names)

    let ArityAtLeast2Names: string[] = [|"CHOOSE"; "COUNTIFS"; "GETPIVOTDATA"; "IFS"; |]
    let ArityAtLeast2FunctionName: P<string> = ArityAtLeastNFunctionNameMaker 2 (lmf ArityAtLeast2Names)

    let ArityAtLeast3Names: string[] = [|"AGGREGATE"; "AVERAGEIFS"; "MAXIFS"; "MINIFS"; |]
    let ArityAtLeast3FunctionName: P<string> = ArityAtLeastNFunctionNameMaker 3 (lmf ArityAtLeast3Names)

    let VarArgsNames: string[] = [|"SUM"|]
    let VarArgsFunctionName: P<string> =
        (lmf VarArgsNames)
        |> Array.map (fun name -> pstring name) |> choice_opt <!> "VarArgsFunctionName"

    let arityNNameArr: P<string>[] = 
        [|
            Arity0FunctionName;
            Arity1FunctionName;
            Arity2FunctionName;
            Arity3FunctionName;
            Arity4FunctionName;
            Arity5FunctionName;
            Arity6FunctionName;
            Arity7FunctionName;
            Arity8FunctionName;
        |]

    let ReservedWords: P<string>[] = 
        [|
            Arity0Names;
            Arity1Names;
            Arity2Names;
            Arity3Names;
            Arity4Names;
            Arity5Names;
            Arity6Names;
            Arity7Names;
            Arity8Names;
            Arity9Names;
            ArityAtLeast1Names;
            ArityAtLeast2Names;
            ArityAtLeast3Names;
            VarArgsNames;
        |]
        |> Array.concat
        |> lmf
        |> Array.map (fun ns -> pstring ns)

    let arityAtLeastNNameArr: P<string>[] =
        [|
            // NOTE: "ArityAtLeast0" is just VarArgs
            ArityAtLeast1FunctionName;
            ArityAtLeast2FunctionName;
            ArityAtLeast3FunctionName;
        |]

    let FunctionNamesForArity i: P<string> = arityNNameArr.[i]
    let FunctionNamesForAtLeastArity i: P<string> = arityAtLeastNNameArr.[i-1]

    // from http://stackoverflow.com/a/22252504/480764
    let rec pmap f xs =
        match xs with
        | x :: xs -> parse { let! y = f x
                             let! ys = pmap f xs
                             return y :: ys }
        | [] -> parse { return [] }

    let Argument R = fun (i: int) -> (((ExpressionDecl R) .>> Comma) <!> ("Argument #" + i.ToString()))

    let ArgumentsN R n =
        (pipe2
            (pmap (Argument R) [1..n-1])
            ((ExpressionDecl R) <!> ("Argument #" + n.ToString() + " (last arg)"))
            (fun exprs expr -> exprs @ [expr] ) <!> ("Arguments" + n.ToString())
        )

    let ArgumentsAtLeastN R n =
        // What I really want is sepByN; this parser basically
        // does the same thing by requiring a mandatory match of
        // n-1 arguments, then one more, and then any number
        // of optional arguments.
        (pipe3
            (parray (n-1) ((ExpressionDecl R) .>> Comma) )
            (sepBy1 (ExpressionDecl R) Comma)
            (ArgumentList R)
            (fun exprArrReqd exprAtLeastOne varArgs -> (Array.toList exprArrReqd) @ exprAtLeastOne @ varArgs) <!> ("Arguments" + n.ToString() + "+")
        )

    let ArityNFunction n R =
                // here, we ignore whatever Range context we are given
                // and use RangeNoUnion instead
                if n = 0 then
                    getUserState >>=
                    fun us ->
                        (FunctionNamesForArity n .>> (pstring "()" <!> "Arity" + n.ToString() + "FunctionOPENCLOSEBRACE"))
                        |>>
                            (fun fname -> ReferenceFunction(us, fname, [], Fixed(0)) :> Reference)
                else
                    getUserState >>=
                    fun us ->
                        pipe2
                            (FunctionNamesForArity n .>> (pstring "(" <!> "Arity" + n.ToString() + "FunctionOPENINGBRACE"))
                            ((ArgumentsN RangeNoUnion n) .>> (pstring ")" <!> "Arity" + n.ToString() + "FunctionCLOSINGBRACE" ))
                            (fun fname arglist -> ReferenceFunction(us, fname, arglist, Fixed(n)) :> Reference)
                <!> ("Arity" + n.ToString() + "Function")

    let ArityAtLeastNFunction n R =
                // here, we ignore whatever Range context we are given
                // and use RangeNoUnion instead
                getUserState >>=
                fun us ->
                    pipe2
                        (FunctionNamesForAtLeastArity n .>> pstring "(")
                        ((ArgumentsAtLeastN RangeNoUnion n) .>> pstring ")")
                        (fun fname arglist -> ReferenceFunction(us, fname, arglist, LowBound(n)) :> Reference)
                <!> ("Arity" + n.ToString() + "PlusFunction")

    let VarArgsFunction R =
                  // here, we ignore whatever Range context we are given
                  // and use RangeAnyUnion instead (i.e., try both)
                  getUserState >>=
                    fun us ->
                        pipe2
                            (VarArgsFunctionName .>> pstring "(")
                            ((ArgumentList RangeAny) .>> pstring ")")
                            (fun fname arglist -> ReferenceFunction(us, fname, arglist, VarArgs) :> Reference)
                   <!> "VarArgsFunction"

    let Function R: P<Reference> =
        (       
            (   // fixed arity
                arityNNameArr |>
                    Array.mapi (fun i _ -> (attempt_opt (ArityNFunction i R))) |> choice_opt)
            <||> (// low-bounded arity
                arityAtLeastNNameArr |>
                    // note: no "at-least-0" parser; that's just a varargs
                    Array.mapi (fun i _ -> (attempt_opt (ArityAtLeastNFunction (i + 1) R))) |> choice_opt)
            <||> (// varargs
                VarArgsFunction R)
        ) <!> "Function"
    
    do ArgumentListImpl := fun (R: P<Range>) -> sepBy ((ExpressionDecl R) <!> "VarArgs Argument") Comma <!> "ArgumentList"

    // Unary operators
    //let UnaryOpChar = (spaces >>. satisfy (fun c -> c = '+' || c = '-') .>> spaces) <!> "UnaryOp"

    // reserved words
    let ReservedWordsReference: P<Reference> =
        (Array.map (fun (rwp: P<string>) -> (attempt_opt rwp)) ReservedWords)
        |> choice_opt
        |>> (fun s ->
                raise (AST.ParseException ("'" + s + "' is a reserved word."))
            )

    // references
    let Reference R = (attempt_opt (RangeReference R))
                        <||> (attempt_opt AddressReference)
                        <||> (attempt_opt BooleanReference)
                        <||> (attempt_opt ConstantReference)
                        <||> (attempt_opt ReservedWordsReference)
                        <||> (attempt_opt StringReference)
                        <||> NamedReference
                      <!> "Reference"

    // Expressions
    let ParensExpr(R: P<Range>): P<Expression> = ((between (pstring "(") (pstring ")") (ExpressionDecl R)) |>> ParensExpr) <!> "ParensExpr"
    let ExpressionAtom(R: P<Range>): P<Expression> = (((attempt_opt (Function R)) <||> (Reference R)) |>> ReferenceExpr) <!> "ExpressionAtom"
    let ExpressionSimple(R: P<Range>): P<Expression> = ((attempt_opt (ExpressionAtom R)) <||> (ParensExpr R)) <!> "ExpressionSimple"
    //let UnaryOpExpr(R: P<Range>): P<Expression> = pipe2 UnaryOpChar (ExpressionDecl R) (fun op rhs -> UnaryOpExpr(op, rhs)) <!> "UnaryOpExpr"
   
    // Unary and Binary arithmetic operator parser generator for a given range parser
    // This exists because the OperatorPrecedenceParser does not take an opaque Term type
    let BinOpForRangeParser(R: P<Range>) =
        let opp = new OperatorPrecedenceParser<Expression,unit,Env>()
        opp.TermParser <- (ExpressionSimple R) .>> spaces
        opp.AddOperator(InfixOperator("<", spaces, 1, Associativity.Left, (fun lhs rhs -> AST.BinOpExpr("<", lhs, rhs))))
        opp.AddOperator(InfixOperator(">", spaces, 1, Associativity.Left, (fun lhs rhs -> AST.BinOpExpr(">", lhs, rhs))))
        opp.AddOperator(InfixOperator("=", spaces, 1, Associativity.None, (fun lhs rhs -> AST.BinOpExpr("=", lhs, rhs))))
        opp.AddOperator(InfixOperator("<=", spaces, 1, Associativity.None, (fun lhs rhs -> AST.BinOpExpr("<=", lhs, rhs))))
        opp.AddOperator(InfixOperator(">=", spaces, 1, Associativity.None, (fun lhs rhs -> AST.BinOpExpr(">=", lhs, rhs))))
        opp.AddOperator(InfixOperator("<>", spaces, 1, Associativity.None, (fun lhs rhs -> AST.BinOpExpr("<>", lhs, rhs))))
        opp.AddOperator(InfixOperator("&", spaces, 2, Associativity.Left, (fun lhs rhs -> AST.BinOpExpr("&", lhs, rhs))))
        opp.AddOperator(InfixOperator("+", spaces, 3, Associativity.Left, (fun lhs rhs -> AST.BinOpExpr("+", lhs, rhs))))
        opp.AddOperator(InfixOperator("-", spaces, 3, Associativity.Left, (fun lhs rhs -> AST.BinOpExpr("-", lhs, rhs))))
        opp.AddOperator(InfixOperator("*", spaces, 4, Associativity.Left, (fun lhs rhs -> AST.BinOpExpr("*", lhs, rhs))))
        opp.AddOperator(InfixOperator("/", spaces, 4, Associativity.Left, (fun lhs rhs -> AST.BinOpExpr("/", lhs, rhs))))
        opp.AddOperator(InfixOperator("^", spaces, 5, Associativity.Left, (fun lhs rhs -> AST.BinOpExpr("^", lhs, rhs))))
        opp.AddOperator(PrefixOperator("-", spaces, 6, false, (fun rhs -> AST.UnaryOpExpr('-', rhs))))
        opp.AddOperator(PrefixOperator("+", spaces, 6, false, (fun rhs -> AST.UnaryOpExpr('+', rhs))))
        opp
    
    let OpExpr(R: P<Range>): P<Expression> = (BinOpForRangeParser R).ExpressionParser <!> "OpExpr"

    // Parsing Excel argument lists is ambiguous without knowing
    // the arity of the function that you are parsing.  Thus parsers in this grammar
    // take a Range parser that parses ranges with the appropriate semantics.
    // The appropriate parser is overridden by the Function parser.
    // By default, the grammar tries to parse both (see top-level Formula parser).
    do ExpressionDeclImpl :=
        fun (R: P<Range>) ->
            (attempt_opt (OpExpr R))
            <||> (ExpressionSimple R)
            <!> "Expression"

    // Formulas
    let Formula = (pstring "=" .>> spaces >>. (ExpressionDecl RangeAny) .>> eof) <!> "Formula"


<|start_filename|>ParcelTest/PrecedenceTests.cs<|end_filename|>
﻿using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using ExprOpt = Microsoft.FSharp.Core.FSharpOption<AST.Expression>;
using Expr = AST.Expression;

namespace ParcelTest
{
    [TestClass]
    public class PrecedenceTests
    {
        [TestMethod]
        public void MultiplicationVsAdditionPrecedenceTest()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);

            var f = "=2*3+1";

            ExprOpt asto = Parcel.parseFormula(f, e.Path, e.WorkbookName, e.WorksheetName);

            Expr correct =
                Expr.NewBinOpExpr(
                    "+",
                    Expr.NewBinOpExpr(
                        "*",
                        Expr.NewReferenceExpr(new AST.ReferenceConstant(e, 2.0)),
                        Expr.NewReferenceExpr(new AST.ReferenceConstant(e, 3.0))
                    ),
                    Expr.NewReferenceExpr(new AST.ReferenceConstant(e, 1.0))
                );

            try
            {
                Expr ast = asto.Value;
                Assert.AreEqual(correct, ast);
            }
            catch (NullReferenceException nre)
            {
                Assert.Fail("Parse error: " + nre.Message);
            }
        }

        [TestMethod]
        public void UnaryPrecedenceTest()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);

            var f = "=-1 * 2";

            ExprOpt asto = Parcel.parseFormula(f, e.Path, e.WorkbookName, e.WorksheetName);

            Expr correct =
                Expr.NewBinOpExpr(
                    "*",
                    Expr.NewUnaryOpExpr(
                        '-',
                        Expr.NewReferenceExpr(new AST.ReferenceConstant(e, 1.0))
                    ),
                    Expr.NewReferenceExpr(new AST.ReferenceConstant(e,2.0))
                );

            try
            {
                Expr ast = asto.Value;
                Assert.AreEqual(correct, ast);
            }
            catch (NullReferenceException nre)
            {
                Assert.Fail("Parse error: " + nre.Message);
            }
        }
    }
}


<|start_filename|>FParsec/AssemblyInfo.fs<|end_filename|>
﻿namespace FParsec

open System.Reflection
open System.Runtime.CompilerServices
open System.Runtime.InteropServices

[<assembly: ComVisible(false)>]

#if LOW_TRUST
 #if CLR4
    [<assembly: System.Security.AllowPartiallyTrustedCallers>]
    [<assembly: System.Security.SecurityTransparent>]
 #endif
#endif

[<assembly: AssemblyTitle("FParsec.dll")>]
[<assembly: AssemblyDescription("FParsec.dll")>]

[<assembly: AssemblyProduct(FParsec.CommonAssemblyInfo.Product)>]
[<assembly: AssemblyCopyright(FParsec.CommonAssemblyInfo.Copyright)>]
[<assembly: AssemblyVersion(FParsec.CommonAssemblyInfo.Version)>]
[<assembly: AssemblyFileVersion(FParsec.CommonAssemblyInfo.FileVersion)>]
[<assembly: AssemblyConfiguration(FParsec.CommonAssemblyInfo.FSConfiguration)>]

[<assembly: InternalsVisibleTo(FParsec.CommonAssemblyInfo.TestAssemblyName)>]
do ()

<|start_filename|>ParcelTest/ArityTests.cs<|end_filename|>
﻿using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using ArgList = Microsoft.FSharp.Collections.FSharpList<AST.Expression>;
using ExprOpt = Microsoft.FSharp.Core.FSharpOption<AST.Expression>;
using Expr = AST.Expression;

namespace ParcelTest
{
    [TestClass]
    public class ArityTests
    {
        [TestMethod]
        public void Arity2Test()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);

            var f = "=SUMX2MY2(A4:A10,B4:B10)";

            ExprOpt asto = Parcel.parseFormula(f, e.Path, e.WorkbookName, e.WorksheetName);

            Expr[] a = {
                Expr.NewReferenceExpr(new AST.ReferenceRange(e, Utility.makeRangeForA1("A4:A10", e))),
                Expr.NewReferenceExpr(new AST.ReferenceRange(e, Utility.makeRangeForA1("B4:B10", e)))
            };
            ArgList args = Utility.makeFSList<AST.Expression>(a);
            Expr correct = Expr.NewReferenceExpr(new AST.ReferenceFunction(e, "SUMX2MY2", args, AST.Arity.NewFixed(2)));

            try
            {
                Expr ast = asto.Value;
                Assert.AreEqual(correct, ast);
            }
            catch (NullReferenceException nre)
            {
                Assert.Fail("Parse error: " + nre.Message);
            }
        }

        [TestMethod]
        public void Arity2Test2()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);

            var f = "=SUMX2MY2(A$4:A$10,B$4:B$10)";

            ExprOpt asto = Parcel.parseFormula(f, e.Path, e.WorkbookName, e.WorksheetName);

            Expr[] a = {
                Expr.NewReferenceExpr(new AST.ReferenceRange(e, Utility.makeRangeForA1("A$4:A$10", e))),
                Expr.NewReferenceExpr(new AST.ReferenceRange(e, Utility.makeRangeForA1("B$4:B$10", e)))
            };
            ArgList args = Utility.makeFSList<AST.Expression>(a);
            Expr correct = Expr.NewReferenceExpr(new AST.ReferenceFunction(e, "SUMX2MY2", args, AST.Arity.NewFixed(2)));

            try
            {
                Expr ast = asto.Value;
                Assert.AreEqual(correct, ast);
            }
            catch (NullReferenceException nre)
            {
                Assert.Fail("Parse error: " + nre.Message);
            }
        }

        [TestMethod]
        public void SpacesInArgListTest()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);

            var f = "=SUM(A1,A2,A3, A4)";

            ExprOpt asto = Parcel.parseFormula(f, e.Path, e.WorkbookName, e.WorksheetName);

            string[] addrs = { "A1", "A2", "A3", "A4" };
            var rng = Utility.makeUnionRangeFromA1Addrs(addrs, e);

            Expr[] a = {
                Expr.NewReferenceExpr(new AST.ReferenceRange(e, rng))
            };
            ArgList args = Utility.makeFSList(a);
            Expr correct = Expr.NewReferenceExpr(new AST.ReferenceFunction(e, "SUM", args, AST.Arity.VarArgs));

            try
            {
                Expr ast = asto.Value;
                Assert.AreEqual(correct, ast);
            }
            catch (NullReferenceException nre)
            {
                Assert.Fail("Parse error: " + nre.Message);
            }
        }
    }
}


<|start_filename|>ParcelTest/Utility.cs<|end_filename|>
﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using FSC = Microsoft.FSharp.Collections;

namespace ParcelTest
{
    public static class Utility
    {
        public static AST.Range makeRangeForA1(string rng, AST.Env e)
        {
            var addrs = rng.Split(':');

            return new AST.Range(
                makeAddressForA1(addrs[0], e),
                makeAddressForA1(addrs[1], e)
            );
        }

        public static AST.Range makeUnionRangeFromA1Addrs(string[] addrs, AST.Env e)
        {
            // note that we reverse the array because this is how it is parsed
            var addr_pairs = addrs.Zip(
                addrs,
                (a1, a2) => new Tuple<AST.Address, AST.Address>(makeAddressForA1(a1, e), makeAddressForA1(a2, e))).Reverse().ToArray();
            var addr_pairs_fs = makeFSList<Tuple<AST.Address, AST.Address>>(addr_pairs);
            return new AST.Range(addr_pairs_fs);
        }

        public static AST.Address makeAddressForA1(string addr, AST.Env e)
        {
            var r = new System.Text.RegularExpressions.Regex(@"(\$?)([A-Z]+)(\$?)([0-9]+)");

            var matches = r.Match(addr);

            var colMode = String.IsNullOrEmpty(matches.Groups[1].Value) ? AST.AddressMode.Relative : AST.AddressMode.Absolute;
            var col = matches.Groups[2].Value;
            var rowMode = String.IsNullOrEmpty(matches.Groups[3].Value) ? AST.AddressMode.Relative : AST.AddressMode.Absolute;
            var row = Convert.ToInt32(matches.Groups[4].Value);

            return AST.Address.fromA1withMode(row, col, rowMode, colMode, e.WorksheetName, e.WorkbookName, e.Path);
        }

        public static AST.Address makeAddressForA1(string col, int row, AST.Env env)
        {
            return AST.Address.fromA1withMode(
                row,
                col,
                AST.AddressMode.Relative,
                AST.AddressMode.Relative,
                env.WorksheetName,
                env.WorkbookName,
                env.Path
            );
        }

        private static FSC.FSharpList<T> mkFSL<T>(T[] arr, int i, FSC.FSharpList<T> fsl)
        {
            if (i >= 0)
            {
                return mkFSL(arr, i - 1, new FSC.FSharpList<T>(arr[i], fsl));
            } else
            {
                return fsl;
            }
        }

        public static FSC.FSharpList<T> makeFSList<T>(T[] arr)
        {
            return mkFSL(arr, arr.Length - 1, FSC.FSharpList<T>.Empty);
        }
    }
}


<|start_filename|>Parcel/RangeVisitor.fs<|end_filename|>
﻿module internal RangeVisitor
    let rec rangesFromRangeRef(ref: AST.ReferenceRange) : AST.Range list = [ref.Range]

    and rangesFromFunctionRef(ref: AST.ReferenceFunction) : AST.Range list =
        List.map (fun arg -> rangesFromExpr(arg)) ref.ArgumentList |> List.concat
        
    and rangesFromExpr(expr: AST.Expression) : AST.Range list =
        match expr with
        | AST.ReferenceExpr(r) -> rangesFromRef(r)
        | AST.BinOpExpr(op, e1, e2) -> rangesFromExpr(e1) @ rangesFromExpr(e2)
        | AST.UnaryOpExpr(op, e) -> rangesFromExpr(e)
        | AST.ParensExpr(e) -> rangesFromExpr(e)

    and rangesFromRef(ref: AST.Reference) : AST.Range list =
        match ref with
        | :? AST.ReferenceRange as r -> rangesFromRangeRef(r)
        | :? AST.ReferenceAddress -> []
        | :? AST.ReferenceNamed -> []   // TODO: symbol table lookup
        | :? AST.ReferenceFunction as r -> rangesFromFunctionRef(r)
        | :? AST.ReferenceConstant -> []
        | :? AST.ReferenceString -> []
        | :? AST.ReferenceBoolean -> []
        | _ -> raise (AST.ParseException "Unknown reference type.")


<|start_filename|>Parcel/Hash.fs<|end_filename|>
﻿module Hash

/// <summary>
///     Computes the Jenkins One-at-a-time general-purpose hash function.
///     See: https://en.wikipedia.org/wiki/Jenkins_hash_function#one-at-a-time
/// </summary>
/// <param name="key">A sequence of values of generic type.</param>
/// <returns>Returns a 32-bit integer.</returns>
let jenkinsOneAtATimeHash(key: seq<'a>) : int32 =
    // https://en.wikipedia.org/wiki/Jenkins_hash_function#one-at-a-time
    let hash = Seq.fold (fun hash elem ->
                   let hash1 = hash + uint32(elem.GetHashCode())
                   let hash2 = hash1 + (hash1 <<< 10)
                   hash2 ^^^ (hash2 >>> 6)
               ) 0u key
    let hash1 = hash + (hash <<< 3)
    let hash2 = hash1 ^^^ (hash1 >>> 11)
    let hash3 = hash2 + (hash2 <<< 15)
    int32 hash3

let cantorPair(k_1: uint32)(k_2: uint32)(r: uint32) : uint32 =
    (((k_1 + k_2) * (k_1 + k_2 + 1u))/2u + k_2) % r


<|start_filename|>FParsecCS/Cloning.cs<|end_filename|>
﻿// Copyright (c) <NAME> 2010-2011
// License: Simplified BSD License. See accompanying documentation.

#if !LOW_TRUST

using System;
using System.Collections.Generic;
using System.Reflection;
using System.Runtime.Serialization;
using System.Diagnostics;
using System.Reflection.Emit;

namespace FParsec.Cloning {

// The classes in this namespace provide a cloning service based on the serialization API.

// Capturing the state of an object and/or cloning it with this API is often at least an
// order of magnitude faster than doing the same with the BinaryFormatter and a MemoryStream
// (ignoring some initial setup costs and the JITing time).

// Some implementation details:
//
// The serialization API of the BCL (as supported by the BinaryFormatter) spans several
// interfaces, classes and attributes under the System.Runtime.Serialization namespace.
// Unfortunately the publicly available API documentation provided by Microsoft does not
// sufficiently cover certain important details of the serialization API behaviour.
//
// For example, the documentation only vaguely discusses whether serialization events
// like OnDeserialized are invoked in a certain order on multiple objects in a graph.
// It seems that many API users intuitively expect a certain ordering, at least in
// simple cases, but it is also clear that an ordering can not be guaranteed in all
// cases (e.g. in the case of a cyclic object graph).
//
// The .NET BinaryFormatter seems to attempt to invoke the deserialization events
// on dependent objects first, but its behaviour is inconsistent and in some situations
// arguably buggy. The following bug report discusses these issues in more detail:
// https://connect.microsoft.com/VisualStudio/feedback/details/549277
//
// For the sake of compatibility with the .NET BinaryFormatter we try to mimic the
// basic principles of its behaviour. However, we certainly do not try to copy every
// bug or inconsistency.
//
// In order to be on the safe side we sort the serialized object graph topologically
// if it contains objects implementing an OnDeserialized handler or the ISerializable
// or IObjectReferenence interfaces. Since the object graph can contain cycles, we
// first identify strongly connected components using a variant of Tarjan's algorithm.
// Any OnDeserializing handler, deserialization constructor, OnDeserialized handler or
// IObjectReference.GetRealObject method (in that order) is then invoked in the
// topological order starting with the most dependent objects. Objects in a strongly
// connected component (with more than 1 object) are processed in the reverse order
// in which the objects in the component where discovered during a depth-first search
// of the serialized object graph. In a first pass the OnDeserializing handlers and
// deserialization constructors of the objects in the component are invoked. Any
// OnDeserialized handlers are then invoked in a second pass.
// OnSerializing handlers are invoked immediately before an object is serialized.
// OnSerialized handlers are invoked in an undefined order at the end of the
// serialization job (not immediately after an object's subgraph has been serialized).
//
// We only allow an object implementing IObjectReference in a cycle of the serialized
// object graph under the following conditions (which are more restrictive than
// what the .NET BinaryFormatter enforces):
// - There may only be 1 object implementing IObjectReference in a cycle.
// - The type implementing IObjectReference must not be a value type.
// - All objects containing references to the IObjectReference object must have reference types.
// - The type implementing IObjectReference must not have any OnDeserialized handler.
// - There must not be any other object in the cycle implementing ISerializable.
//
// Similar to the .NET BinaryFormatter we delay all IDeserializationCallbacks until
// the end of the deserialization of the complete object graph (not just the relevant
// subgraph). As explained in the referenced Connect bug report this behaviour has some
// severe consequences for the usefulness of IDeserializationCallbacks and the
// composability of the whole serialization API. However, for compatibility we really
// have to stick to Microsoft's design, even if in our case it would actually
// be simpler to invoke the callbacks in topological order as soon as an object's
// subgraph (and its strongly connected component) is completely deserialized.
//
// If the serialized object graph contains unboxed value type instances, any event
// handlers are invoked on boxed copies as follows:
// OnSerializing and OnSerialized handlers are not called on the original value type
// instance (which can be a field or an array element), but on a boxed copy of the
// instance. Thus, if the handler mutates the instance, the changes do not show up in the
// object graph that was serialized, though changes made by OnSerializing (but not
// OnSerialized) will show up in the deserialized object graph. This behaviour
// simplifies the implementation and is in accordance with the behaviour of the
// .NET BinaryFormatter.
// OnDeserializing and OnDeserialized handlers are invoked on a boxed value type instance
// too, but this time any changes show up in the deserialized object graph, because
// the boxed instance is copied into the deserialized object graph after the
// OnDeserialized event (at least if the instance is not part of an object cycle).
// This deviates from the BinaryFormatter behaviour in that
// the BinaryFormatter seems to copy the boxed instance into the deserialized object
// graph before the OnDeserialized event. However, since mutating the instance
// in an OnDeserialized handler has no effect when using the BinaryFormatter,
// hopefully no one causes an incompatibility with this implementation by actually trying
// to mutate the instance. (Note that mutable structs with serialization event handlers
// are extremely rare anyway).
// An IDeserializationCallback.OnDeserialization handler is invoked on the boxed instance
// after it has been copied into the deserialized object graph (and then is not copied
// again), so any changes won't show up in the deserialized unboxed value type instance
// (this holds for both the .NET BinaryFormatter and this implementation).


/// <summary>Contains the serialized state of on object.</summary>
public abstract class CloneImage {
    /// <summary>Deserializes the object state into a new object.</summary>
    public abstract object CreateClone();

    internal CloneImage() {}
}

public abstract class Cloner {
    // public interface
    public readonly Type Type;

    /// <summary>Returns a cloner for the given <em>run-time</em> type.</summary>
    /// <param name="type">The run-time type of the objects to clone. The type must be serializable.</param>
    public static Cloner Create(Type type) {
        lock (Cache) return CreateWithoutLock(type);
    }

    /// <summary>Copies the given object using the serialization API.</summary>
    /// <param name="instance">The object to clone. instance.GetType() must equal the Type the Cloner was created for.</param>
    public object Clone(object instance) {
        return CaptureImage(instance, false).CreateClone();
    }

    /// <summary>Returns an image of the given object instance.</summary>
    /// <param name="instance">The object to capture an image of.</param>
    public CloneImage CaptureImage(object instance) {
        return CaptureImage(instance, true);
    }

    // internal/protected interface

    private readonly CloneEventHandlers EventHandlers;

    private Cloner(Type type, CloneEventHandlers eventHandlers) { Type = type; EventHandlers = eventHandlers; }

    internal abstract State CaptureShallowStateAndEnqueueNestedState(object value, CaptureContext captureContext);

    internal sealed class CaptureContext {
        public readonly bool IsReturnedToUser;

        public CaptureContext(bool stateIsReturnedToUser) {
            IsReturnedToUser = stateIsReturnedToUser;
        }

        // currently uses a static queue, but could easily be rewritten to use an instance queue
        public int GetObjectIndex(object instance, Cloner cloner) {
            Debug.Assert(instance.GetType() == cloner.Type);
            int objectIndex;
            if (!ObjectIndices.TryGetValue(instance, out objectIndex)) {
                objectIndex = ObjectIndices.Count;
                ObjectIndices.Add(instance, objectIndex);
                var item = new WorkItem{Cloner = cloner, Instance = instance};
                WorkQueue.Enqueue(item);
            }
            return objectIndex;
        }
    }

    // internal interface

    internal abstract class State {
        /// <summary>May be null.</summary>
        public readonly CloneEventHandlers EventHandlers;

        /// <summary>Indices of nested objects in the object graph. May be null.</summary>
        public readonly int[] ObjectIndices;

        /// <summary>May be null.</summary>
        public int[] StronglyConnectedComponent;

        public abstract Type Type { get; }
        public abstract object CreateUninitializedObject();
        public abstract void WriteToUninitializedObject(object instance, object[] objectGraph);

        public State(CloneEventHandlers eventHandlers, int[] objectIndices) {
            EventHandlers = eventHandlers;
            ObjectIndices = objectIndices;
        }

        private State() {}
        public static readonly State Dummy = new DummyState();
        private sealed class DummyState : State {
            public override Type Type { get {
                throw new NotImplementedException();
            } }

            public override object CreateUninitializedObject() {
               throw new NotImplementedException();
            }

            public override void WriteToUninitializedObject(object instance, object[] objectGraph) {
                throw new NotImplementedException();
            }
        }
    }

    private static readonly StreamingContext StreamingContext = new StreamingContext(StreamingContextStates.Clone);
    private static readonly FormatterConverter FormatterConverter = new FormatterConverter();

    private static readonly Func<object, object> CloneMemberwise = CreateMemberwiseCloneDelegate();
    private static Func<object, object> CreateMemberwiseCloneDelegate() {
        var dynamicMethod = new DynamicMethod("InvokeMemberwiseClone", typeof(object), new Type[]{typeof(object)}, true);
        var ilg = dynamicMethod.GetILGenerator();
        ilg.Emit(OpCodes.Ldarg_0);
        var method = typeof(object).GetMethod("MemberwiseClone", BindingFlags.NonPublic | BindingFlags.Instance);
        ilg.EmitCall(OpCodes.Call, method, null); // non-virtual call
        ilg.Emit(OpCodes.Ret);
        return (Func<object, object>)dynamicMethod.CreateDelegate(typeof(Func<object, object>));
    }

    // private data and methods

    // Cache serves as the synchronization root for the Create and CaptureImage methods
    private static readonly Dictionary<Type, Cloner> Cache = new Dictionary<Type, Cloner>();

    private static Cloner CreateWithoutLock(Type type) {
        Cloner cloner;
        if (Cache.TryGetValue(type, out cloner)) return cloner;

        if (!type.IsSerializable)
            throw new SerializationException("The type '" + type.ToString() + "' is not marked as serializable.");

        if (!type.IsArray) {
            var eventHandlers = CloneEventHandlers.Create(type);
            if (eventHandlers != null && (eventHandlers.Events & CloneEvents.ISerializable) != 0) {
                cloner = new CustomSerializationCloner(type, eventHandlers);
            } else {
                bool typeIsBlittable;
                var fields = GetSerializedFields(type, out typeIsBlittable);
                if (typeIsBlittable && (eventHandlers == null || (eventHandlers.Events & CloneEvents.OnDeserializing) == 0))
                    cloner = new BlittableCloner(type, eventHandlers, fields);
                else
                    cloner = new NativeSerializationCloner(type, eventHandlers, fields);
            }
        } else { // array
            var elementType = type.GetElementType();
            if (elementType.IsPrimitive || elementType == typeof(string)) {
                cloner = new BlittableCloner(type, null, new FieldInfo[0]);
            } else {
                var elementCloner = CreateWithoutLock(elementType);
                if (elementType.IsValueType && elementCloner is BlittableCloner)
                    cloner = new BlittableCloner(type, null, new FieldInfo[0]);
                else if (type.GetArrayRank() == 1)
                    cloner = new Rank1ArrayCloner(type, elementCloner);
                else
                    cloner = new RankNArrayCloner(type, elementCloner);
            }
        }

        Cache.Add(type, cloner);
        return cloner;
    }

    // for optimization purposes CaptureImage uses some static queues

    private sealed class PhyiscalEqualityObjectComparer : System.Collections.Generic.EqualityComparer<object> {
        public override bool Equals(object x, object y) { return x == y; }
        public override int GetHashCode(object obj) {
            return System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(obj);
        }
    }

    private static readonly Dictionary<object, int> ObjectIndices = new Dictionary<object, int>(new PhyiscalEqualityObjectComparer());
    private static readonly List<State> States = new List<State>();
    private static readonly Queue<WorkItem> WorkQueue = new Queue<WorkItem>();
    private static readonly List<OnSerializedListItem> OnSerializedList = new List<OnSerializedListItem>();
    private static readonly List<int> ObjectReferenceList = new List<int>();

    private struct WorkItem {
        public Cloner Cloner;
        public object Instance;

        public WorkItem(Cloner cloner, object instance) {
            Cloner = cloner;
            Instance = instance;
        }
    }

    private struct OnSerializedListItem {
        public CloneEventHandlers EventHandlers;
        public object Instance;

        public OnSerializedListItem(CloneEventHandlers cloneEventHandlers, object instance) {
            EventHandlers = cloneEventHandlers;
            Instance = instance;
        }
    }

    private static bool Contains(int[] arrayOrNull, int element) {
        if (arrayOrNull != null) {
            foreach (var e in arrayOrNull)
                if (e == element) return true;
        }
        return false;
    }

    private CloneImage CaptureImage(object instance, bool imageIsReturnedToUser) {
        if (instance.GetType() != Type)
            throw new ArgumentException("The object instance does not have the run-time type the Cloner was created for.");
        lock (Cache) {
            try {
                bool needSort = false;

                // reserve 0-index spot
                ObjectIndices.Add(State.Dummy, 0);
                States.Add(null);

                var captureInfo = new CaptureContext(imageIsReturnedToUser);

                ObjectIndices.Add(instance, 1);
                WorkQueue.Enqueue(new WorkItem(this, instance));
                int deserializationCallbackCount = 0;
                do {
                    var item = WorkQueue.Dequeue();
                    var cloner = item.Cloner;
                    if (cloner.EventHandlers == null) {
                        States.Add(item.Cloner.CaptureShallowStateAndEnqueueNestedState(item.Instance, captureInfo));
                    } else if (cloner.EventHandlers.Events == CloneEvents.ISerializable) {
                        States.Add(item.Cloner.CaptureShallowStateAndEnqueueNestedState(item.Instance, captureInfo));
                        needSort = true;
                    } else {
                        var eventHandlers = cloner.EventHandlers;
                        if ((eventHandlers.Events & CloneEvents.OnSerializing) != 0)
                            eventHandlers.InvokeOnSerializing(item.Instance, StreamingContext);
                        if ((eventHandlers.Events & CloneEvents.OnSerialized) != 0)
                            OnSerializedList.Add(new OnSerializedListItem(eventHandlers, item.Instance));
                        var state = item.Cloner.CaptureShallowStateAndEnqueueNestedState(item.Instance, captureInfo);
                        States.Add(state);
                        eventHandlers = state.EventHandlers; // may be different from cloner.EventHandlers (for CustomSerializationState)
                        if ((eventHandlers.Events & (  CloneEvents.ISerializable
                                                     | CloneEvents.OnDeserialized
                                                     | CloneEvents.IObjectReference)) != 0) //
                        {
                            needSort = true;
                            if ((eventHandlers.Events & CloneEvents.IObjectReference) != 0)
                                ObjectReferenceList.Add(States.Count - 1);
                        }
                        // unfortunately the BinaryFormatter doesn't guarantee any order for IDeserializationCallbacks
                        if ((eventHandlers.Events & (CloneEvents.IDeserializationCallback)) != 0)
                            ++deserializationCallbackCount;
                    }
                } while (WorkQueue.Count != 0);
                var states = States.ToArray();

                if (OnSerializedList.Count != 0) {
                    foreach (var item in OnSerializedList)
                        item.EventHandlers.InvokeOnSerialized(item.Instance, StreamingContext);
                }

                if (!needSort)
                    return new SimpleImage(states, deserializationCallbackCount);

                int[] order = ComputeTopologicalOrder(states);
                if (ObjectReferenceList.Count != 0) {
                    foreach (var index1 in ObjectReferenceList) {
                        var state1 = states[index1];
                        var scc = state1.StronglyConnectedComponent;
                        if (scc == null) continue;
                        var type1 = state1.Type;
                        if (type1.IsValueType)
                            throw new SerializationException("The serialized object graph contains a cycle that includes a value type object (type: "+ type1.FullName +") implementing IObjectReference.");
                        if ((state1.EventHandlers.Events & CloneEvents.OnDeserialized) != 0)
                            throw new SerializationException("The serialized object graph contains a cycle that includes an object (type: "+ type1.FullName +") implementing IObjectReference and also exposing an OnDeserialized handler.");
                        foreach (var index2 in scc) {
                            if (index2 == index1) continue;
                            var state2 = states[index2];
                            var type2 = state2.Type;
                            if (state2.EventHandlers != null && (state2.EventHandlers.Events & (CloneEvents.ISerializable | CloneEvents.IObjectReference)) != 0) {
                                var msg = String.Format("The serialized object graph contains a cycle that includes an object (type: {0}) implementing IObjectReference and another object (type: {1}) implementing ISerializable and/or IObjectReference .", type1.FullName, type2.FullName);
                                throw new SerializationException(msg);
                            }
                            if (type2.IsValueType && Contains(state2.ObjectIndices, index1)) {
                                var msg = String.Format("The serialized object graph contains a cycle that includes a value type object (type: {0}) referencing an IObjectReference object (type: {1}) in the same cycle.", type2.FullName, type1.FullName);
                                throw new SerializationException(msg);
                            }
                        }
                    }
                }
                return new OrderedImage(states, order, deserializationCallbackCount);
            } finally {
                States.Clear();
                ObjectIndices.Clear();
                if (WorkQueue.Count != 0) WorkQueue.Clear();
                if (OnSerializedList.Count != 0) OnSerializedList.Clear();
                if (ObjectReferenceList.Count != 0) ObjectReferenceList.Clear();
            }
        }
    }

    private sealed class BlittableCloner : Cloner {
        internal readonly FieldInfo[] SerializedFields;

        public BlittableCloner(Type type, CloneEventHandlers eventHandlers, FieldInfo[] serializedFields) : base(type, eventHandlers) {
            Debug.Assert(serializedFields != null);
            SerializedFields = serializedFields;
        }

        internal override State CaptureShallowStateAndEnqueueNestedState(object instance, CaptureContext captureContext) {
            Debug.Assert(Type == instance.GetType());
            if (captureContext.IsReturnedToUser) {
                return new BlittableState(EventHandlers, CloneMemberwise(instance));
            } else {
                return new BlittableState(EventHandlers, instance);
            }
        }
    }

    private sealed class BlittableState : State {
        private object Value;

        public BlittableState(CloneEventHandlers eventHandlers, object value) : base(eventHandlers, null) {
            Value = value;
        }

        public override Type Type { get { return Value.GetType(); } }

        public override object CreateUninitializedObject() {
            return Cloner.CloneMemberwise(Value);
        }

        public override void WriteToUninitializedObject(object instance, object[] objectGraph) { }
    }

    private sealed class Rank1ArrayCloner : Cloner {
        Cloner  PreviousElementCloner;

        public Rank1ArrayCloner(Type type, Cloner elementCloner) : base(type, null) {
            PreviousElementCloner = elementCloner;
        }

        internal override State CaptureShallowStateAndEnqueueNestedState(object instance, CaptureContext captureContext) {
            Debug.Assert(Type == instance.GetType());
            var array = (Array)instance;
            var lowerBound = array.GetLowerBound(0);
            var length = array.Length; // should throw an exception if length > Int32.MaxValue
            if (length == 0) return new BlittableState(null, instance);
            var throwExceptionOnOverflow = checked(lowerBound + length);
            var objectIndices = new int[length];
            var cloner = PreviousElementCloner;
            var previousType = cloner.Type;
            for (int i = 0; i < length; ++i) {
                var value = array.GetValue(lowerBound + i);
                if (value != null) {
                    var type = value.GetType();
                    if (type != previousType) {
                        cloner = CreateWithoutLock(type);
                        previousType = type;
                    }
                    objectIndices[i] = captureContext.GetObjectIndex(value, cloner);
                }
            }
            PreviousElementCloner = cloner;
            return new Rank1ArrayState(Type.GetElementType(), lowerBound, objectIndices);
        }
    }

    private sealed class Rank1ArrayState : State {
        private readonly Type ElementType;
        private readonly int LowerBound;

        public Rank1ArrayState(Type elementType, int lowerBound, int[] objectIndices) : base(null, objectIndices) {
            Debug.Assert(objectIndices != null);
            ElementType = elementType;
            LowerBound = lowerBound;
        }

        public override Type Type { get { return ElementType.MakeArrayType(); } }

        public override object CreateUninitializedObject() {
            if (LowerBound == 0)
                return Array.CreateInstance(ElementType, ObjectIndices.Length);
            else
                return Array.CreateInstance(ElementType, new int[]{ObjectIndices.Length}, new int[]{LowerBound});
        }

        public override void WriteToUninitializedObject(object instance, object[] objectGraph) {
            var array = (Array)instance;
            var objectIndices = ObjectIndices;
            for (int i = 0; i < objectIndices.Length; ++i) {
                var objectIndex = objectIndices[i];
                if (objectIndex == 0) continue;
                array.SetValue(objectGraph[objectIndex], LowerBound + i);
            }
        }
    }

    private sealed class RankNArrayCloner : Cloner {
        Cloner PreviousElementCloner;

        public RankNArrayCloner(Type type, Cloner elementCloner) : base(type, null) {
            PreviousElementCloner = elementCloner;
        }

        internal override State CaptureShallowStateAndEnqueueNestedState(object instance, CaptureContext captureContext) {
            Debug.Assert(Type == instance.GetType());
            var array = (Array)instance;
            var rank = array.Rank;
            var lowerBounds = new int[rank];
            var lengths = new int[rank];
            var ends = new int[rank];
            var numberOfElements = 1;
            for (int d = 0; d < rank; ++d) {
                var lowerBound = array.GetLowerBound(d);
                lowerBounds[d] = lowerBound;
                var length = array.GetLength(d);
                lengths[d] = length;
                ends[d] = checked(lowerBound + length);
                numberOfElements = checked(numberOfElements * length);
            }
            var objectIndices = new int[numberOfElements];
            var cloner = PreviousElementCloner;
            var previousType = cloner.Type;
            var indices = (int[])lowerBounds.Clone();
            for (int i = 0; i < numberOfElements; ++i) {
                var value = array.GetValue(indices);
                if (value != null) {
                    var type = value.GetType();
                    if (type != previousType) {
                        cloner = CreateWithoutLock(type);
                        previousType = type;
                    }
                    objectIndices[i] = captureContext.GetObjectIndex(value, cloner);
                }
                // increment multi-dimensional index
                var d = rank - 1;
                do {
                    if (++indices[d] < ends[d]) break;
                    indices[d] = lowerBounds[d];
                } while (--d >= 0);
            }
            PreviousElementCloner = cloner;
            return new RankNArrayState(Type.GetElementType(), lengths, lowerBounds, ends, objectIndices);
        }
    }

    private sealed class RankNArrayState : State {
        private readonly Type ElementType;
        private readonly int[] Lengths;
        private readonly int[] LowerBounds;
        private readonly int[] Ends;

        public RankNArrayState(Type elementType, int[] lengths, int[] lowerBounds, int[] ends, int[] objectIndices)
               : base(null, objectIndices)
        {
            Debug.Assert(lengths != null && lengths.Length == lowerBounds.Length && lengths.Length == ends.Length && objectIndices != null);
            ElementType = elementType;
            Lengths = lengths;
            LowerBounds = lowerBounds;
            Ends = ends;
        }

        public override Type Type { get { return ElementType.MakeArrayType(Lengths.Length); } }

        public override object CreateUninitializedObject() {
            return Array.CreateInstance(ElementType, Lengths, LowerBounds);
        }

        public override void WriteToUninitializedObject(object instance, object[] objectGraph) {
            var array = (Array)instance;
            var indices = (int[])LowerBounds.Clone();
            foreach (var objectIndex in ObjectIndices) {
                if (objectIndex != 0)
                    array.SetValue(objectGraph[objectIndex], indices);
                // increment multi-dimensional index
                var d = LowerBounds.Length - 1;
                do {
                    if (++indices[d] < Ends[d]) break;
                    indices[d] = LowerBounds[d];
                } while (--d >= 0);
            }
        }
    }

    private sealed class NativeSerializationCloner : Cloner {
        internal readonly FieldInfo[] SerializedFields;
        private readonly Cloner[] Cloners;

        private Func<object, object[]> FieldValuesGetter; // lazily initialized
        internal Action<object, object[], int[], object[]> FieldValuesSetter; // lazily initialized

        public NativeSerializationCloner(Type type, CloneEventHandlers eventHandlers, FieldInfo[] serializedFields) : base(type, eventHandlers) {
            SerializedFields = serializedFields;
            Cloners = new Cloner[serializedFields.Length];
        }

        internal override State CaptureShallowStateAndEnqueueNestedState(object instance, CaptureContext captureContext) {
            Debug.Assert(Type == instance.GetType());
            if (SerializedFields.Length == 0)
                return new NativeSerializationState(this);
            var getter = FieldValuesGetter;
            if (getter == null)
                FieldValuesGetter = getter = CreateFieldValuesGetter(Type, SerializedFields);
            var values = getter(instance); // GetFieldValues(instance, SerializedFields);
            int[] objectIndices = new int[values.Length];
            for (int i = 0; i < values.Length; ++i) {
                var value = values[i];
                if (value == null) continue;
                var type = value.GetType();
                if (type.IsPrimitive || type == typeof(string)) continue;
                values[i] = null;
                var cloner = Cloners[i];
                if (cloner == null || type != cloner.Type) {
                    cloner = CreateWithoutLock(type);
                    Cloners[i] = cloner;
                }
                objectIndices[i] = captureContext.GetObjectIndex(value, cloner);
            }
            return new NativeSerializationState(this, values, objectIndices);
        }
    }

    private sealed class NativeSerializationState : State {
        private readonly NativeSerializationCloner Cloner;
        private readonly object[] Values; // maybe null if object has no fields

        public NativeSerializationState(NativeSerializationCloner cloner)
               : base(cloner.EventHandlers, null)
        {
            Cloner = cloner;
        }

        public NativeSerializationState(NativeSerializationCloner cloner, object[] values, int[] objectIndices)
               : base(cloner.EventHandlers, objectIndices)
        {
            Debug.Assert(cloner != null && values.Length != 0 && values.Length == objectIndices.Length);
            Cloner = cloner;
            Values = values;
        }

        public override Type Type { get { return Cloner.Type; } }

        public override object CreateUninitializedObject() {
            return FormatterServices.GetUninitializedObject(Cloner.Type);
        }

        public override void WriteToUninitializedObject(object instance, object[] objectGraph) {
            if (ObjectIndices == null) return;
            var setter = Cloner.FieldValuesSetter;
            if (setter == null)
                Cloner.FieldValuesSetter = setter = CreateFieldValuesSetter(Cloner.Type, Cloner.SerializedFields);
            setter(instance, Values, ObjectIndices, objectGraph);
        }
    }

    // NativeSerializationProxyState is used by CustomSerializationCloner to store the state of
    // proxy objects which don't implement ISerializable.
    private sealed class NativeSerializationProxyState : State {
        private readonly Type Type_;
        private readonly FieldInfo[] Fields;
        private readonly object[] Values;

        public NativeSerializationProxyState(Type type, CloneEventHandlers eventHandlers)
               : base(eventHandlers, null)
        {
            Type_ = type;
        }

        public NativeSerializationProxyState(Type type, CloneEventHandlers eventHandlers, FieldInfo[] fields, object[] values, int[] objectIndices)
               : base(eventHandlers, objectIndices)
        {
            Debug.Assert(fields.Length == values.Length && values.Length == objectIndices.Length);
            Type_ = type;
            Fields = fields;
            Values = values;
        }

        public override Type Type { get { return Type_; } }

        public override object CreateUninitializedObject() {
            return FormatterServices.GetUninitializedObject(Type_);
        }

        public override void WriteToUninitializedObject(object instance, object[] objectGraph) {
            if (ObjectIndices == null) return;
            // We can't use a NativeSerializationCloner.FieldValuesSetter here
            // because some primitive values might have a type different from the type of the field
            // they are assigned to. FieldInfo.SetValue does some automatic conversions in those
            // cases that the FieldValuesSetter doesn't (e.g. integer type widening).
            for (int i = 0; i < ObjectIndices.Length; ++i) {
                var objectIndex = ObjectIndices[i];
                if (objectIndex == 0) {
                    var value = Values[i];
                    if (value != null) Fields[i].SetValue(instance, value);
                } else {
                    Fields[i].SetValue(instance, objectGraph[objectIndex]);
                }
            }
        }
    }

    private struct CustomSerializationMemberInfo {
        public string Name;
        public Type Type;
        public object Value;
    }

    private sealed class CustomSerializationCloner : Cloner {
        internal readonly ConstructorInfo Constructor;
        internal Action<object, SerializationInfo, StreamingContext> ConstructorCaller; // lazily initalized
        private Cloner PreviousProxyCloner;
        private Cloner[] Cloners;

        private static Type[] SerializableConstructorArgumentTypes = new Type[] {typeof(SerializationInfo), typeof(StreamingContext)};

        public CustomSerializationCloner(Type type,
                                         CloneEventHandlers eventHandlers)
               : base(type, eventHandlers)
        {
            Constructor = type.GetConstructor(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance, null, SerializableConstructorArgumentTypes, null);
            PreviousProxyCloner = this;
        }

        internal override State CaptureShallowStateAndEnqueueNestedState(object instance, CaptureContext captureContext) {
            Debug.Assert(Type == instance.GetType());
            var info = new SerializationInfo(Type, FormatterConverter);
            ((ISerializable)instance).GetObjectData(info, StreamingContext);
            var n = info.MemberCount;
            var members = new CustomSerializationMemberInfo[n];
            var objectIndices = new int[n];
            if (Cloners == null || Cloners.Length != n) Cloners = new Cloner[n];
            var iter = info.GetEnumerator();
            for (int i = 0; iter.MoveNext(); ++i) {
                var entry = iter.Current;
                members[i].Name = entry.Name;
                members[i].Type = entry.ObjectType;
                var value = entry.Value;
                if (value == null) continue;
                Type type = value.GetType();
                if (type.IsPrimitive || type == typeof(string)) {
                    members[i].Value = value;
                    continue;
                }
                var cloner = Cloners[i];
                if (cloner == null || type != cloner.Type) {
                    cloner = CreateWithoutLock(type);
                    Cloners[i] = cloner;
                }
                objectIndices[i] = captureContext.GetObjectIndex(value, cloner);
            }

            Type proxyType;

        #if CLR4
            if (!info.IsFullTypeNameSetExplicit && !info.IsAssemblyNameSetExplicit) {
                proxyType = info.ObjectType;
        #else
            if (info.FullTypeName == Type.FullName && info.AssemblyName == Type.Assembly.FullName) {
                proxyType = Type;
        #endif
            } else {
                try {
                    var assembly = Assembly.Load(info.AssemblyName);
                    proxyType = assembly.GetType(info.FullTypeName, true);
                } catch (Exception e) {
                    var msg = "Can not load the type '" + info.FullTypeName + "' in the assembly '" + info.AssemblyName + "'.";
                    throw new SerializationException(msg, e);
                }
            }

            if (proxyType == Type) {
                if (Constructor == null) throw new SerializationException("The ISerializable type '" + Type.ToString() + "' does not define a proper deserialization constructor.");
                return new CustomSerializationState(this, members, objectIndices);
            }

            Cloner proxyCloner;
            if (proxyType == PreviousProxyCloner.Type) {
                proxyCloner = PreviousProxyCloner;
            } else {
                proxyCloner = CreateWithoutLock(proxyType);
                PreviousProxyCloner = proxyCloner;
            }

            if (proxyType.IsArray) {
                // On .NET a NullReferenceException is thrown on deserialization of an array type proxy.
                throw new SerializationException("The type '" + Type.ToString() + "' uses an array type ('" + proxyType.ToString() +  "') as its serialization proxy type.");
            }

            CustomSerializationCloner csc = proxyCloner as CustomSerializationCloner;
            if (csc != null) {
                if (csc.Constructor == null) throw new SerializationException("The ISerializable type '" + csc.Type.ToString() + "' does not define a proper deserialization constructor.");
                return new CustomSerializationState(csc, members, objectIndices);
            }

            if (n == 0) return new NativeSerializationProxyState(proxyType, proxyCloner.EventHandlers);

            FieldInfo[] proxyFields;
            {
                var nsc = proxyCloner as NativeSerializationCloner;
                if (nsc != null) {
                    proxyFields = nsc.SerializedFields;
                } else {
                    var bc = proxyCloner as BlittableCloner;
                    Debug.Assert(bc != null);
                    proxyFields = bc.SerializedFields;
                }
            }

            // The BinaryFormatter on .NET simply assigns the values in the SerializationInfo
            // to the field with the same name (of the most derived class) in the proxy object.
            // There are no checks whether all fields are assigned values or whether the target has
            // multiple fields with the same name. The types are only checked once the values are
            // assigned to the proxy object fields. Integer types are automatically widened and
            // types are cast to base or interface types if necessary.

            var proxyValues = new object[proxyFields.Length];
            var proxyObjectIndices = new int[proxyFields.Length];
            for (int i = 0; i < n; ++i) {
                var name = members[i].Name;
                for (int j = 0; j < proxyFields.Length; ++j) {
                    if (name == proxyFields[j].Name) {
                        proxyValues[j] = members[i].Value;
                        proxyObjectIndices[j] = objectIndices[i];
                        break;
                    }
                }
            }
            return new NativeSerializationProxyState(proxyType, proxyCloner.EventHandlers, proxyFields, proxyValues, proxyObjectIndices);
        }
    }

    private sealed class CustomSerializationState : State {
        private readonly CustomSerializationCloner Cloner;
        private readonly CustomSerializationMemberInfo[] Members;

        public CustomSerializationState(CustomSerializationCloner cloner,
                                       CustomSerializationMemberInfo[] members,
                                       int[] objectIndices)
               : base(cloner.EventHandlers, objectIndices)
        {
            Cloner = cloner;
            Members = members;
        }

        public override Type Type { get { return Cloner.Type; } }

        public override object CreateUninitializedObject() {
            return FormatterServices.GetUninitializedObject(Cloner.Type);
        }

        public override void WriteToUninitializedObject(object instance, object[] objectGraph) {
            var info = new SerializationInfo(Cloner.Type, FormatterConverter);
            for (int i = 0; i < Members.Length; ++i) {
                var member = Members[i];
                var index = ObjectIndices[i];
                var value = index == 0 ? member.Value : objectGraph[index];
                info.AddValue(member.Name, value, member.Type);
            }
            var constructorCaller = Cloner.ConstructorCaller;
            if (constructorCaller == null)
                Cloner.ConstructorCaller = constructorCaller = CreateISerializableConstructorCaller(Cloner.Constructor);
            constructorCaller(instance, info, StreamingContext);
        }
    }

    private sealed class SimpleImage : CloneImage {
        private readonly Cloner.State[] States;
        private readonly int DeserializationCallbackCount;

        internal SimpleImage(Cloner.State[] states, int deserializationCallbackCount) {
            Debug.Assert(states.Length > 1 && states[0] == null);
            States = states;
            DeserializationCallbackCount = deserializationCallbackCount;
        }

        public override object CreateClone() {
            int callbackIndicesIndex = DeserializationCallbackCount;
            int[] callbackIndices =
                DeserializationCallbackCount == 0 ? null : new int[DeserializationCallbackCount];
            var objects = new object[States.Length];
            // States[0] is null
            for (int i = 1; i < States.Length; ++i)
                objects[i] = States[i].CreateUninitializedObject();
            for (int index = States.Length - 1; index != 0; --index) {
                var state = States[index];
                var instance = objects[index];
                var eventHandlers = state.EventHandlers;
                if (eventHandlers == null) {
                    state.WriteToUninitializedObject(objects[index], objects);
                } else {
                    var events = eventHandlers.Events;
                    Debug.Assert((events & (  CloneEvents.ISerializable
                                            | CloneEvents.OnDeserialized
                                            | CloneEvents.IObjectReference)) == 0);
                    if ((events & CloneEvents.OnDeserializing) != 0)
                        eventHandlers.InvokeOnDeserializing(instance, Cloner.StreamingContext);
                    if ((events & CloneEvents.IDeserializationCallback) != 0)
                        callbackIndices[--callbackIndicesIndex] = index;
                    state.WriteToUninitializedObject(instance, objects);
                }
            }
            if (callbackIndices != null) {
                Debug.Assert(callbackIndicesIndex == 0);
                foreach (var index in callbackIndices)
                    ((IDeserializationCallback)objects[index]).OnDeserialization(null);
            }
            return objects[1];
        }
    }

    private sealed class OrderedImage : CloneImage {
        private readonly Cloner.State[] States;
        private readonly int[] Order;
        private readonly int DeserializationCallbackCount;

        internal OrderedImage(Cloner.State[] states, int[] order, int deserializationCallbackCount) {
            Debug.Assert(states.Length > 1 && states.Length == order.Length && states[0] == null);
            States = states;
            Order = order;
            DeserializationCallbackCount = deserializationCallbackCount;
        }

        public static object GetRealObject(object instance) {
            var or = (IObjectReference)instance;
            instance = or.GetRealObject(Cloner.StreamingContext);
            if (instance != or) {
                or = instance as IObjectReference;
                int i = 0;
                while (or != null) {
                    if (++i == 100) throw new SerializationException("An object's implementation of the IObjectReference interface returned too many nested references to other objects that implement IObjectReference.");
                    instance = or.GetRealObject(Cloner.StreamingContext);
                    if (instance == or) break;
                    or = instance as IObjectReference;
                }
                if (instance == null) throw new SerializationException("An object's IObjectReference.GetRealObject implementation returned null.");
            }
            return instance;
        }

        public override object CreateClone() {
            int callbackIndicesIndex = DeserializationCallbackCount;
            object[] callbackObjects =
                DeserializationCallbackCount == 0 ? null : new object[DeserializationCallbackCount];
            var objects = new object[States.Length];
            for (int i = 1; i < States.Length; ++i)
                objects[i] = States[i].CreateUninitializedObject();
            var delayedOnDeserializedEvents = new List<int>();
            int objectReferenceIndex = 0;
            object objectReference = null;
            int[] lastScc = null;
            for (int i = Order.Length - 1; i != 0; --i) {
                var index = Order[i];
                var state = States[index];
                var scc = state.StronglyConnectedComponent;
                if (scc != lastScc) {
                    lastScc = scc;
                    if (objectReference != null) {
                        ReplaceObjectReferenceInSCCWithRealObject(objectReference, objectReferenceIndex, objects);
                        objectReferenceIndex = 0;
                        objectReference = null;
                    }
                    if (delayedOnDeserializedEvents.Count != 0)
                        InvokeDelayedOnDeserializedEvents(delayedOnDeserializedEvents, objects); // also clears delayedOnDeserializedEvents
                    if (scc != null) {
                        foreach (var idx in scc)  {
                            var handlers = States[idx].EventHandlers;
                            if (handlers != null && (handlers.Events & CloneEvents.IObjectReference) != 0) {
                                objectReferenceIndex = idx;
                                objectReference = objects[idx];
                                objects[idx] = null; // set to null until we call ReplaceObjectReferenceInSCCWithRealObject
                            }
                        }
                    }
                }
                var instance = objects[index];
                var eventHandlers = state.EventHandlers;
                if (eventHandlers == null) {
                    state.WriteToUninitializedObject(instance, objects);
                } else {
                    var events = eventHandlers.Events;
                    if (instance != null) {
                        if ((events & CloneEvents.OnDeserializing) != 0)
                            eventHandlers.InvokeOnDeserializing(instance, Cloner.StreamingContext);
                        state.WriteToUninitializedObject(instance, objects);
                        if ((events & CloneEvents.OnDeserialized) != 0) {
                            if (scc == null) eventHandlers.InvokeOnDeserialized(instance, Cloner.StreamingContext);
                            else delayedOnDeserializedEvents.Add(index);
                        }
                        if ((events & CloneEvents.IObjectReference) != 0) {
                            Debug.Assert(state.StronglyConnectedComponent == null);
                            objects[index] = GetRealObject(instance);
                        }
                    } else {
                        Debug.Assert(index == objectReferenceIndex);
                    }
                    // It's a pity we have to process the IDeserializationCallback separately
                    // from OnDeserialized events to stay compatible with the .NET BinaryFormatter.
                    if ((events & CloneEvents.IDeserializationCallback) != 0)
                        callbackObjects[--callbackIndicesIndex] = instance ?? objectReference;
                }
            }
            if (objectReference != null)
                ReplaceObjectReferenceInSCCWithRealObject(objectReference, objectReferenceIndex, objects);
            if (delayedOnDeserializedEvents.Count != 0)
                InvokeDelayedOnDeserializedEvents(delayedOnDeserializedEvents, objects);
            if (callbackObjects != null) {
                Debug.Assert(callbackIndicesIndex == 0);
                // We call the callback in in the reverse topological order at the end of
                // deserialization, which is similar to what the BinaryFormatter does, unfortunately.
                foreach (var obj in callbackObjects)
                    ((IDeserializationCallback)obj).OnDeserialization(null);
            }
            return objects[1];
        }

        private void InvokeDelayedOnDeserializedEvents(List<int> indices, object[] objects) {
            foreach (var index in indices) {
                var handlers = States[index].EventHandlers;
                handlers.InvokeOnDeserialized(objects[index], Cloner.StreamingContext);
            }
            indices.Clear();
        }

        private void ReplaceObjectReferenceInSCCWithRealObject(object objectReference, int objectReferenceIndex, object[] objects) {
            var state = States[objectReferenceIndex];
            var eventHandlers = state.EventHandlers;
            var events = eventHandlers.Events;
            if ((events & CloneEvents.OnDeserializing) != 0)
                eventHandlers.InvokeOnDeserializing(objectReference, Cloner.StreamingContext);
            state.WriteToUninitializedObject(objectReference, objects);
            Debug.Assert((events & CloneEvents.OnDeserialized) == 0);
            objects[objectReferenceIndex] = GetRealObject(objectReference);
            // set all references to real object
            foreach (var index2 in state.StronglyConnectedComponent) {
                if (index2 == objectReferenceIndex) continue;
                var state2 = States[index2];
                Debug.Assert(state2.EventHandlers == null || (state2.EventHandlers.Events & (CloneEvents.ISerializable | CloneEvents.IObjectReference)) == 0);
                if (Cloner.Contains(state2.ObjectIndices, objectReferenceIndex)) {
                    Debug.Assert(!state2.Type.IsValueType);
                    state2.WriteToUninitializedObject(objects[index2], objects); // overwrite all fields
                }
            }
        }
    }

    /// <summary>Returns the public and non-public fields of the type (and its base types),
    /// except fields with the NonSerialized attribute. In the returned array fields from
    /// derived types come before fields from base types.</summary>
    internal static FieldInfo[] GetSerializedFields(Type type, out bool typeIsBlittable) {
        Debug.Assert(type.IsSerializable && !type.IsInterface);
        // We need the fields of the most derived type first, but GetFields returns the
        // field in an undefined order, so we have to climb the type hierarchy.
        var fields = type.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly);
        bool isBlittable = true;
        int nonSerialized = 0;
        foreach (var f in fields) {
            if (f.IsNotSerialized) ++nonSerialized;
            var ft = f.FieldType;
            if (!ft.IsPrimitive && ft != typeof(string)) {
                if (!ft.IsValueType) isBlittable = false;
                else {
                    bool fIsBlittable;
                    GetSerializedFields(ft, out fIsBlittable);
                    isBlittable &= fIsBlittable;
                }
            }
        }
        int numberOfBases = 0;
        var bt = type.BaseType;
        while (bt != null && bt != typeof(object)) {
            if (!bt.IsSerializable)
                throw new SerializationException(BaseTypeNotSerializableMessage(bt, type));
            ++numberOfBases;
            bt = bt.BaseType;
        }
        if (numberOfBases == 0) {
            if (nonSerialized == 0) {
                typeIsBlittable = isBlittable;
                return fields;
            } else {
                typeIsBlittable = false;
                var serializedFields = new FieldInfo[fields.Length - nonSerialized];
                int i = 0;
                foreach (var f in fields) if (!f.IsNotSerialized) serializedFields[i++] = f;
                return serializedFields;
            }
        } else {
            var baseFieldArrays = new FieldInfo[numberOfBases][];
            bt = type.BaseType;
            for (int i = 0; i < numberOfBases; ++i, bt = bt.BaseType) {
                var baseFields = bt.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly);
                foreach (var bf in baseFields) {
                    if (bf.IsNotSerialized) ++nonSerialized;
                    var bft = bf.FieldType;
                    if (!bft.IsPrimitive && bft != typeof(string)) {
                        if (!bft.IsValueType) isBlittable = false;
                        else {
                            bool bfIsBlittable;
                            GetSerializedFields(bft, out bfIsBlittable);
                            isBlittable &= bfIsBlittable;
                        }
                    }
                }
                baseFieldArrays[i] = baseFields;
            }

            typeIsBlittable = nonSerialized == 0 & isBlittable;

            var numberOfSerializedFields = fields.Length - nonSerialized;
            foreach (var baseFields in baseFieldArrays) numberOfSerializedFields += baseFields.Length;

            if (nonSerialized == 0 && numberOfSerializedFields == fields.Length) return fields;

            var combinedFields = new FieldInfo[numberOfSerializedFields];
            if (nonSerialized == 0) {
                int i = 0;
                foreach (var f in fields) combinedFields[i++] = f;
                foreach (var baseFields in baseFieldArrays)
                    foreach (var bf in baseFields) combinedFields[i++] = bf;
            } else {
                int i = 0;
                foreach (var f in fields)
                    if (!f.IsNotSerialized) combinedFields[i++] = f;
                foreach (var baseFields in baseFieldArrays)
                    foreach (var bf in baseFields)
                        if (!bf.IsNotSerialized) combinedFields[i++] = bf;
            }
            return combinedFields;
        }
    }

    internal static string BaseTypeNotSerializableMessage(Type baseType, Type childType) {
        return "The serializable type '" + childType.ToString() + "' has a base type '" + baseType.ToString() + "' that is not serializable.";
    }

    /*
    private static object[] GetFieldValues(object instance, FieldInfo[] fields) {
        var values = new object[fields.Length];
        for (int i = 0; i < fields.Length; ++i) {
            var f = fields[i];
            values[i] = f.GetValue(instance);
        }
        return values;
    }
    */
    internal static Func<object, object[]> CreateFieldValuesGetter(Type type, FieldInfo[] fields) {
        if (fields.Length == 0) throw new ArgumentException("The fields array must be non-empty.");

        var dynamicMethod = new DynamicMethod("FieldValuesGetter",
                                               MethodAttributes.Public | MethodAttributes.Static,
                                               CallingConventions.Standard,
                                               typeof(object[]), new Type[]{typeof(object), typeof(object)},
                                               type, true);
        var ilg = dynamicMethod.GetILGenerator();
        var isValueType = type.IsValueType;

        // arg 0: dummy argument (makes delegate invocation faster)
        // arg 1: (boxed) object instance

        ilg.DeclareLocal(typeof(object[])); // local 0: the returned values array
        ilg.DeclareLocal(typeof(object)); // local 1: temporary object value

        // create the values array
        ilg.Emit(OpCodes.Ldc_I4, fields.Length);
        ilg.Emit(OpCodes.Newarr, typeof(object));
        ilg.Emit(OpCodes.Stloc_0);

        // cast/unbox the object instace
        ilg.Emit(OpCodes.Ldarg_1);
        if (!isValueType)
            ilg.Emit(OpCodes.Castclass, type);
        else
            ilg.Emit(OpCodes.Unbox, type);

        // The unbox IL construction doesn't return a normal managed pointer
        // but a "controlled-mutability" managed pointer. Since there's no way
        // to declare a controlled-mutability managed pointer local and one
        // can't convert such a pointer into a normal managed pointer, we can't
        // store away the pointer for later field accesses. Instead we use
        // OpCodes.Dup to keep the pointer around. Alternatively we could copy
        // the value type instance onto the stack, but that can be costly for
        // large value types.

        for (int i = 0; i < fields.Length; ++i) {
            if (i + 1 != fields.Length) ilg.Emit(OpCodes.Dup);

            var field = fields[i];
            ilg.Emit(OpCodes.Ldfld, field);
            if (field.FieldType.IsValueType)
                ilg.Emit(OpCodes.Box, field.FieldType);
            ilg.Emit(OpCodes.Stloc_1);

            // store object into result array
            ilg.Emit(OpCodes.Ldloc_0);
            ilg.Emit(OpCodes.Ldc_I4, i);
            ilg.Emit(OpCodes.Ldloc_1);
            ilg.Emit(OpCodes.Stelem_Ref);
        }

        ilg.Emit(OpCodes.Ldloc_0);
        ilg.Emit(OpCodes.Ret);

        return (Func<object, object[]>)dynamicMethod.CreateDelegate(typeof(Func<object, object[]>), null);
    }

    /*
    private static void SetFieldValues(FieldInfo[] fields, object instance, object[] values, int[] objectIndices, object[] objectGraph) {
        for (int i = 0; i < objectIndices.Length; ++i) {
            var objectIndex = ObjectIndices[i];
            if (objectIndex == 0)
                fields[i].SetValue(instance, values[i]);
            else
                fields[i].SetValue(instance, objectGraph[objectIndex]);
        }
    }
    */
    internal static Action<object, object[], int[], object[]> CreateFieldValuesSetter(Type type, FieldInfo[] fields) {
        if (fields.Length == 0) throw new ArgumentException("The fields array must be non-empty.");

        // It is important that we use the 8 argument DynamicMethod constructor
        // to associate the method with the type, so that the method is allowed
        // to set readonly (initonly) fields.
        var dynamicMethod = new DynamicMethod("FieldValuesSetter",
                                              MethodAttributes.Public | MethodAttributes.Static, CallingConventions.Standard,
                                              null, new Type[]{typeof(object), typeof(object), typeof(object[]), typeof(int[]),
                                              typeof(object[])},
                                              type, true);
        var ilg = dynamicMethod.GetILGenerator();
        var isValueType = type.IsValueType;

        // arg0: dummy argument (makes delegate invocation faster)
        // arg1: (boxed) object instance
        // arg2: values array
        // arg3: objectIndices array
        // arg4: objectGraph array

        // local 0: object index
        ilg.DeclareLocal(typeof(int));

        ilg.Emit(OpCodes.Ldarg_1);
        if (!isValueType)
            ilg.Emit(OpCodes.Castclass, type);
        else
            ilg.Emit(OpCodes.Unbox, type); // returns a controlled-mutability pointer
                                          // which we can't store in a local...
        for (int i = 0; i < fields.Length; ++i) {
            if (i + 1 != fields.Length) ilg.Emit(OpCodes.Dup); // ... so we use OpCodes.Dup to keep it around

            var field = fields[i];

            // is field value an object in the object graph array?
            ilg.Emit(OpCodes.Ldarg_3);
            ilg.Emit(OpCodes.Ldc_I4, i);
            ilg.Emit(OpCodes.Ldelem, typeof(int));
            ilg.Emit(OpCodes.Stloc_0);
            ilg.Emit(OpCodes.Ldloc_0);
            var label1 = ilg.DefineLabel();
            ilg.Emit(OpCodes.Brtrue, label1);

            // load boxed value
            ilg.Emit(OpCodes.Ldarg_2);
            ilg.Emit(OpCodes.Ldc_I4, i);
            ilg.Emit(OpCodes.Ldelem, typeof(object));
            var label2 = ilg.DefineLabel();
            ilg.Emit(OpCodes.Br, label2);

            // load object graph array
            ilg.MarkLabel(label1);
            ilg.Emit(OpCodes.Ldarg, 4);
            ilg.Emit(OpCodes.Ldloc_0);
            ilg.Emit(OpCodes.Ldelem, typeof(object));

            ilg.MarkLabel(label2);
            // store value into field
            if (field.FieldType != typeof(object))
                ilg.Emit(OpCodes.Unbox_Any, field.FieldType);
            ilg.Emit(OpCodes.Stfld, field);
        }

        ilg.Emit(OpCodes.Ret);

        return (Action<object, object[], int[], object[]>)dynamicMethod.CreateDelegate(typeof(Action<object, object[], int[], object[]>), null);
    }

    internal static Action<object, SerializationInfo, StreamingContext> CreateISerializableConstructorCaller(ConstructorInfo constructor) {
        var type = constructor.DeclaringType;
        var dynamicMethod = new DynamicMethod("SerializableConstructorCaller",
                                              MethodAttributes.Public | MethodAttributes.Static, CallingConventions.Standard,
                                              null, new Type[]{typeof(object), typeof(object), typeof(SerializationInfo), typeof(StreamingContext)},
                                              type, true);
        var ilg = dynamicMethod.GetILGenerator();
        var isValueType = type.IsValueType;
        ilg.Emit(OpCodes.Ldarg_1);
        if (!isValueType)
            ilg.Emit(OpCodes.Castclass, type);
        else
            ilg.Emit(OpCodes.Unbox, type);
        ilg.Emit(OpCodes.Ldarg_2);
        ilg.Emit(OpCodes.Ldarg_3);
        ilg.Emit(OpCodes.Call, constructor);
        ilg.Emit(OpCodes.Ret);
        return (Action<object, SerializationInfo, StreamingContext>)dynamicMethod.CreateDelegate(typeof(Action<object, SerializationInfo, StreamingContext>), null);
    }

    // The following is a non-recursive implementation of <NAME>'s improved
    // version of Tarjan's algorithm for finding the strongly connected components of
    // a directed graph, see http://homepages.ecs.vuw.ac.nz/~djp/files/P05.pdf
    // The straighforward recursive version is obviously more elegant, but the
    // non-recursive one has the principal advantage of not ending in a stack overflow
    // for large components.
    // (We test this version against the simpler one in CloningTests.fs, of course)
    // Due to the non-recursive implementation we can also exploit that part of
    // what would otherwise be the call stack can be shared with the stack used
    // for holding elements of identified components (see the last paragraph of
    // section 2 in the referenced paper).

    // For optimization purposes we use a static stack, which makes
    // FindStronglyConnectedComponents and ComputeTopologicalOrder not thread-safe.

    private static int[] TopoIndices = new int[8];
    private static void GrowTopoIndices() {
        var newArray = new int[2*TopoIndices.Length];
        TopoIndices.CopyTo(newArray, 0);
        TopoIndices = newArray;
    }
    private static int GrowTopoIndices(int splitIndex) {
        Debug.Assert(splitIndex >= 0 && splitIndex <= TopoIndices.Length);
        int n = TopoIndices.Length;
        var newArray = new int[2*n];
        Array.Copy(TopoIndices, newArray, splitIndex);
        var newSplitIndex = 2*n;
        int d = n - splitIndex;
        if (d != 0) {
            newSplitIndex -= d;
            Array.Copy(TopoIndices, splitIndex, newArray, newSplitIndex, n - splitIndex);
        }
        TopoIndices = newArray;
        return newSplitIndex;
    }

    private static int[] TopoSubIndices = new int[8];
    private static void GrowTopoSubIndices() {
        var newArray = new int[2*TopoSubIndices.Length];
        TopoSubIndices.CopyTo(newArray, 0);
        TopoSubIndices = newArray;
    }

    /// <summary>Fills the Strongly StronglyConnectedComponent fields of the
    /// states passed in the array. Returns an array mapping each state to an
    /// integer component identifier.
    /// </summary>
    /// <param name="states">The object states to traverse. The object with array index
    /// 0 is ignored. All other objects are assumed to be reachable from the object
    /// with array index 1.</param>
    internal static int[] FindStronglyConnectedComponents(State[] states) {
        Debug.Assert(states.Length > 1);
        int[] components = new int[states.Length];
        // The path stack and the component stack are both stored in TopoIndices.
        // The path stack starts at the beginning of TopoIndices, while
        // the component stack starts at the end and progresses in reverse direction.
        int pathStackCount = 0; // number of elements in the path stack
        int componentStackIndex = TopoIndices.Length; // index of element last inserted into component stack
        int counter = 1; // in the paper this variable is called "index"
        int reverseCounter = states.Length - 1; // in the paper this variable is called "C"
        bool root = true;
        int objectIndex = 1; // states[1] is state for the root object, states[0] is null
        int subIndex = 0;
        var subObjectIndices = states[objectIndex].ObjectIndices;
        components[1] = counter;
        ++counter;
        if (subObjectIndices != null) {
            for (;;) {
                while (subIndex < subObjectIndices.Length) {
                    var subObjectIndex = subObjectIndices[subIndex];
                    ++subIndex;
                    if (subObjectIndex == 0) continue;
                    var subObjectComponent = components[subObjectIndex];
                    if (subObjectComponent == 0) {
                        var subSubObjectIndices = states[subObjectIndex].ObjectIndices;
                        if (subSubObjectIndices == null) {
                            components[subObjectIndex] = reverseCounter;
                            --reverseCounter;
                        } else {
                            subObjectIndices = subSubObjectIndices;
                            components[subObjectIndex] = counter;
                            ++counter;
                            TopoIndices[pathStackCount] = objectIndex;
                            TopoSubIndices[pathStackCount] = root ? subIndex : -subIndex;
                            root = true;
                            objectIndex = subObjectIndex;
                            subIndex = 0;
                            ++pathStackCount;
                            if (pathStackCount == componentStackIndex)
                                componentStackIndex = GrowTopoIndices(componentStackIndex);
                            if (pathStackCount == TopoSubIndices.Length)
                                GrowTopoSubIndices();
                            continue;
                        }
                    } else if (subObjectComponent < components[objectIndex]) {
                        components[objectIndex] = subObjectComponent;
                        root = false;
                    }
                }
                if (root) {
                    if (componentStackIndex < TopoIndices.Length) {
                        int component = components[objectIndex];
                        if (components[TopoIndices[componentStackIndex]] >= component) {
                            int next = componentStackIndex + 1;
                            while (next < TopoIndices.Length && components[TopoIndices[next]] >= component) ++next;
                            int d = next - componentStackIndex;
                            var scc = new int[d + 1];
                            for (int i = 0; i < d; ++i) {
                                int idx = TopoIndices[componentStackIndex + i];
                                scc[1 + i] = idx;
                                states[idx].StronglyConnectedComponent = scc;
                                components[idx] = reverseCounter;
                                --counter;
                            }
                            scc[0] = objectIndex;
                            states[objectIndex].StronglyConnectedComponent = scc;
                            componentStackIndex = next;
                        }
                    }
                    components[objectIndex] = reverseCounter;
                    --counter;
                    --reverseCounter;
                    if (pathStackCount == 0) break;
                } else {
                    TopoIndices[--componentStackIndex] = objectIndex;
                    // we never need to grow the TopoIndices array here
                    // because we immediately decrement pathStackCount next
                }
                --pathStackCount;
                int subObjectComponent_ = components[objectIndex];
                objectIndex = TopoIndices[pathStackCount];
                subIndex = TopoSubIndices[pathStackCount];
                if (subIndex > 0) {
                    root = true;
                } else {
                    subIndex = -subIndex;
                    root = false;
                }
                subObjectIndices = states[objectIndex].ObjectIndices;
                if (subObjectComponent_ < components[objectIndex]) {
                    components[objectIndex] = subObjectComponent_;
                    root = false;
                }
            }
        }
        return components;
    }

    private static int[] SccIndexStack = new int[8];
    private static void GrowSccIndexStack() {
        var newStack = new int[2*SccIndexStack.Length];
        SccIndexStack.CopyTo(newStack, 0);
        SccIndexStack = newStack;
    }

    /// <summary>Returns an array with the topologically sorted indices of the states.
    /// In the returned array the indices of states belonging to the same strongly
    /// connected component are adjacent (but the order within a strongly connected
    /// component is undefined).
    /// </summary>
    /// <param name="states">The object states to traverse. The object with array index
    /// 0 is ignored. All other objects are assumed to be reachable from the object
    /// with array index 1.</param>
    internal static int[] ComputeTopologicalOrder(State[] states) {
        Debug.Assert(states.Length > 1);

        // Fill the State.StronglyConnectedComponent fields.
        // (We don't need the returned array, so we can recycle it for our purposes.)
        int[] orderedObjectIndices = FindStronglyConnectedComponents(states);
        Array.Clear(orderedObjectIndices, 0, orderedObjectIndices.Length);
        int nextPosition = orderedObjectIndices.Length - 1;

        TopoIndices = new int[2];
        TopoSubIndices = new int[2];
        SccIndexStack = new int[2];

        // We traverse the graph non-recursively in depth-first order.

        int topoStackCount = 0;
        int sccIndexStackCount = 0;

        int[] visitedBits = new int[(checked(states.Length + 31))/32];

        int objectIndex;
        int[] subObjectIndices;
        {
            var state = states[1];
            if (state.StronglyConnectedComponent == null) {
                objectIndex = 1;
                subObjectIndices = state.ObjectIndices;
                if (subObjectIndices == null) {
                    orderedObjectIndices[1] = 1;
                    return orderedObjectIndices;
                }
                visitedBits[0] = 1 << 1;
            } else {
                foreach (var sccIndex in state.StronglyConnectedComponent)
                    visitedBits[sccIndex/32] |= 1 << (sccIndex%32);
                objectIndex = state.StronglyConnectedComponent[0];
                subObjectIndices = states[objectIndex].ObjectIndices;
                SccIndexStack[0] = 1;
                sccIndexStackCount = 1;
            }
        }
        int subIndex = subObjectIndices.Length - 1;

        for (;;) {
            // First we iterate over the sub objects...
            Debug.Assert(subObjectIndices != null);

            // (The states array was constructed in breadth-first order, while we construct
            //  the topological order using depth-first search. With a bit of luck we can
            //  keep the resulting orderedObjectIndices close to a simple increasing sequence
            //  by iterating over the sub-objects in the depth-first search in reverse order.)
            while (subIndex >= 0) {
                var subObjectIndex = subObjectIndices[subIndex];
                --subIndex;
                if (subObjectIndex == 0) continue;
                int w = subObjectIndex/32, b = subObjectIndex%32;
                if (((visitedBits[w] >> b) & 1) == 0) {
                    var subState = states[subObjectIndex];
                    var subSubObjectIndices = subState.ObjectIndices;
                    if (subState.StronglyConnectedComponent == null) {
                        visitedBits[w] |= 1 << b;
                        if (subSubObjectIndices == null) {
                            orderedObjectIndices[nextPosition] = subObjectIndex;
                            --nextPosition;
                            continue;
                        }
                        subObjectIndices = subSubObjectIndices;
                    } else {
                        foreach (var sccIndex in subState.StronglyConnectedComponent)
                            visitedBits[sccIndex/32] |= 1 << (sccIndex%32);
                        subObjectIndex = subState.StronglyConnectedComponent[0];
                        subObjectIndices = states[subObjectIndex].ObjectIndices;
                        SccIndexStack[sccIndexStackCount] = 1;
                        if (++sccIndexStackCount == SccIndexStack.Length) GrowSccIndexStack();
                    }
                    TopoIndices[topoStackCount] = objectIndex;
                    TopoSubIndices[topoStackCount] = subIndex;
                    ++topoStackCount;
                    if (topoStackCount == TopoIndices.Length) GrowTopoIndices();
                    if (topoStackCount == TopoSubIndices.Length) GrowTopoSubIndices();
                    objectIndex = subObjectIndex;
                    subIndex = subObjectIndices.Length - 1;
                    continue;
                }
            }

            // ... then we iterate over other object in the same strongly connected component.
            var scc = states[objectIndex].StronglyConnectedComponent;
            if (scc == null) {
                orderedObjectIndices[nextPosition] = objectIndex;
                --nextPosition;
            } else {
                Debug.Assert(sccIndexStackCount > 0);
                int sccIndex = SccIndexStack[sccIndexStackCount - 1];
                if (sccIndex < scc.Length) {
                    objectIndex = scc[sccIndex];
                    subObjectIndices = states[objectIndex].ObjectIndices;
                    subIndex = subObjectIndices.Length - 1;
                    SccIndexStack[sccIndexStackCount - 1] = ++sccIndex;
                    continue;
                }
                --sccIndexStackCount;
                for (int i = scc.Length - 1; i >= 0; --i) {
                    sccIndex = scc[i];
                    orderedObjectIndices[nextPosition] = sccIndex;
                    --nextPosition;
                }
            }
            if (topoStackCount == 0) break;
            --topoStackCount;
            objectIndex = TopoIndices[topoStackCount];
            subIndex = TopoSubIndices[topoStackCount];
            subObjectIndices = states[objectIndex].ObjectIndices;
        }
        return orderedObjectIndices;
    }
}

[Flags]
internal enum CloneEvents {
    None = 0,
    OnSerializing = 1,
    OnSerialized = 2,
    OnDeserializing = 4,
    OnDeserialized = 8,
    ISerializable = 16,
    IDeserializationCallback = 32,
    IObjectReference = 64
}

internal sealed class CloneEventHandlers {
    public readonly CloneEvents Events;

    private delegate void Handler(object instance, StreamingContext context);

    private readonly Handler OnSerializingHandler;
    private readonly Handler OnSerializedHandler;
    private readonly Handler OnDeserializingHandler;
    private readonly Handler OnDeserializedHandler;

    private CloneEventHandlers(CloneEvents events,
                               Handler onSerializingHandler,
                               Handler onSerializedHandler,
                               Handler onDeserializingHandler,
                               Handler onDeserializedHandler)
    {
        Events = events;
        OnSerializingHandler = onSerializingHandler;
        OnSerializedHandler = onSerializedHandler;
        OnDeserializingHandler = onDeserializingHandler;
        OnDeserializedHandler = onDeserializedHandler;
    }

    public void InvokeOnSerializing(object instance, StreamingContext context) {
        OnSerializingHandler.Invoke(instance, context);
    }

    public void InvokeOnSerialized(object instance, StreamingContext context) {
        OnSerializedHandler.Invoke(instance, context);
    }

    public void InvokeOnDeserializing(object instance, StreamingContext context) {
        OnDeserializingHandler.Invoke(instance, context);
    }

    public void InvokeOnDeserialized(object instance, StreamingContext context) {
        OnDeserializedHandler.Invoke(instance, context);
    }

    private static readonly CloneEventHandlers ISerializableOnly = new CloneEventHandlers(CloneEvents.ISerializable, null, null, null, null);
    private static readonly CloneEventHandlers ISerializableAndObjectReferenceOnly = new CloneEventHandlers(CloneEvents.ISerializable | CloneEvents.IObjectReference, null, null, null, null);

    private static Handler WithBoxedArgument<T>(Action<T,StreamingContext> handler) {
        return (object obj, StreamingContext context) => handler((T)obj, context);
    }
    private static readonly MethodInfo WithBoxedArgumentMethodInfo = typeof(CloneEventHandlers).GetMethod("WithBoxedArgument", BindingFlags.Static | BindingFlags.NonPublic);

    private static Handler CreateHandler(Type type, MethodInfo mi) {
        var delegateType = typeof(Action<,>).MakeGenericType(type, typeof(StreamingContext));
        var d = Delegate.CreateDelegate(delegateType, null, mi);
        return (Handler)WithBoxedArgumentMethodInfo.MakeGenericMethod(type).Invoke(null, new object[]{d});
    }

    private static readonly Type typeofObject                   = typeof(object);
    private static readonly Type typeofISerializable            = typeof(ISerializable);
    private static readonly Type typeofIObjectReference         = typeof(IObjectReference);
    private static readonly Type typeofIDeserializationCallback = typeof(IDeserializationCallback);
    private static readonly Type typeofOnSerializingAttribute   = typeof(OnSerializingAttribute);
    private static readonly Type typeofOnSerializedAttribute    = typeof(OnSerializedAttribute);
    private static readonly Type typeofOnDeserializingAttribute = typeof(OnDeserializingAttribute);
    private static readonly Type typeofOnDeserializedAttribute  = typeof(OnDeserializedAttribute);

    public static CloneEventHandlers Create(Type type) {
        Debug.Assert(type != null);
        if (type == typeofObject) return null;
        var events = CloneEvents.None;
        if (typeofISerializable.IsAssignableFrom(type)) events |= CloneEvents.ISerializable;
        if (typeofIObjectReference.IsAssignableFrom(type)) events |= CloneEvents.IObjectReference;
        if (typeofIDeserializationCallback.IsAssignableFrom(type)) events |= CloneEvents.IDeserializationCallback;
        var bt = type;
        for (;;) {
            var methods = bt.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly);
            for (int i = 0; i < methods.Length; ++i) {
                var mi = methods[i];
                if (   mi.IsDefined(typeofOnSerializingAttribute, false)
                    || mi.IsDefined(typeofOnSerializedAttribute, false)
                    || mi.IsDefined(typeofOnDeserializingAttribute, false)
                    || mi.IsDefined(typeofOnDeserializedAttribute, false)) return CreateContinue(type, events, bt, methods, i);
            }
            bt = bt.BaseType;
            if (bt == null || bt == typeofObject) break;
            if (!bt.IsSerializable) throw new SerializationException(Cloner.BaseTypeNotSerializableMessage(bt, type));
       }
       if (events == 0) return null;
       if (events == CloneEvents.ISerializable) return ISerializableOnly;
       if (events == (CloneEvents.ISerializable | CloneEvents.IObjectReference)) return ISerializableAndObjectReferenceOnly;
       return new CloneEventHandlers(events, null, null, null, null);
    }
    private static CloneEventHandlers CreateContinue(Type type, CloneEvents events, Type baseType, MethodInfo[] methods, int i) {
        Delegate onSerializingHandler = null, onSerializedHandlers = null, onDeserializingHandlers = null, onDeserializedHandlers = null;
        var bt = baseType;
        for (;;) {
            for (; i < methods.Length; ++i) {
                var mi = methods[i];
                if (mi.IsDefined(typeofOnSerializingAttribute, false)) {
                    var d = CreateHandler(bt, mi);
                    onSerializingHandler = onSerializingHandler == null ? d : Delegate.Combine(d, onSerializingHandler); // call base handler first
                }
                if (mi.IsDefined(typeofOnSerializedAttribute, false)) {
                    var d = CreateHandler(bt, mi);
                    onSerializedHandlers = onSerializedHandlers == null ? d : Delegate.Combine(d, onSerializedHandlers);
                }
                if (mi.IsDefined(typeofOnDeserializingAttribute, false)) {
                    var d = CreateHandler(bt, mi);
                    onDeserializingHandlers = onDeserializingHandlers == null ? d : Delegate.Combine(d, onDeserializingHandlers);
                }
                if (mi.IsDefined(typeofOnDeserializedAttribute, false)) {
                    var d = CreateHandler(bt, mi);
                    onDeserializedHandlers = onDeserializedHandlers == null ? d : Delegate.Combine(d, onDeserializedHandlers);
                }
            }
            bt = bt.BaseType;
            if (bt == null || bt == typeofObject) break;
            if (!bt.IsSerializable) throw new SerializationException(Cloner.BaseTypeNotSerializableMessage(bt, type));
            methods = bt.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly);
            i = 0;
        }
        Handler onSerializing = null, onSerialized = null, onDeserializing = null, onDeserialized = null;
        if (onSerializingHandler != null) {
            events |= CloneEvents.OnSerializing;
            onSerializing = (Handler)onSerializingHandler;
        }
        if (onSerializedHandlers != null) {
            events |= CloneEvents.OnSerialized;
            onSerialized = (Handler)onSerializedHandlers;
        }
        if (onDeserializingHandlers != null) {
            events |= CloneEvents.OnDeserializing;
            onDeserializing = (Handler)onDeserializingHandlers;
        }
        if (onDeserializedHandlers != null) {
            events |= CloneEvents.OnDeserialized;
            onDeserialized = (Handler)onDeserializedHandlers;
        }
        return new CloneEventHandlers(events, onSerializing, onSerialized, onDeserializing, onDeserialized);
    }
}

}

#endif

<|start_filename|>ParcelTest/BasicTests.cs<|end_filename|>
﻿using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.Collections.Generic;
using System.Linq;
using ArgList = Microsoft.FSharp.Collections.FSharpList<AST.Expression>;
using ExprOpt = Microsoft.FSharp.Core.FSharpOption<AST.Expression>;
using Expr = AST.Expression;

namespace ParcelTest
{
    [TestClass]
    public class BasicTests
    {
        [TestMethod]
        public void standardAddress()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            String s = "A3";

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct = new AST.ReferenceAddress(e, AST.Address.fromA1withMode(3, "A", AST.AddressMode.Relative, AST.AddressMode.Relative, e.WorksheetName, e.WorkbookName, e.Path));
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void standardRange()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            String s = "A3:B22";

            AST.Reference r = Parcel.simpleReferenceParser(s, e);
            AST.Reference correct = new AST.ReferenceRange(e,
                                                           new AST.Range(Utility.makeAddressForA1("A", 3, e),
                                                                         Utility.makeAddressForA1("B", 22, e))
                                                          );
            Assert.AreEqual(r, correct);
        }

        [TestMethod]
        public void relativeAddress()
        {
            var addrstr = "A1";
            var worksheet = "Worksheet";
            var workbook = "Workbook";
            var path = "";

            var addr = AST.Address.FromA1String(addrstr, worksheet, workbook, path);

            Assert.IsTrue(addr.RowMode == AST.AddressMode.Relative);
            Assert.IsTrue(addr.ColMode == AST.AddressMode.Relative);
        }

        [TestMethod]
        public void absoluteAddress()
        {
            var addrstr = "$A$1";
            var worksheet = "Worksheet";
            var workbook = "Workbook";
            var path = "";

            var addr = AST.Address.FromA1String(addrstr, worksheet, workbook, path);

            Assert.IsTrue(addr.RowMode == AST.AddressMode.Absolute);
            Assert.IsTrue(addr.ColMode == AST.AddressMode.Absolute);
        }

        [TestMethod]
        public void halfAbsoluteAddress()
        {
            var addrstr = "A$1";
            var worksheet = "Worksheet";
            var workbook = "Workbook";
            var path = "";

            var addr = AST.Address.FromA1String(addrstr, worksheet, workbook, path);

            Assert.IsTrue(addr.RowMode == AST.AddressMode.Absolute);
            Assert.IsTrue(addr.ColMode == AST.AddressMode.Relative);
        }

        [TestMethod]
        public void halfSharkAlligatorHalfMan()
        {
            var addrstr = "$A1";
            var worksheet = "Worksheet";
            var workbook = "Workbook";
            var path = "";

            var addr = AST.Address.FromA1String(addrstr, worksheet, workbook, path);

            Assert.IsTrue(addr.RowMode == AST.AddressMode.Relative);
            Assert.IsTrue(addr.ColMode == AST.AddressMode.Absolute);
        }

        [TestMethod]
        public void basicIfExpression()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var e = mwb.envForSheet(1);
            string f = "=IF(TRUE, 1, 2)";

            ExprOpt asto = Parcel.parseFormula(f, e.Path, e.WorkbookName, e.WorksheetName);

            Expr[] a = {
                Expr.NewReferenceExpr(new AST.ReferenceBoolean(e, true)),
                Expr.NewReferenceExpr(new AST.ReferenceConstant(e, 1.0)),
                Expr.NewReferenceExpr(new AST.ReferenceConstant(e, 2.0))
            };
            ArgList args = Utility.makeFSList<AST.Expression>(a);
            Expr correct = Expr.NewReferenceExpr(new AST.ReferenceFunction(e, "IF", args, AST.Arity.NewFixed(3)));

            try
            {
                Expr ast = asto.Value;
                Assert.AreEqual(ast, correct);
            } catch (NullReferenceException nre)
            {
                Assert.Fail("Parse error: " + nre.Message);
            }
        }

        [TestMethod]
        public void ifStatement()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            string s = "=IF(SUM(A1:A5) = 10, \"yes\", \"no\")";

            IEnumerable<AST.Range> rngs = new List<AST.Range>();
            try
            {
                rngs = Parcel.rangeReferencesFromFormula(s, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1), false);
            }
            catch (AST.ParseException)
            {
                Assert.Fail(String.Format("\"{0}\" should parse.", s));
            }

            Assert.AreEqual(1, rngs.Count());

            var addr = rngs.First().A1Local();
            Assert.AreEqual("A1:A5", addr);
        }

        [TestMethod]
        public void unOpAndBinOp()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            string formula = "=(+E6+E7)*0.28";

            IEnumerable<AST.Address> addrs = new List<AST.Address>();
            try
            {
                addrs = Parcel.addrReferencesFromFormula(formula, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1), false);
            }
            catch (AST.ParseException)
            {
                Assert.Fail(String.Format("\"{0}\" should parse.", formula));
            }

            Assert.AreEqual(2, addrs.Count());

            var a1 = Utility.makeAddressForA1("E", 6, mwb.envForSheet(1));
            var a2 = Utility.makeAddressForA1("E", 7, mwb.envForSheet(1));

            Assert.AreEqual(true, addrs.Contains(a1));
            Assert.AreEqual(true, addrs.Contains(a2));
        }

        [TestMethod]
        [Ignore]    // temporarily ignore test
        public void brutalEUSESTest()
        {
            var failures = new System.Collections.Concurrent.ConcurrentQueue<string>();

            var mwb = MockWorkbook.standardMockWorkbook();

            var formulas = System.IO.File.ReadAllLines(@"..\..\TestData\formulas_distinct.txt");

            System.Threading.Tasks.Parallel.ForEach(formulas, f =>
            {
                try
                {
                    Parcel.parseFormula(f, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
                }
                catch (Exception e)
                {
                    if (e is AST.IndirectAddressingNotSupportedException)
                    {
                        // OK   
                    }
                    else if (e is AST.ParseException)
                    {
                        System.Diagnostics.Debug.WriteLine("Fail: " + f);
                        failures.Enqueue(f);
                    }
                }
            });

            Assert.AreEqual(0, failures.Count());
            if (failures.Count > 0)
            {
                String.Join("\n", failures);
            }
        }

        [TestMethod]
        public void indirectReferences()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            String formula = "=TRANSPOSE(INDIRECT(ADDRESS(1,4,3,1,Menus!$K$10)):INDIRECT(ADDRESS(1,256,3,1,Menus!$K$10)))";
            try
            {
                Parcel.parseFormula(formula, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
            }
            catch (Exception e)
            {
                if (e is AST.IndirectAddressingNotSupportedException)
                {
                    // we pass
                }
                else
                {
                    Assert.Fail(e.Message);
                }
            }
        }

        [TestMethod]
        public void worksheetNameQuoteEscaping()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            String formula1 = "=Dan!H45";
            String formula2 = "='Dan Stuff'!H45";
            String formula3 = "='Dan''s Stuff'!H45";
            String formula4 = "='Dan's Stuff'!H45";

            // first, the good ones
            try
            {
                Parcel.parseFormula(formula1, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
                Parcel.parseFormula(formula2, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
                Parcel.parseFormula(formula3, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
            }
            catch (AST.ParseException e)
            {
                Assert.Fail(e.Message);
            }

            // a bad one
            try
            {
                Parcel.parseFormula(formula4, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
            }
            catch (AST.ParseException)
            {
                // OK
            }
        }

        [TestMethod]
        public void crossWorkbookAddrExtraction()
        {
            var mwb = MockWorkbook.standardMockWorkbook();
            var xmwb = new MockWorkbook("C:\\FINRES\\FIRMAS\\FORCASTS\\MODELS\\", "models.xls", new[] { "Forecast Assumptions" });

            var f1 = "=L66*('C:\\FINRES\\FIRMAS\\FORCASTS\\MODELS\\[models.xls]Forecast Assumptions'!J27)^0.25";
            var f1a1 = Utility.makeAddressForA1("L", 66, mwb.envForSheet(1));
            var f1a2 = Utility.makeAddressForA1("J", 27, xmwb.envForSheet(1));

            // extract
            try
            {
                var addrs = Parcel.addrReferencesFromFormula(f1, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1), false);
                Assert.IsTrue(addrs.Contains(f1a1));
                Assert.IsTrue(addrs.Contains(f1a2));
                Assert.IsTrue(addrs.Length == 2);
            }
            catch (AST.ParseException e)
            {
                Assert.Fail(e.Message);
            }
        }

        [TestMethod]
        public void missingWorkbookAddrExtraction()
        {
            var mwb = new MockWorkbook("C:\\FOOBAR", "workbook.xls", new[] { "budget" });
            var f = "=budget!A43";
            var addr = Utility.makeAddressForA1("A", 43, mwb.envForSheet(1));

            // extract
            try
            {
                var addrs = Parcel.addrReferencesFromFormula(f, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1), false);
                Assert.IsTrue(addrs.Contains(addr));
                Assert.IsTrue(addrs.Length == 1);
            }
            catch (AST.ParseException e)
            {
                Assert.Fail(e.Message);
            }
        }

        [TestMethod]
        public void crossWorksheetRangeExtraction()
        {
            var mwb = new MockWorkbook("C:\\FOOBAR", "workbook.xls", new[] { "sheet1", "One Country Charts", "One Country Data" });
            var f = "=IF('One Country Charts'!F17=\"\",VLOOKUP('One Country Charts'!F11,'One Country Data'!M5:O187,3,FALSE),VLOOKUP('One Country Charts'!F17,'One Country Data'!M5:O187,3,FALSE))";

            var data_env = mwb.envForSheet(3);

            var rng = new AST.Range(Utility.makeAddressForA1("M", 5, data_env), Utility.makeAddressForA1("O", 187, data_env));

            // extract
            try
            {
                var rngs = Parcel.rangeReferencesFromFormula(f, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1), false);
                Assert.IsTrue(rngs.Contains(rng));
                Assert.IsTrue(rngs.Length == 2);
            }
            catch (AST.ParseException e)
            {
                Assert.Fail(e.Message);
            }
        }

        [TestMethod]
        public void multipleRangeExtraction()
        {
            var mwb = new MockWorkbook("C:\\FOOBAR", "workbook.xls", new[] { "sheet1", "Calculations", "Status" });
            var f = "=IF(Status!G11=\"stand-alone hub\",SUMIF(Calculations!B7:P7,\"include\",Calculations!B163:P163),IF(Status!G11=\"remote\",SUMIF(Calculations!B7:P7,\"include\",Calculations!B184:P184),SUMIF(Calculations!B7:P7,\"include\",Calculations!B205:P205)))";

            var calc_env = mwb.envForSheet(2);

            var rng1 = new AST.Range(Utility.makeAddressForA1("B", 7, calc_env), Utility.makeAddressForA1("P", 7, calc_env));
            var rng2 = new AST.Range(Utility.makeAddressForA1("B", 163, calc_env), Utility.makeAddressForA1("P", 163, calc_env));
            var rng3 = new AST.Range(Utility.makeAddressForA1("B", 184, calc_env), Utility.makeAddressForA1("P", 184, calc_env));
            var rng4 = new AST.Range(Utility.makeAddressForA1("B", 205, calc_env), Utility.makeAddressForA1("P", 205, calc_env));

            // extract
            try
            {
                var rngs = Parcel.rangeReferencesFromFormula(f, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1), false);
                Assert.IsTrue(rngs.Contains(rng1));
                Assert.IsTrue(rngs.Contains(rng2));
                Assert.IsTrue(rngs.Contains(rng3));
                Assert.IsTrue(rngs.Contains(rng4));
                Assert.IsTrue(rngs.Length == 6);
            }
            catch (AST.ParseException e)
            {
                Assert.Fail(e.Message);
            }
        }

        [TestMethod]
        public void testHLOOKUP()
        {
            var mwb = new MockWorkbook("C:\\FOOBAR", "workbook.xls", new[] { "sheet1", "Calculations", "Status" });

            var f = "=HLOOKUP(J9,$C$45:$AH$46,2,TRUE)";

            // parse
            try
            {
                var ast = Parcel.parseFormula(f, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
                Assert.IsTrue(ExprOpt.get_IsSome(ast));
                var expr = (Expr.ReferenceExpr)ast.Value;
                var formula = (AST.ReferenceFunction)expr.Item;

                Assert.IsTrue(formula.FunctionName == "HLOOKUP");
            }
            catch (AST.ParseException e)
            {
                Assert.Fail(e.Message);
            }

        }

        [TestMethod]
        public void testCOUNTA()
        {
            // a parse that failed in the wild:
            var mwb = new MockWorkbook("C:\\FOOBAR", "workbook.xls", new[] { "sheet1", "Calculations", "Status" });
            var f = "=COUNTA(A6:A33)";

            // parse
            try
            {
                var ast = Parcel.parseFormula(f, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
                Assert.IsTrue(ExprOpt.get_IsSome(ast));
                var expr = (Expr.ReferenceExpr)ast.Value;
                var formula = (AST.ReferenceFunction)expr.Item;

                Assert.IsTrue(formula.FunctionName == "COUNTA");
            }
            catch (AST.ParseException e)
            {
                Assert.Fail(e.Message);
            }
        }

        [TestMethod]
        public void longestMatchFirstTest()
        {
            string[] isa = { "COLUMN", "FALSE", "NA", "NOW", "PI", "RAND", "ROW", "SHEET", "SHEETS", "TODAY", "TRUE" };
            string[] esa = { "COLUMN", "SHEETS", "FALSE", "SHEET", "TODAY", "RAND", "TRUE", "NOW", "ROW", "NA", "PI" };
            string[] asa = Grammar.lmf(isa);
            Assert.IsTrue(esa.SequenceEqual(asa));
        }

        [TestMethod]
        public void testMAXVarArgs()
        {
            // a parse that failed in the wild:
            var mwb = new MockWorkbook("C:\\FOOBAR", "workbook.xls", new[] { "sheet1", "Calculations", "Status" });
            var f = "=MAX(B6-10,0)";

            // parse
            try
            {
                var ast = Parcel.parseFormula(f, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
                Assert.IsTrue(ExprOpt.get_IsSome(ast));
                var expr = (Expr.ReferenceExpr)ast.Value;
                var formula = (AST.ReferenceFunction)expr.Item;

                Assert.IsTrue(formula.FunctionName == "MAX");
                Assert.AreEqual(formula.ArgumentList.Length, 2);
                Assert.IsTrue(formula.ArgumentList.First().IsBinOpExpr);
                Assert.IsTrue(formula.ArgumentList.Last().IsReferenceExpr);
            }
            catch (AST.ParseException e)
            {
                Assert.Fail(e.Message);
            }
        }

        [TestMethod]
        public void testTODAY()
        {
            // a parse that failed in the wild:
            var mwb = new MockWorkbook("C:\\FOOBAR", "workbook.xls", new[] { "sheet1", "Calculations", "Status" });
            var f = "=TODAY()";

            // parse
            try
            {
                var ast = Parcel.parseFormula(f, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
                Assert.IsTrue(ExprOpt.get_IsSome(ast));
                var expr = (Expr.ReferenceExpr)ast.Value;
                var formula = (AST.ReferenceFunction)expr.Item;

                Assert.IsTrue(formula.FunctionName == "TODAY");
                Assert.AreEqual(formula.ArgumentList.Length, 0);
            }
            catch (AST.ParseException e)
            {
                Assert.Fail(e.Message);
            }
        }

        [TestMethod]
        public void testTODAY2()
        {
            // a parse that should fail:
            var mwb = new MockWorkbook("C:\\FOOBAR", "workbook.xls", new[] { "sheet1", "Calculations", "Status" });
            var f = "=TODAY";

            // parse
            try
            {
                var ast = Parcel.parseFormula(f, mwb.Path, mwb.WorkbookName, mwb.worksheetName(1));
                Assert.Fail("Should not parse.");
            }
            catch (AST.ParseException)
            {
                // pass
                Assert.IsTrue(true);
            }
        }
    }
}


<|start_filename|>Parcel/CellVisitor.fs<|end_filename|>
﻿module internal CellVisitor
    let rec addrsFromExpr(expr: AST.Expression) : AST.Address list =
        match expr with
        | AST.ReferenceExpr(r) -> addrsFromRef r
        | AST.BinOpExpr(op, e1, e2) -> addrsFromExpr e1 @ addrsFromExpr e2
        | AST.UnaryOpExpr(op, e) -> addrsFromExpr e
        | AST.ParensExpr(e) -> addrsFromExpr e

    and addrsFromRef(ref: AST.Reference) : AST.Address list =
        match ref with
        | :? AST.ReferenceRange -> []
        | :? AST.ReferenceAddress as r -> addrsFromAddrRef r
        | :? AST.ReferenceNamed -> []   // TODO: symbol table lookup
        | :? AST.ReferenceFunction as r -> addrsFromFunctionRef r
        | :? AST.ReferenceConstant -> []
        | :? AST.ReferenceString -> []
        | :? AST.ReferenceBoolean -> []
        | _ -> raise (AST.ParseException "Unknown reference type.")

    and addrsFromAddrRef(ref: AST.ReferenceAddress) : AST.Address list = [ref.Address]

    and addrsFromFunctionRef(ref: AST.ReferenceFunction) : AST.Address list =
        List.map (fun arg -> addrsFromExpr(arg)) ref.ArgumentList |> List.concat

    let rec constantsFromExpr(expr: AST.Expression) : AST.ReferenceConstant list =
        match expr with
        | AST.ReferenceExpr(r) -> constantsFromRef r
        | AST.BinOpExpr(op, e1, e2) -> constantsFromExpr e1 @ constantsFromExpr e2
        | AST.UnaryOpExpr(op, e) -> constantsFromExpr e
        | AST.ParensExpr(e) -> constantsFromExpr e

    and constantsFromRef(ref: AST.Reference) : AST.ReferenceConstant list =
        match ref with
        | :? AST.ReferenceRange -> []
        | :? AST.ReferenceAddress as r -> []
        | :? AST.ReferenceNamed -> []
        | :? AST.ReferenceFunction as r -> r.ArgumentList |> List.map constantsFromExpr |> List.concat
        | :? AST.ReferenceConstant as r -> [r]
        | :? AST.ReferenceString -> []
        | :? AST.ReferenceBoolean -> []
        | _ -> raise (AST.ParseException "Unknown reference type.")