text:
try:
    frozenset
except NameError:
    # Import from the sets module for python 2.3
    from sets import Set as set
    from sets import ImmutableSet as frozenset
try:
    from collections import deque
except ImportError:
    from utils import deque

from constants import contentModelFlags, spaceCharacters
from constants import entitiesWindows1252, entities
from constants import asciiLowercase, asciiLetters, asciiUpper2Lower
from constants import digits, hexDigits, EOF
from constants import tokenTypes, tagTokenTypes

from inputstream import HTMLInputStream

# Group entities by their first character, for faster lookups
entitiesByFirstChar = {}
for e in entities:
    entitiesByFirstChar.setdefault(e[0], []).append(e)

class HTMLTokenizer:
    """ This class takes care of tokenizing HTML.

    * self.currentToken
      Holds the token that is currently being processed.

    * self.state
      Holds a reference to the method to be invoked... XXX

    * self.states
      Holds a mapping between states and methods that implement the state.

    * self.stream
      Points to HTMLInputStream object.
    """

    # XXX need to fix documentation

    def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True,
                 lowercaseElementName=True, lowercaseAttrName=True):
        self.stream = HTMLInputStream(stream, encoding, parseMeta, useChardet)

        #Perform case conversions?
        self.lowercaseElementName = lowercaseElementName
        self.lowercaseAttrName = lowercaseAttrName

        self.states = {
            "data":self.dataState,
            "entityData":self.entityDataState,
            "tagOpen":self.tagOpenState,
            "closeTagOpen":self.closeTagOpenState,
            "tagName":self.tagNameState,
            "beforeAttributeName":self.beforeAttributeNameState,
            "attributeName":self.attributeNameState,
            "afterAttributeName":self.afterAttributeNameState,
            "beforeAttributeValue":self.beforeAttributeValueState,
            "attributeValueDoubleQuoted":self.attributeValueDoubleQuotedState,
            "attributeValueSingleQuoted":self.attributeValueSingleQuotedState,
            "attributeValueUnQuoted":self.attributeValueUnQuotedState,
            "afterAttributeValue":self.afterAttributeValueState,
            "selfClosingStartTag":self.selfClosingStartTagState,
            "bogusComment":self.bogusCommentState,
            "bogusCommentContinuation":self.bogusCommentContinuationState,
            "markupDeclarationOpen":self.markupDeclarationOpenState,
            "commentStart":self.commentStartState,
            "commentStartDash":self.commentStartDashState,
            "comment":self.commentState,
            "commentEndDash":self.commentEndDashState,
            "commentEnd":self.commentEndState,
            "commentEndBang":self.commentEndBangState,
            "commentEndSpace":self.commentEndSpaceState,
            "doctype":self.doctypeState,
            "beforeDoctypeName":self.beforeDoctypeNameState,
            "doctypeName":self.doctypeNameState,
            "afterDoctypeName":self.afterDoctypeNameState,
            "beforeDoctypePublicIdentifier":self.beforeDoctypePublicIdentifierState,
            "doctypePublicIdentifierDoubleQuoted":self.doctypePublicIdentifierDoubleQuotedState,
            "doctypePublicIdentifierSingleQuoted":self.doctypePublicIdentifierSingleQuotedState,
            "afterDoctypePublicIdentifier":self.afterDoctypePublicIdentifierState,
            "beforeDoctypeSystemIdentifier":self.beforeDoctypeSystemIdentifierState,
            "doctypeSystemIdentifierDoubleQuoted":self.doctypeSystemIdentifierDoubleQuotedState,
            "doctypeSystemIdentifierSingleQuoted":self.doctypeSystemIdentifierSingleQuotedState,
            "afterDoctypeSystemIdentifier":self.afterDoctypeSystemIdentifierState,
            "bogusDoctype":self.bogusDoctypeState
        }

        # Setup the initial tokenizer state
        self.contentModelFlag = contentModelFlags["PCDATA"]
        self.escapeFlag = False
        self.lastFourChars = []
        self.state = self.dataState
        self.escape = False

        # The current token being created
        self.currentToken = None

    def __iter__(self):
        """ This is where the magic happens.

        We do our usually processing through the states and when we have a token
        to return we yield the token which pauses processing until the next token
        is requested.
        """
        self.tokenQueue = deque([])
        # Start processing. When EOF is reached self.state will return False
        # instead of True and the loop will terminate.
        while self.state():
            while self.stream.errors:
                yield {"type": tokenTypes["ParseError"], "data": self.stream.errors.pop(0)}
            while self.tokenQueue:
                yield self.tokenQueue.popleft()

    def consumeNumberEntity(self, isHex):
        """This function returns either U+FFFD or the character based on the
        decimal or hexadecimal representation. It also discards ";" if present.
        If not present self.tokenQueue.append({"type": tokenTypes["ParseError"]}) is invoked.
        """

        allowed = digits
        radix = 10
        if isHex:
            allowed = hexDigits
            radix = 16

        charStack = []

        # Consume all the characters that are in range while making sure we
        # don't hit an EOF.
        c = self.stream.char()
        while c in allowed and c is not EOF:
            charStack.append(c)
            c = self.stream.char()

        # Convert the set of characters consumed to an int.
        charAsInt = int("".join(charStack), radix)

        if charAsInt == 13:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "incorrect-cr-newline-entity"})
            charAsInt = 10
        elif 127 < charAsInt < 160:
            # If the integer is between 127 and 160 (so 128 and bigger and 159
            # and smaller) we need to do the "windows trick".
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "illegal-windows-1252-entity"})

            charAsInt = entitiesWindows1252[charAsInt - 128]

        # Certain characters get replaced with U+FFFD
        if ((charAsInt <= 0x0008) or (charAsInt == 0x000B) or (0x000E <= charAsInt <= 0x001F)
         or (0x007F <= charAsInt <= 0x009F)
         or (0xD800 <= charAsInt <= 0xDFFF) or (0xFDD0 <= charAsInt <= 0xFDEF)
         or (charAsInt & 0xFFFE == 0xFFFE) # catch all U+?FFFE and U+?FFFF, where ? is 0..10
         or (0x10FFFF < charAsInt)):
            char = u"\uFFFD"
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "illegal-codepoint-for-numeric-entity",
              "datavars": {"charAsInt": charAsInt}})
        else:
            try:
                # XXX We should have a separate function that does "int" to
                # "unicodestring" conversion since this doesn't always work
                # according to hsivonen. Also, unichr has a limitation of 65535
                char = unichr(charAsInt)
            except:
                try:
                    char = eval("u'\\U%08x'" % charAsInt)
                except:
                    self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                      "cant-convert-numeric-entity",
                      "datavars": {"charAsInt": charAsInt}})

        # Discard the ; if present. Otherwise, put it back on the queue and
        # invoke parseError on parser.
        if c != u";":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "numeric-entity-without-semicolon"})
            self.stream.unget(c)

        return char

    def consumeEntity(self, allowedChar=None, fromAttribute=False):
        # Initialise to the default output for when no entity is matched
        output = u"&"

        charStack = [self.stream.char()]
        if charStack[0] in spaceCharacters or charStack[0] in (EOF, u"<", u"&") \
         or (allowedChar is not None and allowedChar == charStack[0]):
            self.stream.unget(charStack[0])

        elif charStack[0] == u"#":
            # Read the next character to see if it's hex or decimal
            hex = False
            charStack.append(self.stream.char())
            if charStack[-1] in (u"x", u"X"):
                hex = True
                charStack.append(self.stream.char())

            # charStack[-1] should be the first digit
            if (hex and charStack[-1] in hexDigits) \
             or (not hex and charStack[-1] in digits):
                # At least one digit found, so consume the whole number
                self.stream.unget(charStack[-1])
                output = self.consumeNumberEntity(hex)
            else:
                # No digits found
                self.tokenQueue.append({"type": tokenTypes["ParseError"],
                    "data": "expected-numeric-entity"})
                self.stream.unget(charStack.pop())
                output = u"&" + u"".join(charStack)

        else:
            # At this point in the process might have named entity. Entities
            # are stored in the global variable "entities".
            #
            # Consume characters and compare to these to a substring of the
            # entity names in the list until the substring no longer matches.
            filteredEntityList = entitiesByFirstChar.get(charStack[0], [])

            def entitiesStartingWith(name):
                return [e for e in filteredEntityList if e.startswith(name)]

            while charStack[-1] is not EOF and\
              entitiesStartingWith("".join(charStack)):
                charStack.append(self.stream.char())

            # At this point we have a string that starts with some characters
            # that may match an entity
            entityName = None

            # Try to find the longest entity the string will match to take care
            # of &noti for instance.
            for entityLength in xrange(len(charStack)-1, 1, -1):
                possibleEntityName = "".join(charStack[:entityLength])
                if possibleEntityName in entities:
                    entityName = possibleEntityName
                    break

            if entityName is not None:
                if entityName[-1] != ";":
                    self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                      "named-entity-without-semicolon"})
                if entityName[-1] != ";" and fromAttribute and \
                  (charStack[entityLength] in asciiLetters
                  or charStack[entityLength] in digits):
                    self.stream.unget(charStack.pop())
                    output = u"&" + u"".join(charStack)
                else:
                    output = entities[entityName]
                    self.stream.unget(charStack.pop())
                    output += u"".join(charStack[entityLength:])
            else:
                self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                  "expected-named-entity"})
                self.stream.unget(charStack.pop())
                output = u"&" + u"".join(charStack)

        if fromAttribute:
            self.currentToken["data"][-1][1] += output
        else:
            self.tokenQueue.append({"type": tokenTypes["Characters"], "data": output})

    def processEntityInAttribute(self, allowedChar):
        """This method replaces the need for "entityInAttributeValueState".
        """
        self.consumeEntity(allowedChar=allowedChar, fromAttribute=True)

    def emitCurrentToken(self):
        """This method is a generic handler for emitting the tags. It also sets
        the state to "data" because that's what's needed after a token has been
        emitted.
        """
        token = self.currentToken
        # Add token to the queue to be yielded
        if (token["type"] in tagTokenTypes):
            if self.lowercaseElementName:
                token["name"] = token["name"].translate(asciiUpper2Lower)
            if token["type"] == tokenTypes["EndTag"]:
                if token["data"]:
                    self.tokenQueue.append({"type":tokenTypes["ParseError"],
                                            "data":"attributes-in-end-tag"})
                if token["selfClosing"]:
                    self.tokenQueue.append({"type":tokenTypes["ParseError"],
                                            "data":"self-closing-flag-on-end-tag"})
        self.tokenQueue.append(token)
        self.state = self.dataState

    # Below are the various tokenizer states worked out.

    def dataState(self):
        #XXX - consider splitting this state based on the content model flag
        data = self.stream.char()

        # Keep a charbuffer to handle the escapeFlag
        if (self.contentModelFlag in
            (contentModelFlags["CDATA"], contentModelFlags["RCDATA"])):
            if len(self.lastFourChars) == 4:
                self.lastFourChars.pop(0)
            self.lastFourChars.append(data)

        # The rest of the logic
        if (data == "&" and self.contentModelFlag in
            (contentModelFlags["PCDATA"], contentModelFlags["RCDATA"]) and
            not self.escapeFlag):
            self.state = self.states["entityData"]
        elif (data == "-" and self.contentModelFlag in
              (contentModelFlags["CDATA"], contentModelFlags["RCDATA"]) and
              not self.escapeFlag and "".join(self.lastFourChars) == "<!--"):
            self.escapeFlag = True
            self.tokenQueue.append({"type": tokenTypes["Characters"],
                                    "data":data})
        elif (data == "<" and (self.contentModelFlag ==
                               contentModelFlags["PCDATA"]
                               or (self.contentModelFlag in
                                   (contentModelFlags["CDATA"],
                                    contentModelFlags["RCDATA"]) and
                                   self.escapeFlag == False))):
            self.state = self.states["tagOpen"]
        elif (data == ">" and self.contentModelFlag in
              (contentModelFlags["CDATA"], contentModelFlags["RCDATA"]) and
              self.escapeFlag and "".join(self.lastFourChars)[1:] == "-->"):
            self.escapeFlag = False
            self.tokenQueue.append({"type": tokenTypes["Characters"], "data":data})
        elif data is EOF:
            # Tokenization ends.
            return False
        elif data in spaceCharacters:
            # Directly after emitting a token you switch back to the "data
            # state". At that point spaceCharacters are important so they are
            # emitted separately.
            self.tokenQueue.append({"type": tokenTypes["SpaceCharacters"], "data":
              data + self.stream.charsUntil(spaceCharacters, True)})
            # No need to update lastFourChars here, since the first space will
            # have already been appended to lastFourChars and will have broken
            # any <!-- or --> sequences
        else:
            if (self.contentModelFlag in
                (contentModelFlags["CDATA"], contentModelFlags["RCDATA"])):
                chars = self.stream.charsUntil((u"&", u"<", u">", u"-"))
                self.lastFourChars += chars[-4:]
                self.lastFourChars = self.lastFourChars[-4:]
            else:
                chars = self.stream.charsUntil((u"&", u"<"))
            self.tokenQueue.append({"type": tokenTypes["Characters"], "data":
              data + chars})
        return True

    def entityDataState(self):
        self.consumeEntity()
        self.state = self.states["data"]
        return True

    def tagOpenState(self):
        data = self.stream.char()
        if self.contentModelFlag == contentModelFlags["PCDATA"]:
            if data == u"!":
                self.state = self.states["markupDeclarationOpen"]
            elif data == u"/":
                self.state = self.states["closeTagOpen"]
            elif data in asciiLetters:
                self.currentToken = {"type": tokenTypes["StartTag"],
                                     "name": data, "data": [],
                                     "selfClosing": False,
                                     "selfClosingAcknowledged": False}
                self.state = self.states["tagName"]
            elif data == u">":
                # XXX In theory it could be something besides a tag name. But
                # do we really care?
                self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                  "expected-tag-name-but-got-right-bracket"})
                self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<>"})
                self.state = self.states["data"]
            elif data == u"?":
                # XXX In theory it could be something besides a tag name. But
                # do we really care?
                self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                  "expected-tag-name-but-got-question-mark"})
                self.stream.unget(data)
                self.state = self.states["bogusComment"]
            else:
                # XXX
                self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                  "expected-tag-name"})
                self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"})
                self.stream.unget(data)
                self.state = self.states["data"]
        else:
            # We know the content model flag is set to either RCDATA or CDATA
            # now because this state can never be entered with the PLAINTEXT
            # flag.
            if data == u"/":
                self.state = self.states["closeTagOpen"]
            else:
                self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"})
                self.stream.unget(data)
                self.state = self.states["data"]
        return True

    def closeTagOpenState(self):
        if (self.contentModelFlag in (contentModelFlags["RCDATA"],
            contentModelFlags["CDATA"])):

            charStack = []
            if self.currentToken:
                # So far we know that "</" has been consumed. We now need to know
                # whether the next few characters match the name of last emitted
                # start tag which also happens to be the currentToken.
                matched = True
                for expected in self.currentToken["name"].lower():
                    charStack.append(self.stream.char())
                    if charStack[-1] not in (expected, expected.upper()):
                        matched = False
                        break

                # If the tag name prefix matched, we also need to check the
                # subsequent character
                if matched:
                    charStack.append(self.stream.char())
                    if charStack[-1] in (spaceCharacters | frozenset((u">", u"/", EOF))):
                        self.contentModelFlag = contentModelFlags["PCDATA"]
                        # Unget the last character, so it can be re-processed
                        # in the next state
                        self.stream.unget(charStack.pop())
                        # The remaining characters in charStack are the tag name
                        self.currentToken = {"type": tokenTypes["EndTag"],
                                             "name": u"".join(charStack),
                                             "data": [],
                                             "selfClosing":False}
                        self.state = self.states["tagName"]
                        return True

                # Didn't find the end tag. The last character in charStack could be
                # anything, so it has to be re-processed in the data state
                self.stream.unget(charStack.pop())

            # The remaining characters are a prefix of the tag name, so they're
            # just letters and digits, so they can be output as character
            # tokens immediately
            self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</" + u"".join(charStack)})
            self.state = self.states["data"]
            return True

        data = self.stream.char()
        if data in asciiLetters:
            self.currentToken = {"type": tokenTypes["EndTag"], "name": data,
                                 "data": [], "selfClosing":False}
            self.state = self.states["tagName"]
        elif data == u">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-closing-tag-but-got-right-bracket"})
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-closing-tag-but-got-eof"})
            self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"})
            self.state = self.states["data"]
        else:
            # XXX data can be _'_...
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-closing-tag-but-got-char",
              "datavars": {"data": data}})
            self.stream.unget(data)
            self.state = self.states["bogusComment"]
        return True

    def tagNameState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            self.state = self.states["beforeAttributeName"]
        elif data == u">":
            self.emitCurrentToken()
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-tag-name"})
            self.state = self.states["data"]
        elif data == u"/":
            self.state = self.states["selfClosingStartTag"]
        else:
            self.currentToken["name"] += data
            # (Don't use charsUntil here, because tag names are
            # very short and it's faster to not do anything fancy)
        return True

    def beforeAttributeNameState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            self.stream.charsUntil(spaceCharacters, True)
        elif data in asciiLetters:
            self.currentToken["data"].append([data, ""])
            self.state = self.states["attributeName"]
        elif data == u">":
            self.emitCurrentToken()
        elif data == u"/":
            self.state = self.states["selfClosingStartTag"]
        elif data == u"'" or data == u'"' or data == u"=":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "invalid-character-in-attribute-name"})
            self.currentToken["data"].append([data, ""])
            self.state = self.states["attributeName"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-attribute-name-but-got-eof"})
            self.state = self.states["data"]
        else:
            self.currentToken["data"].append([data, ""])
            self.state = self.states["attributeName"]
        return True

    def attributeNameState(self):
        data = self.stream.char()
        leavingThisState = True
        emitToken = False
        if data == u"=":
            self.state = self.states["beforeAttributeValue"]
        elif data in asciiLetters:
            self.currentToken["data"][-1][0] += data +\
              self.stream.charsUntil(asciiLetters, True)
            leavingThisState = False
        elif data == u">":
            # XXX If we emit here the attributes are converted to a dict
            # without being checked and when the code below runs we error
            # because data is a dict not a list
            emitToken = True
        elif data in spaceCharacters:
            self.state = self.states["afterAttributeName"]
        elif data == u"/":
            self.state = self.states["selfClosingStartTag"]
        elif data == u"'" or data == u'"':
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "invalid-character-in-attribute-name"})
            self.currentToken["data"][-1][0] += data
            leavingThisState = False
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-attribute-name"})
            self.state = self.states["data"]
            emitToken = True
        else:
            self.currentToken["data"][-1][0] += data
            leavingThisState = False

        if leavingThisState:
            # Attributes are not dropped at this stage. That happens when the
            # start tag token is emitted so values can still be safely appended
            # to attributes, but we do want to report the parse error in time.
            if self.lowercaseAttrName:
                self.currentToken["data"][-1][0] = (
                    self.currentToken["data"][-1][0].translate(asciiUpper2Lower))
            for name, value in self.currentToken["data"][:-1]:
                if self.currentToken["data"][-1][0] == name:
                    self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                      "duplicate-attribute"})
                    break
            # XXX Fix for above XXX
            if emitToken:
                self.emitCurrentToken()
        return True

    def afterAttributeNameState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            self.stream.charsUntil(spaceCharacters, True)
        elif data == u"=":
            self.state = self.states["beforeAttributeValue"]
        elif data == u">":
            self.emitCurrentToken()
        elif data in asciiLetters:
            self.currentToken["data"].append([data, ""])
            self.state = self.states["attributeName"]
        elif data == u"/":
            self.state = self.states["selfClosingStartTag"]
        elif data == u"'" or data == u'"':
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "invalid-character-after-attribute-name"})
            self.currentToken["data"].append([data, ""])
            self.state = self.states["attributeName"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-end-of-tag-but-got-eof"})
            self.emitCurrentToken()
        else:
            self.currentToken["data"].append([data, ""])
            self.state = self.states["attributeName"]
        return True

    def beforeAttributeValueState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            self.stream.charsUntil(spaceCharacters, True)
        elif data == u"\"":
            self.state = self.states["attributeValueDoubleQuoted"]
        elif data == u"&":
            self.state = self.states["attributeValueUnQuoted"]
            self.stream.unget(data);
        elif data == u"'":
            self.state = self.states["attributeValueSingleQuoted"]
        elif data == u">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-attribute-value-but-got-right-bracket"})
            self.emitCurrentToken()
        elif data == u"=":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "equals-in-unquoted-attribute-value"})
            self.currentToken["data"][-1][1] += data
            self.state = self.states["attributeValueUnQuoted"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-attribute-value-but-got-eof"})
            self.emitCurrentToken()
        else:
            self.currentToken["data"][-1][1] += data
            self.state = self.states["attributeValueUnQuoted"]
        return True

    def attributeValueDoubleQuotedState(self):
        data = self.stream.char()
        if data == "\"":
            self.state = self.states["afterAttributeValue"]
        elif data == u"&":
            self.processEntityInAttribute(u'"')
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-attribute-value-double-quote"})
            self.emitCurrentToken()
        else:
            self.currentToken["data"][-1][1] += data +\
              self.stream.charsUntil(("\"", u"&"))
        return True

    def attributeValueSingleQuotedState(self):
        data = self.stream.char()
        if data == "'":
            self.state = self.states["afterAttributeValue"]
        elif data == u"&":
            self.processEntityInAttribute(u"'")
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-attribute-value-single-quote"})
            self.emitCurrentToken()
        else:
            self.currentToken["data"][-1][1] += data +\
              self.stream.charsUntil(("'", u"&"))
        return True

    def attributeValueUnQuotedState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            self.state = self.states["beforeAttributeName"]
        elif data == u"&":
            self.processEntityInAttribute(None)
        elif data == u">":
            self.emitCurrentToken()
        elif data in (u'"', u"'", u"=", u"<"):
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-character-in-unquoted-attribute-value"})
            self.currentToken["data"][-1][1] += data
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-attribute-value-no-quotes"})
            self.emitCurrentToken()
        else:
            self.currentToken["data"][-1][1] += data + self.stream.charsUntil( \
              frozenset(("&", ">", "<", "=", "'", '"')) | spaceCharacters)
        return True

    def afterAttributeValueState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            self.state = self.states["beforeAttributeName"]
        elif data == u">":
            self.emitCurrentToken()
        elif data == u"/":
            self.state = self.states["selfClosingStartTag"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-EOF-after-attribute-value"})
            self.emitCurrentToken()
            self.stream.unget(data)
            self.state = self.states["data"]
        else:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-character-after-attribute-value"})
            self.stream.unget(data)
            self.state = self.states["beforeAttributeName"]
        return True

    def selfClosingStartTagState(self):
        data = self.stream.char()
        if data == ">":
            self.currentToken["selfClosing"] = True
            self.emitCurrentToken()
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"],
                                    "data":
                                        "unexpected-EOF-after-solidus-in-tag"})
            self.stream.unget(data)
            self.state = self.states["data"]
        else:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-character-after-soldius-in-tag"})
            self.stream.unget(data)
            self.state = self.states["beforeAttributeName"]
        return True

    def bogusCommentState(self):
        # Make a new comment token and give it as value all the characters
        # until the first > or EOF (charsUntil checks for EOF automatically)
        # and emit it.
        self.tokenQueue.append(
          {"type": tokenTypes["Comment"], "data": self.stream.charsUntil(u">")})

        # Eat the character directly after the bogus comment which is either a
        # ">" or an EOF.
        self.stream.char()
        self.state = self.states["data"]
        return True

    def bogusCommentContinuationState(self):
        # Like bogusCommentState, but the caller must create the comment token
        # and this state just adds more characters to it
        self.currentToken["data"] += self.stream.charsUntil(u">")
        self.tokenQueue.append(self.currentToken)

        # Eat the character directly after the bogus comment which is either a
        # ">" or an EOF.
        self.stream.char()
        self.state = self.states["data"]
        return True

    def markupDeclarationOpenState(self):
        charStack = [self.stream.char()]
        if charStack[-1] == u"-":
            charStack.append(self.stream.char())
            if charStack[-1] == u"-":
                self.currentToken = {"type": tokenTypes["Comment"], "data": u""}
                self.state = self.states["commentStart"]
                return True
        elif charStack[-1] in (u'd', u'D'):
            matched = True
            for expected in ((u'o', u'O'), (u'c', u'C'), (u't', u'T'),
                             (u'y', u'Y'), (u'p', u'P'), (u'e', u'E')):
                charStack.append(self.stream.char())
                if charStack[-1] not in expected:
                    matched = False
                    break
            if matched:
                self.currentToken = {"type": tokenTypes["Doctype"],
                                     "name": u"",
                                     "publicId": None, "systemId": None,
                                     "correct": True}
                self.state = self.states["doctype"]
                return True

        self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
          "expected-dashes-or-doctype"})
        # charStack[:-2] consists of 'safe' characters ('-', 'd', 'o', etc)
        # so they can be copied directly into the bogus comment data, and only
        # the last character might be '>' or EOF and needs to be ungetted
        self.stream.unget(charStack.pop())
        self.currentToken = {"type": tokenTypes["Comment"],
                             "data": u"".join(charStack)}
        self.state = self.states["bogusCommentContinuation"]
        return True

    def commentStartState(self):
        data = self.stream.char()
        if data == "-":
            self.state = self.states["commentStartDash"]
        elif data == ">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "incorrect-comment"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-comment"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["data"] += data + self.stream.charsUntil(u"-")
            self.state = self.states["comment"]
        return True

    def commentStartDashState(self):
        data = self.stream.char()
        if data == "-":
            self.state = self.states["commentEnd"]
        elif data == ">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "incorrect-comment"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-comment"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["data"] += "-" + data + self.stream.charsUntil(u"-")
            self.state = self.states["comment"]
        return True

    def commentState(self):
        data = self.stream.char()
        if data == u"-":
            self.state = self.states["commentEndDash"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-comment"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["data"] += data + self.stream.charsUntil(u"-")
        return True

    def commentEndDashState(self):
        data = self.stream.char()
        if data == u"-":
            self.state = self.states["commentEnd"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-comment-end-dash"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["data"] += u"-" + data +\
              self.stream.charsUntil(u"-")
            # Consume the next character which is either a "-" or an EOF as
            # well so if there's a "-" directly after the "-" we go nicely to
            # the "comment end state" without emitting a ParseError() there.
            self.stream.char()
        return True

    def commentEndState(self):
        data = self.stream.char()
        if data == u">":
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data == u"-":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
             "unexpected-dash-after-double-dash-in-comment"})
            self.currentToken["data"] += data
        elif data in spaceCharacters:
            self.currentToken["data"] += "--" + data
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-space-after-double-dash-in-comment"})
            self.state = self.states["commentEndSpace"]
        elif data == "!":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-bang-after-double-dash-in-comment"})
            self.state = self.states["commentEndBang"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-comment-double-dash"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            # XXX
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-char-in-comment"})
            self.currentToken["data"] += u"--" + data
            self.state = self.states["comment"]
        return True

    def commentEndBangState(self):
        data = self.stream.char()
        if data == u">":
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data == u"-":
            self.currentToken["data"] += "--!"
            self.state = self.states["commentEndDash"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-comment-end-bang-state"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["data"] += u"--!" + data
            self.state = self.states["comment"]
        return True

    def commentEndSpaceState(self):
        data = self.stream.char()
        if data == u">":
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data == u"-":
            self.state = self.states["commentEndDash"]
        elif data in spaceCharacters:
            self.currentToken["data"] += data
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-comment-end-space-state"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["data"] += data
            self.state = self.states["comment"]
        return True

    def doctypeState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            self.state = self.states["beforeDoctypeName"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-doctype-name-but-got-eof"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "need-space-after-doctype"})
            self.stream.unget(data)
            self.state = self.states["beforeDoctypeName"]
        return True

    def beforeDoctypeNameState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            pass
        elif data == u">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-doctype-name-but-got-right-bracket"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "expected-doctype-name-but-got-eof"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["name"] = data
            self.state = self.states["doctypeName"]
        return True

    def doctypeNameState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower)
            self.state = self.states["afterDoctypeName"]
        elif data == u">":
            self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower)
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype-name"})
            self.currentToken["correct"] = False
            self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower)
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["name"] += data
        return True

    def afterDoctypeNameState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            pass
        elif data == u">":
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.currentToken["correct"] = False
            self.stream.unget(data)
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype"})
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            if data in (u"p", u"P"):
                matched = True
                for expected in ((u"u", u"U"), (u"b", u"B"), (u"l", u"L"),
                                 (u"i", u"I"), (u"c", u"C")):
                    data = self.stream.char()
                    if data not in expected:
                        matched = False
                        break
                if matched:
                    self.state = self.states["beforeDoctypePublicIdentifier"]
                    return True
            elif data in (u"s", u"S"):
                matched = True
                for expected in ((u"y", u"Y"), (u"s", u"S"), (u"t", u"T"),
                                 (u"e", u"E"), (u"m", u"M")):
                    data = self.stream.char()
                    if data not in expected:
                        matched = False
                        break
                if matched:
                    self.state = self.states["beforeDoctypeSystemIdentifier"]
                    return True

            # All the characters read before the current 'data' will be
            # [a-zA-Z], so they're garbage in the bogus doctype and can be
            # discarded; only the latest character might be '>' or EOF
            # and needs to be ungetted
            self.stream.unget(data)
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                "expected-space-or-right-bracket-in-doctype", "datavars":
                {"data": data}})
            self.currentToken["correct"] = False
            self.state = self.states["bogusDoctype"]

        return True

    def beforeDoctypePublicIdentifierState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            pass
        elif data == "\"":
            self.currentToken["publicId"] = u""
            self.state = self.states["doctypePublicIdentifierDoubleQuoted"]
        elif data == "'":
            self.currentToken["publicId"] = u""
            self.state = self.states["doctypePublicIdentifierSingleQuoted"]
        elif data == ">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-end-of-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-char-in-doctype"})
            self.currentToken["correct"] = False
            self.state = self.states["bogusDoctype"]
        return True

    def doctypePublicIdentifierDoubleQuotedState(self):
        data = self.stream.char()
        if data == "\"":
            self.state = self.states["afterDoctypePublicIdentifier"]
        elif data == ">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-end-of-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["publicId"] += data
        return True

    def doctypePublicIdentifierSingleQuotedState(self):
        data = self.stream.char()
        if data == "'":
            self.state = self.states["afterDoctypePublicIdentifier"]
        elif data == ">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-end-of-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["publicId"] += data
        return True

    def afterDoctypePublicIdentifierState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            pass
        elif data == "\"":
            self.currentToken["systemId"] = u""
            self.state = self.states["doctypeSystemIdentifierDoubleQuoted"]
        elif data == "'":
            self.currentToken["systemId"] = u""
            self.state = self.states["doctypeSystemIdentifierSingleQuoted"]
        elif data == ">":
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-char-in-doctype"})
            self.currentToken["correct"] = False
            self.state = self.states["bogusDoctype"]
        return True

    def beforeDoctypeSystemIdentifierState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            pass
        elif data == "\"":
            self.currentToken["systemId"] = u""
            self.state = self.states["doctypeSystemIdentifierDoubleQuoted"]
        elif data == "'":
            self.currentToken["systemId"] = u""
            self.state = self.states["doctypeSystemIdentifierSingleQuoted"]
        elif data == ">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-char-in-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-char-in-doctype"})
            self.currentToken["correct"] = False
            self.state = self.states["bogusDoctype"]
        return True

    def doctypeSystemIdentifierDoubleQuotedState(self):
        data = self.stream.char()
        if data == "\"":
            self.state = self.states["afterDoctypeSystemIdentifier"]
        elif data == ">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-end-of-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["systemId"] += data
        return True

    def doctypeSystemIdentifierSingleQuotedState(self):
        data = self.stream.char()
        if data == "'":
            self.state = self.states["afterDoctypeSystemIdentifier"]
        elif data == ">":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-end-of-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.currentToken["systemId"] += data
        return True

    def afterDoctypeSystemIdentifierState(self):
        data = self.stream.char()
        if data in spaceCharacters:
            pass
        elif data == ">":
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "eof-in-doctype"})
            self.currentToken["correct"] = False
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
              "unexpected-char-in-doctype"})
            self.state = self.states["bogusDoctype"]
        return True

    def bogusDoctypeState(self):
        data = self.stream.char()
        if data == u">":
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        elif data is EOF:
            # XXX EMIT
            self.stream.unget(data)
            self.tokenQueue.append(self.currentToken)
            self.state = self.states["data"]
        else:
            pass
        return True


text:
package io.deepsense.deeplang.doperables

import org.apache.spark.sql.Row
import org.apache.spark.sql.types._

import io.deepsense.deeplang.DeeplangIntegTestSupport
import io.deepsense.deeplang.doperables.dataframe.DataFrame
import io.deepsense.deeplang.doperables.multicolumn.MultiColumnParams.SingleOrMultiColumnChoices.SingleColumnChoice
import io.deepsense.deeplang.doperables.multicolumn.SingleColumnParams.SingleTransformInPlaceChoices.NoInPlaceChoice
import io.deepsense.deeplang.doperables.spark.wrappers.transformers.TransformerSerialization
import io.deepsense.deeplang.doperations.exceptions._
import io.deepsense.deeplang.params.selections.NameSingleColumnSelection

class SqlColumnTransformationIntegSpec
    extends DeeplangIntegTestSupport
    with TransformerSerialization {

  import TransformerSerialization._

  val resultColumn = 3
  val delta = 0.01
  val column0 = "c0"
  val column1 = "c1"
  val column2 = "c2"
  val column3 = "c3"
  val column3needsEscaping = "c.strange name!"

  "SqlColumnTransformation" should {

    "create Transformation that counts ABS properly" in {
      runTest("ABS(x)", column1, column3, Seq(1.0, 1.1, 1.2, 1.3, null))
    }

    "create Transformation that counts POW properly" in {
      runTest("POW(x, 2.0)", column1, column3, Seq(1.0, 1.21, 1.44, 1.69, null))
    }

    "create Transformation that counts SQRT properly" in {
      runTest("SQRT(x)", column2, column3, Seq(0.447, 1.483, null, 2.04, null))
    }

    "create Transformation that counts SIN properly" in {
      runTest("SIN(x)", column1, column3, Seq(0.841, -0.891, 0.932, -0.96, null))
    }

    "create Transformation that counts COS properly" in {
      runTest("COS(x)", column1, column3, Seq(0.540, 0.453, 0.362, 0.267, null))
    }

    "create Transformation that counts TAN properly" in {
      runTest("TAN(x)", column1, column3, Seq(1.557, -1.964, 2.572, -3.602, null))
    }

    "create Transformation that counts LN properly" in {
      runTest("LN(x)", column2, column3, Seq(-1.609, 0.788, null, 1.435, null))
    }

    "create Transformation that counts FLOOR properly" in {
      runTest("FLOOR(x)", column1, column3, Seq(1.0, -2.0, 1.0, -2.0, null))
    }

    "create Transformation that counts CEIL properly" in {
      runTest("CEIL(x)", column1, column3, Seq(1.0, -1.0, 2.0, -1.0, null))
    }

    "create Transformation that counts SIGNUM properly" in {
      runTest("SIGNUM(x)", column1, column3, Seq(1.0, -1.0, 1.0, -1.0, null))
    }
  }

  it should {
    "create Transformation that counts MINIMUM properly" in {
      runTest(s"MINIMUM($column1, $column2)", column1, column3, Seq(0.2, -1.1, null, -1.3, null))
    }

    "create Transformation that counts MAXIMUM properly" in {
      runTest(s"MAXIMUM($column1, $column2)", column1, column3, Seq(1.0, 2.2, null, 4.2, null))
    }

    "create Transformation that counts complex formulas properly" in {
      runTest(s"MAXIMUM(SIN($column2) + 1.0, ABS($column1 - 2.0))", column1, column3,
        Seq(1.19, 3.1, null, 3.3, null))
    }
  }

  it should {
    "detect missing inputColumn during inference" in {
      a[ColumnDoesNotExistException] shouldBe thrownBy {
        val inPlace = NoInPlaceChoice()
          .setOutputColumn(column3)
        val single = SingleColumnChoice()
          .setInputColumn(NameSingleColumnSelection("nonExistingCol"))
          .setInPlace(inPlace)
        SqlColumnTransformer()
          .setFormula("x * 2")
          .setSingleOrMultiChoice(single)
          ._transformSchema(schema)
      }
    }
    val inPlace = NoInPlaceChoice()
      .setOutputColumn(column3)
    val single = SingleColumnChoice()
      .setInputColumn(NameSingleColumnSelection(column1))
      .setInPlace(inPlace)
    "detect SQL syntax error during inference" in {
      a[SqlColumnExpressionSyntaxException] shouldBe thrownBy(
        SqlColumnTransformer()
          .setFormula("+++---")
          .setSingleOrMultiChoice(single)
          ._transformSchema(schema))
    }
    "detect non-existent column during inference" in {
      a[ColumnsDoNotExistException] shouldBe thrownBy(
        SqlColumnTransformer()
          .setFormula("nonExistingCol")
          .setSingleOrMultiChoice(single)
          ._transformSchema(schema)
      )
    }
    "detect that alias conflicts with a column name form input DF" in {
      a[ColumnAliasNotUniqueException] shouldBe thrownBy(
        SqlColumnTransformer()
          .setFormula("c0")
          .setInputColumnAlias("c0")
          .setSingleOrMultiChoice(single)
          ._transformSchema(schema))
    }
  }

  it should {
    "work with user-defined column alias" in {
      runTest("ABS(y)", column1, column3, Seq(1.0, 1.1, 1.2, 1.3, null), "y")
    }

    "create Transformation that produces properly escaped column name" in {
      val dataFrame = applyFormulaToDataFrame(
        "COS(x)",
        column1,
        s"$column3needsEscaping",
        "x",
        prepareDataFrame())
      val rows = dataFrame.sparkDataFrame.collect()
      validateColumn(rows, Seq(0.540, 0.453, 0.362, 0.267, null))
      val schema = dataFrame.sparkDataFrame.schema
      schema.fieldNames shouldBe Array(column0, column1, column2, column3needsEscaping)
    }

    // TODO: This is Spark bug. Test will be pending till SPARK-13297 is fixed.
    "create Transformation that works on DataFrame with backtics in column names" is pending

    // Regression test
    "create Transformation that works on DataFrame with non-standard column names" in {
      val column0 = "c 0"
      val column1 = "c-2"
      val column2 = "c - 2"
      def prepareDataFrame(): DataFrame = {
        val schema: StructType = StructType(List(
          StructField(column0, StringType),
          StructField(column1, DoubleType),
          StructField(column2, DoubleType)))
        val manualRowsSeq: Seq[Row] = Seq(
          Row("aaa", 1.0, 0.2),
          Row("bbb", -1.1, 2.2),
          Row("ccc", 1.2, null),
          Row("ddd", -1.3, 4.2),
          Row("eee", null, null))
        createDataFrame(manualRowsSeq, schema)
      }
      val dataFrame = applyFormulaToDataFrame(
        "COS(x)",
        column1,
        s"$column3needsEscaping",
        "x",
        prepareDataFrame())
      val rows = dataFrame.sparkDataFrame.collect()
      validateColumn(rows, Seq(0.540, 0.453, 0.362, 0.267, null))
      val schema = dataFrame.sparkDataFrame.schema
      schema.fieldNames shouldBe Array(column0, column1, column2, column3needsEscaping)
    }

    "fail when 2 comma-separated formulas are provided" in {
      intercept[SqlColumnExpressionSyntaxException] {
        val dataFrame = applyFormulaToDataFrame(
          "MAXIMUM(x), SIN(x)",
          column1,
          "name",
          "x",
          prepareDataFrame())
        dataFrame.sparkDataFrame.collect()
      };()
    }

    "fail when formula is not correct" in {
      intercept[DOperationExecutionException] {
        val dataFrame =
          applyFormulaToDataFrame("MAXIMUM(", "name", column1, "x", prepareDataFrame())
        dataFrame.sparkDataFrame.collect()
      };()
    }

    "produce NaN if the argument given to the function is not correct" in {
      // counting LN from negative number
      val dataFrame =
        applyFormulaToDataFrame("LN(x)", column1, column3, "x", prepareDataFrame())
      val rowWithNegativeValue = 1
      val rowWithNaN = dataFrame.sparkDataFrame.collect()(rowWithNegativeValue)
      rowWithNaN.getDouble(resultColumn).isNaN shouldBe true
    }

    "always create nullable columns" in {
      runTest("cast(1.0 as double)", column1, column3, Seq(1.0, 1.0, 1.0, 1.0, 1.0))
    }
  }

  def runTest(
      formula: String,
      inputColumnName: String,
      outputColumnName: String,
      expectedValues: Seq[Any],
      columnAlias: String = "x") : Unit = {
    val dataFrame = applyFormulaToDataFrame(
      formula,
      inputColumnName,
      outputColumnName,
      columnAlias,
      prepareDataFrame())
    val rows = dataFrame.sparkDataFrame.collect()
    validateSchema(dataFrame.sparkDataFrame.schema)
    validateColumn(rows, expectedValues)
  }

  def applyFormulaToDataFrame(
      formula: String,
      inputColumnName: String,
      outputColumnName: String,
      columnAlias: String,
      df: DataFrame): DataFrame = {
    val transformation =
      prepareTransformation(formula, inputColumnName, outputColumnName, columnAlias)
    applyTransformation(transformation, df)
  }

  def applyTransformation(transformation: SqlColumnTransformer, df: DataFrame): DataFrame = {
    transformation.applyTransformationAndSerialization(tempDir, df)
  }

  def prepareTransformation(
      formula: String,
      inputColumnName: String,
      outputColumnName: String,
      columnAlias: String): SqlColumnTransformer = {
    val inPlace = NoInPlaceChoice()
      .setOutputColumn(outputColumnName)
    val single = SingleColumnChoice()
      .setInputColumn(NameSingleColumnSelection(inputColumnName))
      .setInPlace(inPlace)
    SqlColumnTransformer()
      .setFormula(formula)
      .setSingleOrMultiChoice(single)
      .setInputColumnAlias(columnAlias)
  }

  def validateSchema(schema: StructType): Unit = {
    schema.fieldNames shouldBe Array(column0, column1, column2, column3)
    schema.fields(0).dataType shouldBe StringType
    schema.fields(0) shouldBe 'nullable
    schema.fields(1).dataType shouldBe DoubleType
    schema.fields(1) shouldBe 'nullable
    schema.fields(2).dataType shouldBe DoubleType
    schema.fields(3) shouldBe 'nullable
    schema.fields(3).dataType shouldBe DoubleType
    schema.fields(3) shouldBe 'nullable
  }

  /**
   * Check if produced column matches the expected values
   */
  def validateColumn(
    rows: Array[Row], expectedValues: Seq[Any], column: Integer = resultColumn): Unit = {
    forAll(expectedValues.zipWithIndex) {
      case (expectedVal, i) =>
        val value = rows(i).get(column)
        value match {
          case d: Double => d should equal (expectedVal.asInstanceOf[Double] +- delta)
          case _ => expectedVal shouldBe value
        }
    }
  }

  def prepareDataFrame(): DataFrame = {
    val manualRowsSeq: Seq[Row] = Seq(
      Row("aaa", 1.0, 0.2),
      Row("bbb", -1.1, 2.2),
      Row("ccc", 1.2, null),
      Row("ddd", -1.3, 4.2),
      Row("eee", null, null))
    createDataFrame(manualRowsSeq, schema)
  }

  val schema: StructType = StructType(List(
    StructField(column0, StringType),
    StructField(column1, DoubleType),
    StructField(column2, DoubleType)))
}

redpajama_set_name: RedPajamaGithub


text:
Make Fedora
====================================
Very opinionated Makefile to help you install all things required on Fedora 24 linux more easily.

Usage
====================================

```shell

	$ curl -o /root/Makefile https://raw.githubusercontent.com/vodolaz095/makefedora/master/Makefile
	$ sudo make upgrade

```

List of available recipes
====================================
Majority of commands requires root access. The ones, that require ordinary user access,
are marked with `*`

- `clean`,`clear` - remove standard Fedora linux distribution programs i dislike

###Console###
This commands installs packages aimed to be used on headless server with remote access, no X server required

- `console` - install many usefull console commands and enable `SSHD` server for remote access.

- `docker` - install [http://docker.io/](Docker)  (with stack settings), enable and start it

- `golang` - install [https://golang.org/](Golang) compiler
- `golangEnv` `*` - set Golang environment for current user like they say in [https://golang.org/doc/code.html#Workspaces]

- `nginx` - install [http://nginx.org/](nginx) web server (with stack settings), enable and start it. With some showel dancing around selinux.
- `exposeNginx` - use firewalld to allow nginx listen on 80 and 443

- `redis` - install [http://redis.io/](redis) server (with stack settings), enable and start it
- `exposeRedis` - make redis listening on 0.0.0.0:6379 and tune firewalld accordingly. `TODO` - you need to test password on it

- `music` - install music related applications - [https://www.musicpd.org/](mpd), mp3 codecs and so on from [http://rpmfusion.org/](RPMFusion) repositories

- `mongo`  - install [https://www.mongodb.com/](mongo) server (with stack settings), enable and start it
- `exposeMongo` - make mongodb listening on 0.0.0.0:27017 and tune firewalld accordingly. `TODO` - you need to set password for it!

- `syncthing` - install [https://syncthing.net/](Syncthing) open source file synchronization system via [https://copr.fedoraproject.org/coprs/decathorpe/syncthing/](COPR) repo

- `tor` - install [http://torproject.org/](tor) proxy server (with stack settings), enable and start it

- `mariadb` - install [https://mariadb.org/](MariaDB) relational database and use sligh changed stack settings for it suitable for development (supports INNODB, use <64 MB of memory)
- `exposeMariadb` - make `mariadb` listen on 0.0.0.0:3306 and tune `firewalld` accordingly

- `nodejs` - install most recent [http://nodejs.org/](Nodejs) from [https://copr.fedoraproject.org/coprs/nibbler/nodejs/](Nibbler's repo)
- `heroku` - install [http://heroku.com/](Heroku) hosting provider [http://toolchain.heroku.com/](toolbelt)
- `aws` - install [http://aws.amazon.com/cli](Amazon Webservices Console)

###GUI commands###
This commands aimed to be executed on computer with X server enabled

- `gui` - install basic gui tools i like
- `viber` - install [http://viber.com/](viber) client
- `hipchat` - install [http://hipchat.com/](hipchat) client
- `foto` - install foto editing tools - [http://yorba.org/shotwell](Shotwell) [https://rawstudio.org/](Rawstudio) and [https://www.gimp.org/](Gimp)
- `video` - install [http://smplayer.sourceforge.net/](smplayer) for playing videos alongside with [https://github.com/alessio/camorama](Camorama) to play with webcam
- `steam` - install [http://store.steampowered.com/](Steam) client to play games
- `skype` - install [https://www.skype.com/](Skype) to chat
- `xflux` - install [https://justgetflux.com/linux.html](xflux) to make your eyes healthier

### Profile commands ###

- `env` - set user environment.

### Misc commands###

- `upgrade` - upgrade system
- `rpmfusion` - enable [http://rpmfusion.org/](RPMFusion) repositories

### Target commands###

- `server` - install as Anatolij like to set up homeserver - golang, docker, nodejs for development  and nginx  redis, mariadb servers  listening for connections
- `desctop` - install as Anatolij likes to set desctop/laptop PCs


text:
package org.valkyriercp.core;

/**
 * An interface to be implemented by objects that are capable of receiving messages to be
 * provided to the user.
 *
 * @author Keith Donald
 */
public interface Messagable extends PropertyChangePublisher {

	/** The name of the message property, to be used for publishing update events. */
	public static final String MESSAGE_PROPERTY = "message";

    /**
     * Set the message.
     * @param message The message.
     */
    public void setMessage(Message message);

}


text:
<?php

namespace Trive\Fiskal\Ui\DataProvider;

use Trive\Fiskal\Model\ResourceModel\Invoice\Listing\CollectionFactory;
use Magento\Framework\App\RequestInterface;
use Magento\Framework\App\Request\DataPersistorInterface;
use Trive\Fiskal\Api\Data\InvoiceInterface;

class InvoiceDataProvider extends \Magento\Ui\DataProvider\AbstractDataProvider
{
    /**
     * @var RequestInterface
     */
    private $request;

    /**
     * @var DataPersistorInterface
     */
    private $dataPersistor;

    /**
     * @param string                 $name
     * @param string                 $primaryFieldName
     * @param string                 $requestFieldName
     * @param CollectionFactory      $collectionFactory
     * @param RequestInterface       $request
     * @param DataPersistorInterface $dataPersistor
     * @param array                  $meta
     * @param array                  $data
     */
    public function __construct(
        $name,
        $primaryFieldName,
        $requestFieldName,
        CollectionFactory $collectionFactory,
        RequestInterface $request,
        DataPersistorInterface $dataPersistor,
        array $meta = [],
        array $data = []
    ) {
        parent::__construct(
            $name,
            $primaryFieldName,
            $requestFieldName,
            $meta,
            $data
        );
        $this->collection = $collectionFactory->create();
        $this->request = $request;
        $this->dataPersistor = $dataPersistor;
    }

    /**
     * {@inheritdoc}
     */
    public function getData()
    {
        $data = [];
        $dataFromForm = $this->dataPersistor->get('trive_fiskal_invoice');
        if (!empty($dataFromForm)) {
            $object = $this->collection->getNewEmptyItem();
            $object->setData($dataFromForm);
            $data[$object->getId()] = $object->getData();
            $this->dataPersistor->clear('trive_fiskal_invoice');
        } else {
            $identifier = $this->request->getParam($this->getRequestFieldName());
            /** @var InvoiceInterface $invoice */
            foreach ($this->getCollection()->getItems() as $invoice) {
                if ($identifier == $invoice->getId()) {
                    $data[$identifier] = $invoice->getData();
                }
            }
        }

        return $data;
    }
}


text:
DeleteTreeWorkItem::DeleteTreeWorkItem(const base::FilePath& root_path,
                                       const base::FilePath& temp_path)
    : root_path_(root_path), temp_path_(temp_path) {}

DeleteTreeWorkItem::~DeleteTreeWorkItem() = default;

bool DeleteTreeWorkItem::DoImpl() {
  if (root_path_.empty() || !base::PathExists(root_path_))
    return true;

  // If rollback is not enabled, try to delete the root without making a backup.
  // When that fails, fall back to moving the root to the temporary backup path.
  // Consumers are responsible for making a best-effort attempt to remove the
  // backup path. SelfCleaningTempDir is generally used for the backup path, so
  // in the worst case the file(s) will be removed after the next reboot.
  if (!rollback_enabled() && DeleteRoot())
    return true;

  // Attempt to move the root to the backup.
  return MoveRootToBackup();
}

void DeleteTreeWorkItem::RollbackImpl() {
  if (moved_to_backup_) {
    const base::FilePath& backup = GetBackupPath();
    DCHECK(!backup.empty());
    if (base::PathExists(backup))
      base::Move(backup, root_path_);
  }
}

const base::FilePath& DeleteTreeWorkItem::GetBackupPath() {
  if (backup_path_.empty()) {
    if (!backup_dir_.CreateUniqueTempDirUnderPath(temp_path_)) {
      PLOG(ERROR) << "Failed to get backup path in folder "
                  << temp_path_.value();
      return backup_path_;
    }
    backup_path_ = backup_dir_.GetPath().Append(root_path_.BaseName());
  }

  DCHECK(!backup_path_.empty());
  return backup_path_;
}

bool DeleteTreeWorkItem::DeleteRoot() {
  if (base::DeletePathRecursively(root_path_))
    return true;
  LOG(ERROR) << "Failed to delete " << root_path_.value();
  return false;
}

bool DeleteTreeWorkItem::MoveRootToBackup() {
  const base::FilePath& backup = GetBackupPath();
  if (backup.empty())
    return false;
  if (base::Move(root_path_, backup)) {
    moved_to_backup_ = true;
    return true;
  }
  PLOG(ERROR) << "Failed to move " << root_path_.value() << " to backup path "
              << backup.value();
  return false;
}


text:
"use strict";
import appolo = require('appolo');
import router from '../routes/router';
import app, {App} from '../app/app';
import view, {} from '../app/view';
import    path = require('path');
import    http = require('http');
import    https = require('https');
import    fs = require('fs');
import    Q = require('bluebird');
import    _ = require('lodash');
import {IOptions} from "../interfaces/IOptions";
import ErrnoException = NodeJS.ErrnoException;

export class Launcher extends appolo.Launcher {

    protected _server: http.Server | https.Server;
    protected _options: IOptions;
    protected _port: number;

    protected readonly Defaults: IOptions = {
        startMessage: "Appolo Server listening on port: ${port} version:${version} environment: ${environment}",
        startServer: true,
        errorStack: false,
        errorMessage: true,
        maxRouteCache: 10000,
        qsParser: "qs",
        urlParser: "fast",
        viewExt: "html",
        viewEngine: null,
        viewFolder: "server/views",
        validatorOptions: {
            abortEarly: false,
            allowUnknown: true,
            stripUnknown: true
        }
    };

    public async launch(config?: IOptions, callback?: (err?: any,app?:App) => void): Promise<App> {

        try {
            appolo.inject.addObject('appolo', require("../../index"));

            this._options = this.loadOptions(config);

            super.loadEnvironments();

            this.createServer();

            this.bindProcessEvents();

            await super.loadModules();

            super.loadFiles();

            super.loadInjector();

            this.setPort();

            this.loadCustomConfigurations();

            router.initialize(null, this._options.validatorOptions);

            app.initialize(this._options);

            view.initialize(this._options);

            await super.loadBootStrap();

            await this._startServer();

            callback && callback(null,app);

            this.fireEvent('appolo-launched');

            return app;

        } catch (e) {
            if (!callback) {
                throw e;
            }

            callback(e);
        }
    }

    public get options(): IOptions {
        return this._options
    }

    protected loadOptions(config: IOptions): IOptions {

        let dto = super.loadOptions(config);

        return _.merge(this.Defaults, dto || {});
    }

    protected bindProcessEvents() {
        process.on('uncaughtException', (err: ErrnoException) => {
            if (err.code !== 'EADDRINUSE') {
                console.error(err.stack || err.toString())
                return;
            }

            console.error(`EADDRINUSE!!!! address in use port: ${this._port}`);
            process.exit(1);
        })
    }

    protected createServer() {

        let server: http.Server | https.Server;

        let ssl = this._options.ssl || appolo.environment.ssl;

        if (ssl && ssl.key && ssl.cert) {
            let options = {
                key: fs.readFileSync(ssl.key),
                cert: fs.readFileSync(ssl.cert)
            };

            server = https.createServer(options, (req, res) => app.handleRequest(req, res));

        } else {

            server = http.createServer((req, res) => app.handleRequest(req, res));
        }

        this._server = server;

        appolo.inject.addObject('app', app);

        appolo.inject.addObject('injector', appolo.inject);

        appolo.inject.addObject('router', router);

        appolo.inject.addObject('httpServer', server);
    }

    protected setPort() {

        this._port = process.env.APP_PORT || process.env.PORT || this._options.port || appolo.environment.port || 8080;
    }

    protected loadCustomConfigurations() {

        let allPath = path.join(this._options.root, 'config/express/all.js'),
            environmentPath = path.join(this._options.root, 'config/express/', this._options.environment + '.js');

        _.forEach([allPath, environmentPath], (filePath) => {

            if (!fs.existsSync(filePath)) {
                return;
            }

            this.cachedRequire.push(filePath);

            let func = require(filePath);

            let args = appolo.Util.getFunctionArgs(func);

            let dependencies = _.map(args, (arg) => appolo.inject.getObject(arg));

            func.apply(func, dependencies);

        });
    }

    protected async _startServer() {

        if (!this._options.startServer) {
            return
        }

        return this.startServer();
    }

    public async startServer() {

        await Q.fromCallback((callback: (err: any, result?: any) => void) => this._server.listen(this._port, callback));

        let msg = _.template(this._options.startMessage, {interpolate: /\${([\s\S]+?)}/g})({
            port: this._port,
            version: appolo.environment.version,
            environment: appolo.environment.type
        });

        console.log(msg);

        this.fireEvent('appolo-server-started');

    }

    public get app(): App {
        return app
    }

    public handleRequest(req, res): void {
        app.handleRequest(req, res)
    }

    public get server(): http.Server | https.Server {
        return this._server
    }

    public get port(): number {
        return this._port
    }

    public reset(isSoftReset?: boolean) {

        super.reset(isSoftReset);

        if (!isSoftReset) {

            _.forEach(this.cachedRequire, (filePath) => delete require.cache[filePath]);

            router.reset();
        }

        this.cachedRequire.length = 0;
        this._options = null;

        try {
            this._server.close();
        } catch (e) {
            console.log("failed to close server", e)
        }

        process.removeAllListeners();
    }

    public softReset() {
        this.reset(true)
    }
}

export default new Launcher();