text:
package org.drools.mvel.parser.printer;

import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;

import com.github.javaparser.ast.NodeList;
import com.github.javaparser.ast.comments.Comment;
import com.github.javaparser.ast.expr.AnnotationExpr;
import com.github.javaparser.ast.expr.Expression;
import com.github.javaparser.ast.nodeTypes.NodeWithTypeArguments;
import com.github.javaparser.ast.stmt.Statement;
import com.github.javaparser.ast.type.Type;
import com.github.javaparser.printer.PrettyPrintVisitor;
import com.github.javaparser.printer.PrettyPrinterConfiguration;
import org.drools.mvel.parser.ast.expr.BigDecimalLiteralExpr;
import org.drools.mvel.parser.ast.expr.BigIntegerLiteralExpr;
import org.drools.mvel.parser.ast.expr.DrlNameExpr;
import org.drools.mvel.parser.ast.expr.DrlxExpression;
import org.drools.mvel.parser.ast.expr.FullyQualifiedInlineCastExpr;
import org.drools.mvel.parser.ast.expr.HalfBinaryExpr;
import org.drools.mvel.parser.ast.expr.HalfPointFreeExpr;
import org.drools.mvel.parser.ast.expr.InlineCastExpr;
import org.drools.mvel.parser.ast.expr.ListCreationLiteralExpression;
import org.drools.mvel.parser.ast.expr.ListCreationLiteralExpressionElement;
import org.drools.mvel.parser.ast.expr.MapCreationLiteralExpression;
import org.drools.mvel.parser.ast.expr.MapCreationLiteralExpressionKeyValuePair;
import org.drools.mvel.parser.ast.expr.ModifyStatement;
import org.drools.mvel.parser.ast.expr.NullSafeFieldAccessExpr;
import org.drools.mvel.parser.ast.expr.NullSafeMethodCallExpr;
import org.drools.mvel.parser.ast.expr.OOPathChunk;
import org.drools.mvel.parser.ast.expr.OOPathExpr;
import org.drools.mvel.parser.ast.expr.PointFreeExpr;
import org.drools.mvel.parser.ast.expr.RuleBody;
import org.drools.mvel.parser.ast.expr.RuleDeclaration;
import org.drools.mvel.parser.ast.expr.TemporalChunkExpr;
import org.drools.mvel.parser.ast.expr.TemporalLiteralChunkExpr;
import org.drools.mvel.parser.ast.expr.TemporalLiteralExpr;
import org.drools.mvel.parser.ast.expr.TemporalLiteralInfiniteChunkExpr;
import org.drools.mvel.parser.ast.expr.WithStatement;
import org.drools.mvel.parser.ast.visitor.DrlVoidVisitor;

import static com.github.javaparser.utils.Utils.isNullOrEmpty;
import static org.drools.mvel.parser.printer.PrintUtil.printConstraint;

public class ConstraintPrintVisitor extends PrettyPrintVisitor implements DrlVoidVisitor<Void> {

    public ConstraintPrintVisitor(PrettyPrinterConfiguration prettyPrinterConfiguration) {
        super(prettyPrinterConfiguration);
    }

    @Override
    public void visit( RuleDeclaration n, Void arg ) {
        printComment(n.getComment(), arg);

        for (AnnotationExpr ae : n.getAnnotations()) {
            ae.accept(this, arg);
            printer.print(" ");
        }

        printer.print("rule ");
        n.getName().accept(this, arg);
        printer.println(" {");
        n.getRuleBody().accept(this, arg);
        printer.println("}");
    }

    @Override
    public void visit( RuleBody ruleBody, Void arg ) {
    }

    @Override
    public void visit( InlineCastExpr inlineCastExpr, Void arg ) {
        printComment(inlineCastExpr.getComment(), arg);
        inlineCastExpr.getExpression().accept( this, arg );
        printer.print( "#" );
        inlineCastExpr.getType().accept( this, arg );
    }

    @Override
    public void visit( FullyQualifiedInlineCastExpr inlineCastExpr, Void arg ) {
        printComment(inlineCastExpr.getComment(), arg);
        inlineCastExpr.getScope().accept( this, arg );
        printer.print( "#" );
        inlineCastExpr.getName().accept( this, arg );
        if (inlineCastExpr.hasArguments()) {
            printer.print( "(" );
            inlineCastExpr.getArguments().accept( this, arg );
            printer.print( ")" );
        }
    }

    @Override
    public void visit( NullSafeFieldAccessExpr nullSafeFieldAccessExpr, Void arg ) {
        printComment(nullSafeFieldAccessExpr.getComment(), arg);
        nullSafeFieldAccessExpr.getScope().accept( this, arg );
        printer.print( "!." );
        nullSafeFieldAccessExpr.getName().accept( this, arg );
    }

    @Override
    public void visit( NullSafeMethodCallExpr nullSafeMethodCallExpr, Void arg ) {
        printComment(nullSafeMethodCallExpr.getComment(), arg);
        Optional<Expression> scopeExpression = nullSafeMethodCallExpr.getScope();
        if (scopeExpression.isPresent()) {
            scopeExpression.get().accept( this, arg );
            printer.print("!.");
        }
        printTypeArgs(nullSafeMethodCallExpr, arg);
        nullSafeMethodCallExpr.getName().accept( this, arg );
        printArguments(nullSafeMethodCallExpr.getArguments(), arg);
    }

    @Override
    public void visit( PointFreeExpr pointFreeExpr, Void arg ) {
        printComment(pointFreeExpr.getComment(), arg);
        pointFreeExpr.getLeft().accept( this, arg );
        if(pointFreeExpr.isNegated()) {
            printer.print(" not");
        }
        printer.print(" ");
        pointFreeExpr.getOperator().accept( this, arg );
        if (pointFreeExpr.getArg1() != null) {
            printer.print("[");
            pointFreeExpr.getArg1().accept( this, arg );
            if (pointFreeExpr.getArg2() != null) {
                printer.print(",");
                pointFreeExpr.getArg2().accept( this, arg );
            }
            if (pointFreeExpr.getArg3() != null) {
                printer.print(",");
                pointFreeExpr.getArg3().accept( this, arg );
            }
            if (pointFreeExpr.getArg4() != null) {
                printer.print(",");
                pointFreeExpr.getArg4().accept( this, arg );
            }
            printer.print("]");
        }
        printer.print(" ");
        NodeList<Expression> rightExprs = pointFreeExpr.getRight();
        if (rightExprs.size() == 1) {
            rightExprs.get(0).accept( this, arg );
        } else {
            printer.print("(");
            if(rightExprs.isNonEmpty()) {
                rightExprs.get(0).accept(this, arg);
            }
            for (int i = 1; i < rightExprs.size(); i++) {
                printer.print(", ");
                rightExprs.get(i).accept( this, arg );
            }
            printer.print(")");
        }
    }

    @Override
    public void visit(TemporalLiteralExpr temporalLiteralExpr, Void arg) {
        printComment(temporalLiteralExpr.getComment(), arg);
        NodeList<TemporalChunkExpr> chunks = temporalLiteralExpr.getChunks();
        for (TemporalChunkExpr c : chunks) {
            c.accept(this, arg);
        }
    }

    @Override
    public void visit(TemporalLiteralChunkExpr temporalLiteralExpr, Void arg) {
        printComment(temporalLiteralExpr.getComment(), arg);
        printer.print("" + temporalLiteralExpr.getValue());
        switch (temporalLiteralExpr.getTimeUnit()) {
            case MILLISECONDS:
                printer.print("ms");
                break;
            case SECONDS:
                printer.print("s");
                break;
            case MINUTES:
                printer.print("m");
                break;
            case HOURS:
                printer.print("h");
                break;
            case DAYS:
                printer.print("d");
                break;
        }
    }

    @Override
    public void visit(TemporalLiteralInfiniteChunkExpr temporalLiteralInfiniteChunkExpr, Void arg) {
        printer.print("*");
    }

    @Override
    public void visit( DrlxExpression expr, Void arg ) {
        if (expr.getBind() != null) {
            expr.getBind().accept( this, arg );
            printer.print( " : " );
        }
        expr.getExpr().accept(this, arg);
    }

    @Override
    public void visit(OOPathExpr oopathExpr, Void arg ) {
        printComment(oopathExpr.getComment(), arg);
        NodeList<OOPathChunk> chunks = oopathExpr.getChunks();
        for (int i = 0; i <  chunks.size(); i++) {
            final OOPathChunk chunk = chunks.get(i);
            printer.print(chunk.isSingleValue() ? "." : "/");
            chunk.accept(this, arg);
            printer.print(chunk.getField().toString());

            if (chunk.getInlineCast() != null) {
                printer.print("#");
                chunk.getInlineCast().accept( this, arg );
            }

            List<DrlxExpression> condition = chunk.getConditions();
            final Iterator<DrlxExpression> iterator = condition.iterator();
            if (!condition.isEmpty()) {
                printer.print("[");
                DrlxExpression first = iterator.next();
                first.accept(this, arg);
                while(iterator.hasNext()) {
                    printer.print(",");
                    iterator.next().accept(this, arg);
                }
                printer.print("]");
            }
        }
    }

    @Override
    public void visit(HalfBinaryExpr n, Void arg) {
        printComment(n.getComment(), arg);
        printer.print(n.getOperator().asString());
        printer.print(" ");
        n.getRight().accept(this, arg);
    }

    @Override
    public void visit(HalfPointFreeExpr pointFreeExpr, Void arg ) {
        printComment(pointFreeExpr.getComment(), arg);
        if(pointFreeExpr.isNegated()) {
            printer.print("not ");
        }
        pointFreeExpr.getOperator().accept( this, arg );
        if (pointFreeExpr.getArg1() != null) {
            printer.print("[");
            pointFreeExpr.getArg1().accept( this, arg );
            if (pointFreeExpr.getArg2() != null) {
                printer.print(",");
                pointFreeExpr.getArg2().accept( this, arg );
            }
            if (pointFreeExpr.getArg3() != null) {
                printer.print(",");
                pointFreeExpr.getArg3().accept( this, arg );
            }
            if (pointFreeExpr.getArg4() != null) {
                printer.print(",");
                pointFreeExpr.getArg4().accept( this, arg );
            }
            printer.print("]");
        }
        printer.print(" ");
        NodeList<Expression> rightExprs = pointFreeExpr.getRight();
        if (rightExprs.size() == 1) {
            rightExprs.get(0).accept( this, arg );
        } else {
            printer.print("(");
            rightExprs.get(0).accept( this, arg );
            for (int i = 1; i < rightExprs.size(); i++) {
                printer.print(", ");
                rightExprs.get(i).accept( this, arg );
            }
            printer.print(")");
        }
    }

    @Override
    public void visit(BigDecimalLiteralExpr bigDecimalLiteralExpr, Void arg) {
        printer.print(bigDecimalLiteralExpr.asBigDecimal().toString());
        printer.print("B");
    }

    @Override
    public void visit(BigIntegerLiteralExpr bigIntegerLiteralExpr, Void arg) {
        printer.print(bigIntegerLiteralExpr.asBigInteger().toString());
        printer.print("I");
    }

    @Override
    public void visit(ModifyStatement modifyExpression, Void arg) {
        printer.print("modify (");
        modifyExpression.getModifyObject().accept(this, arg);
        printer.print(") { ");

        String expressionWithComma = modifyExpression.getExpressions()
                .stream()
                .filter(Objects::nonNull)
                .filter(Statement::isExpressionStmt)
                .map(n -> printConstraint(n.asExpressionStmt().getExpression()))
                .collect(Collectors.joining(", "));

        printer.print(expressionWithComma);
        printer.print(" }");

        printer.print(";");
    }

    @Override
    public void visit(WithStatement withExpression, Void arg) {
        printer.print("with (");
        withExpression.getWithObject().accept(this, arg);
        printer.print(") { ");

        String expressionWithComma = withExpression.getExpressions()
                .stream()
                .filter(Objects::nonNull)
                .filter(Statement::isExpressionStmt)
                .map(n -> printConstraint(n.asExpressionStmt().getExpression()))
                .collect(Collectors.joining(", "));

        printer.print(expressionWithComma);
        printer.print(" }");

        printer.print(";");
    }

    public void printComment(final Optional<Comment> comment, final Void arg) {
        comment.ifPresent(c -> c.accept(this, arg));
    }

    public void printTypeArgs(final NodeWithTypeArguments<?> nodeWithTypeArguments, final Void arg) {
        NodeList<Type> typeArguments = nodeWithTypeArguments.getTypeArguments().orElse(null);
        if (!isNullOrEmpty(typeArguments)) {
            printer.print("<");
            for (final Iterator<Type> i = typeArguments.iterator(); i.hasNext(); ) {
                final Type t = i.next();
                t.accept(this, arg);
                if (i.hasNext()) {
                    printer.print(", ");
                }
            }
            printer.print(">");
        }
    }

    public void printArguments(final NodeList<Expression> args, final Void arg) {
        printer.print("(");
        if (!isNullOrEmpty(args)) {
            boolean columnAlignParameters = (args.size() > 1) && configuration.isColumnAlignParameters();
            if (columnAlignParameters) {
                printer.indentWithAlignTo(printer.getCursor().column);
            }
            for (final Iterator<Expression> i = args.iterator(); i.hasNext(); ) {
                final Expression e = i.next();
                e.accept(this, arg);
                if (i.hasNext()) {
                    printer.print(",");
                    if (columnAlignParameters) {
                        printer.println();
                    } else {
                        printer.print(" ");
                    }
                }
            }
            if (columnAlignParameters) {
                printer.unindent();
            }
        }
        printer.print(")");
    }

    @Override
    public void visit(DrlNameExpr n, Void arg) {
        printComment(n.getComment(), arg);
        java.util.stream.IntStream.range(0, n.getBackReferencesCount()).forEach(s -> printer.print("../"));
        n.getName().accept(this, arg);
    }

    @Override
    public void visit(MapCreationLiteralExpression n, Void arg) {
        printer.print("[");

        Iterator<Expression> expressions = n.getExpressions().iterator();
        while(expressions.hasNext()) {
            expressions.next().accept(this, arg);
            if(expressions.hasNext()) {
                printer.print(", ");
            }
        }
        printer.print("]");
    }

    @Override
    public void visit(MapCreationLiteralExpressionKeyValuePair n, Void arg) {
        n.getKey().accept(this, arg);
        printer.print(" : ");
        n.getValue().accept(this, arg);
    }

    @Override
    public void visit(ListCreationLiteralExpression n, Void arg) {
        printer.print("[");

        Iterator<Expression> expressions = n.getExpressions().iterator();
        while(expressions.hasNext()) {
            expressions.next().accept(this, arg);
            if(expressions.hasNext()) {
                printer.print(", ");
            }
        }
        printer.print("]");
    }

    @Override
    public void visit(ListCreationLiteralExpressionElement n, Void arg) {
        n.getValue().accept(this, arg);
    }
}


text:
JRUBY NOKOGIRI PENDING - MathML inline with HTML
(JRuby Nokogiri is broken for empty tags: https://github.com/sparklemotion/nokogiri/issues/971)
*** Parameters: ***
{}
*** Markdown input: ***
In <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline" class="maruku-mathml"><mi>SU</mi><mo stretchy="false">(</mo><mn>3</mn><mo stretchy="false">)</mo></math>, <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline" class="maruku-mathml"><semantics><annotation-xml encoding="SVG1.1">
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="16" viewBox="0 0 30 16">
  <desc>Rank-2 Symmetric Tensor Representation</desc>
  <g transform="translate(5,5)" fill="#FCC" stroke="#000" stroke-width="2">
    <rect width="10" height="10"></rect>
    <rect width="10" height="10" x="10"></rect>
  </g>
</svg>
</annotation-xml></semantics><mo>=</mo><semantics><annotation-xml encoding="SVG1.1">
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="16" viewBox="0 0 30 16">
  <desc>Rank-2 Symmetric Tensor Representation</desc>
  <g transform="translate(5,5)" fill="#FCC" stroke="#000" stroke-width="2">
    <rect width="10" height="10"></rect>
    <rect width="10" height="10" x="10"></rect>
  </g>
</svg>
</annotation-xml></semantics></math>.

<math xmlns="http://www.w3.org/1998/Math/MathML" display="block" class="maruku-mathml"><semantics><annotation-xml encoding="SVG1.1">
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="16" viewBox="0 0 30 16">
  <desc>Rank-2 Symmetric Tensor Representation</desc>
  <g transform="translate(5,5)" fill="#FCC" stroke="#000" stroke-width="2">
    <rect width="10" height="10"></rect>
    <rect width="10" height="10" x="10"></rect>
  </g>
</svg>
</annotation-xml></semantics><mo>=</mo><semantics><annotation-xml encoding="SVG1.1">
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="16" viewBox="0 0 30 16">
  <desc>Rank-2 Symmetric Tensor Representation</desc>
  <g transform="translate(5,5)" fill="#FCC" stroke="#000" stroke-width="2">
    <rect width="10" height="10"></rect>
    <rect width="10" height="10" x="10"></rect>
  </g>
</svg>
</annotation-xml></semantics></math>

*** Output of inspect ***
md_el(:document,[
	md_par(["In ",
	  md_html("<math xmlns=\"http://www.w3.org/1998/Math/MathML\" display=\"inline\" class=\"maruku-mathml\"><mi>SU</mi><mo stretchy=\"false\">(</mo><mn>3</mn><mo stretchy=\"false\">)</mo></math>"),
	  ", ",
	  md_html("<math xmlns=\"http://www.w3.org/1998/Math/MathML\" display=\"inline\" class=\"maruku-mathml\"><semantics><annotation-xml encoding=\"SVG1.1\">\n<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"30\" height=\"16\" viewBox=\"0 0 30 16\">\n  <desc>Rank-2 Symmetric Tensor Representation</desc>\n  <g transform=\"translate(5,5)\" fill=\"#FCC\" stroke=\"#000\" stroke-width=\"2\">\n    <rect width=\"10\" height=\"10\"></rect>\n    <rect width=\"10\" height=\"10\" x=\"10\"></rect>\n  </g>\n</svg>\n</annotation-xml></semantics><mo>=</mo><semantics><annotation-xml encoding=\"SVG1.1\">\n<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"30\" height=\"16\" viewBox=\"0 0 30 16\">\n  <desc>Rank-2 Symmetric Tensor Representation</desc>\n  <g transform=\"translate(5,5)\" fill=\"#FCC\" stroke=\"#000\" stroke-width=\"2\">\n    <rect width=\"10\" height=\"10\"></rect>\n    <rect width=\"10\" height=\"10\" x=\"10\"></rect>\n  </g>\n</svg>\n</annotation-xml></semantics></math>"),
	  "."
	  ]),
	  md_html("<math xmlns=\"http://www.w3.org/1998/Math/MathML\" display=\"block\" class=\"maruku-mathml\"><semantics><annotation-xml encoding=\"SVG1.1\">\n<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"30\" height=\"16\" viewBox=\"0 0 30 16\">\n  <desc>Rank-2 Symmetric Tensor Representation</desc>\n  <g transform=\"translate(5,5)\" fill=\"#FCC\" stroke=\"#000\" stroke-width=\"2\">\n    <rect width=\"10\" height=\"10\"></rect>\n    <rect width=\"10\" height=\"10\" x=\"10\"></rect>\n  </g>\n</svg>\n</annotation-xml></semantics><mo>=</mo><semantics><annotation-xml encoding=\"SVG1.1\">\n<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"30\" height=\"16\" viewBox=\"0 0 30 16\">\n  <desc>Rank-2 Symmetric Tensor Representation</desc>\n  <g transform=\"translate(5,5)\" fill=\"#FCC\" stroke=\"#000\" stroke-width=\"2\">\n    <rect width=\"10\" height=\"10\"></rect>\n    <rect width=\"10\" height=\"10\" x=\"10\"></rect>\n  </g>\n</svg>\n</annotation-xml></semantics></math>")
],{},[])
*** Output of to_html ***

In <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline" class="maruku-mathml"><mi>SU</mi><mo stretchy="false">(</mo><mn>3</mn><mo stretchy="false">)</mo></math>, <math xmlns="http://www.w3.org/1998/Math/MathML" display="inline" class="maruku-mathml"><semantics><annotation-xml encoding="SVG1.1">
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="16" viewBox="0 0 30 16">
  <desc>Rank-2 Symmetric Tensor Representation</desc>
  <g transform="translate(5,5)" fill="#FCC" stroke="#000" stroke-width="2">
    <rect width="10" height="10"></rect>
    <rect width="10" height="10" x="10"></rect>
  </g>
</svg>
</annotation-xml></semantics><mo>=</mo><semantics><annotation-xml encoding="SVG1.1">
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="16" viewBox="0 0 30 16">
  <desc>Rank-2 Symmetric Tensor Representation</desc>
  <g transform="translate(5,5)" fill="#FCC" stroke="#000" stroke-width="2">
    <rect width="10" height="10"></rect>
    <rect width="10" height="10" x="10"></rect>
  </g>
</svg>
</annotation-xml></semantics></math>.

<math xmlns="http://www.w3.org/1998/Math/MathML" display="block" class="maruku-mathml"><semantics><annotation-xml encoding="SVG1.1">
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="16" viewBox="0 0 30 16">
  <desc>Rank-2 Symmetric Tensor Representation</desc>
  <g transform="translate(5,5)" fill="#FCC" stroke="#000" stroke-width="2">
    <rect width="10" height="10"></rect>
    <rect width="10" height="10" x="10"></rect>
  </g>
</svg>
</annotation-xml></semantics><mo>=</mo><semantics><annotation-xml encoding="SVG1.1">
<svg xmlns="http://www.w3.org/2000/svg" width="30" height="16" viewBox="0 0 30 16">
  <desc>Rank-2 Symmetric Tensor Representation</desc>
  <g transform="translate(5,5)" fill="#FCC" stroke="#000" stroke-width="2">
    <rect width="10" height="10"></rect>
    <rect width="10" height="10" x="10"></rect>
  </g>
</svg>
</annotation-xml></semantics></math>

redpajama_set_name: RedPajamaGithub


text:
package terrastore.server.impl.support;

import java.io.IOException;
import java.io.OutputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.ext.MessageBodyWriter;
import javax.ws.rs.ext.Provider;
import terrastore.common.ErrorMessage;
import terrastore.util.json.JsonUtils;

/**
 * @author Sergio Bossa
 */
@Provider
@Produces("application/json")
public class JsonErrorMessageProvider implements MessageBodyWriter<ErrorMessage> {

    public boolean isWriteable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
        return ErrorMessage.class.isAssignableFrom(type);
    }

    public void writeTo(ErrorMessage errorMessage, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, Object> httpHeaders, OutputStream entityStream) throws IOException, WebApplicationException {
        JsonUtils.write(errorMessage, entityStream);
    }

    public long getSize(ErrorMessage errorMessage, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
        return -1;
    }
}


text:
begin
  # Require the preresolved locked set of gems.
  require File.expand_path('../../.bundle/environment', __FILE__)
rescue LoadError
  # Fallback on doing the resolve at runtime.
  require 'rubygems'
  require 'bundler'
  Bundler.setup
end

Bundler.require(:test, :default)

require 'eson-dsl'
require 'faraday'
require 'elasticsearch-node/external'

module Node
  module External
    def self.instance
      @node ||= begin
        node = ElasticSearch::Node::External.new("gateway.type" => "none") do
          def client
            conn = Faraday.new(:url => "http://#{self.ip}:#{self.port}") do |builder|
              #builder.response :raise_error
              builder.request :json
              builder.adapter :net_http
            end
          end
        end
        at_exit do
          node.close
        end
        node
      end
    end
  end

end
Node::External.instance

require 'riot'

#Riot.dots

class EsonQueries < Riot::ContextMiddleware
  register

  def call(context)
    middleware.call(context)

    return unless context.option(:query_name)

    type = context.option(:type) || "bar"
    index = context.option(:index) || "default"
    file = context.option(:query_name)

    Dir["#{file}*.documents"].each do |f|

      f.match(/\A[^.]+(?:\.([^.]*))?.documents\Z/)
      type = $1 if $1

      context.setup(true) do
        create_index :index => index

        File.open(f) do |p|
          begin
            p.each_line do |l|
              index_doc :index => index,
                        :source => l,
                        :type => type
            end
          rescue Exception => e
            puts e.inspect
            raise e
          end
        end
      end

    end

    Dir["#{file}*.mapping"].each do |f|
      context.setup(true) do
        f.match(/\A[^.]+(?:\.([^.]*))?.mapping\Z/)
        type = $1 if $1

        delete_mapping :index => index, :type => type

        create_index :index => index
        put_mapping :index => index,
                    :type => type,
                    :source => File.read(f)
      end
    end

    if File.exists?("#{file}.rb")
      context.asserts(:to_query_hash).equals(eval(File.read("#{file}.rb")))
    end

    if File.exists?("#{file}.json")
      context.asserts("json format") do
        MultiJson.decode(MultiJson.encode(topic.to_query_hash))
      end.equals(MultiJson.decode(File.read("#{file}.json").strip))
    end

    context.asserts("search status") do
      begin
        result = search :index => index, :type => type, :query => topic
        result.status
      rescue Exception => e
        puts e.inspect
        raise e
      end
    end.equals(200)

  end
end

class Riot::Context
  def query_name(name)
    set :query_name, name
  end
end

class Riot::Situation
  def node
    Node::External.instance
  end

  def create_index(opts)
    node.client.post "/#{opts[:index]}"
  end

  def index_doc(opts)
    node.client.post "/#{opts[:index]}/#{opts[:type]}", opts[:source]
  end

  def delete_mapping(opts)
    node.client.delete "/#{opts[:index]}/#{opts[:type]}"
  end

  def put_mapping(opts)
    node.client.put "/#{opts[:index]}/#{opts[:type]}/_mapping", opts[:source]
  end

  def search(opts)
    source = opts[:query].to_query_hash
    if opts[:type]
      node.client.post "/#{opts[:index]}/#{opts[:type]}/_search", source
    else
      node.client.post "/#{opts[:index]}/_search", source
    end
  end

  def example(file)
    eval(File.read(File.expand_path(File.join('examples', "#{file}.rb"))))
  end
end


text:
/*! \file EventFactory.hpp
\brief  Factory to generate user-defined Events
*/

#ifndef EventFactory_H
#define EventFactory_H

# include "Event.hpp"
# include <map>

/** Namespace for Events factory related objects. */
namespace EventFactory
{

/** A pointer to function, returning a pointer to Event, built with its type (ie class name) and a pointer to Model.*/
typedef SP::Event(*object_creator)(double, int);

/** The type of the factory map */
typedef std::map<int, object_creator> MapFactory;

/** An iterator through the MapFactory */
typedef MapFactory::iterator MapFactoryIt;

/** Template function to return a new object of type SubType
 * \param time time of the Event
 * \param type type of the Event
 * \return an Event
 */
template<class SubType> SP::Event factory(double time, int type)
{
  SP::Event e(new SubType(time, type));
  return e;
}

/** Registry Class for Events.
 *
 *
 */
class Registry
{

private :

  /** map that links a std::string, the type of the class, to a pointer to function, used to build the object. */
  MapFactory factory_map;

public :

  /** get access to the Registry
   * \return reference to the registry
   */
  static Registry& get() ;

  /** Add an object_creator into the factory_map, factory_map[name] = object.
   * \param type the type of the object added
   * \param creator object creator
   */
  void add(int type, object_creator creator);

  /**
   *  \param time time of Event
   *  \param type type of Event
   *  \return an Event
   */
  SP::Event instantiate(double time, int type);

} ;

/** Registration Class for Events.
 *
 *
 * Class used for auto-registration of Event-type objects.
 *
 */
class Registration
{

public :

  /** To register some new object into the factory
   * \param type the type of the object added
   * \param creator object creator
   */
  Registration(int type, object_creator creator);
} ;

}
// end of namespace EventFactory

#define AUTO_REGISTER_EVENT(class_name,class_type) EventFactory::Registration _registration_## class_type(class_name,&EventFactory::factory<class_type>);
#endif


text:
package characteristic

import (
	"net"
)

type String struct {
	*Characteristic
}

func NewString(typ string) *String {
	char := NewCharacteristic(typ)
	char.Format = FormatString

	return &String{char}
}

// SetValue sets a value
func (c *String) SetValue(str string) {
	c.UpdateValue(str)
}

// GetValue returns the value as string
func (c *String) GetValue() string {
	return c.Characteristic.GetValue().(string)
}

// OnValueRemoteGet calls fn when the value was read by a client.
func (c *String) OnValueRemoteGet(fn func() string) {
	c.OnValueGet(func() interface{} {
		return fn()
	})
}

// OnValueRemoteUpdate calls fn when the value was updated by a client.
func (c *String) OnValueRemoteUpdate(fn func(string)) {
	c.OnValueUpdateFromConn(func(conn net.Conn, c *Characteristic, new, old interface{}) {
		fn(new.(string))
	})
}


text:
title:  UJosefa, Inkosana YaseGibhithe
date:  11/06/2022
---

### Funda Lokhu Kuleliviki
Gen. 41:37–46; 1 Amakhosi 3:12; Genesis 42; Rom. 5:7–11; Genesis 43; Genesis 44, Genesis 45.

>
Indimana Yenkumbulo

> "Wathi uFaro kuJosefa: 'Bheka, sengikubeke phezu kwezwe lonke laseGibithe'" (Genesis 41:41, NKJV).

UJosefa manje ungumholi waseGibhithe, futhi Abafowabo bazokhothama phambi kwakhe bengazi ukuthi ungubani (Genesisi 42). Abafowabo bakaJosefa babezozithoba ngenkathi uJosefa ebaphoqa ukuba babuye no-Benjamin (Genesisi 43), futhi—lapho ukuphepha kuka-Benjamin, besaba ukuthi, kusengozini (Genesisi 44)—bazoncengela umusa phambi kwalendoda enamandla, abayibuka "njengoFaro." Ekugcineni, lapho uJosefa ezembula ukuthi ungubani, bazoqonda ukuthi, ngaphandle kwalokho ababekwenzile, uNkulunkulu waveza ubuhle kukho konke.

Ngokuthathekisayo, zonke izigameko ezilandelayo, okumele ngabe zimayelana nokuchuma kukaJosefa, zimayelana kakhulu nokuphenduka kwabafowabo. Ukuya bebuya phakathi kukaJosefa nobaba wabo, izingqinamba abanqwamana nazo, zabenza bakhumbula izenzo zabo zenkohlakalo ngakuJosefa nobaba wabo. Abafowabo bakaJosefa baphila onke lawo mava njengesahlulelo sezulu. Kepha Isiphetho esinomunyu, esiletha kubo bonke izinyembezi kanye nenjabulo, siqukethe umbiko wentethelelo kubo, nakuba benze bonke lobu bubi.

_*Funda isifundo saleliviki ukulungiselela iSabatha, uNhlangulana 18._


text:
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models, connection

class Migration(SchemaMigration):

    def forwards(self, orm):
        table_names = connection.introspection.table_names()
        if 'cmsplugin_flash' in table_names:
            db.rename_table('cmsplugin_flash', 'djangocms_flash_flash')
        elif 'flash_flash' in table_names:
            db.rename_table('flash_flash', 'djangocms_flash_flash')
        else:
            # Adding model 'Flash'
            db.create_table(u'djangocms_flash_flash', (
                (u'cmsplugin_ptr', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['cms.CMSPlugin'], unique=True, primary_key=True)),
                ('file', self.gf('django.db.models.fields.files.FileField')(max_length=100)),
                ('width', self.gf('django.db.models.fields.CharField')(max_length=6)),
                ('height', self.gf('django.db.models.fields.CharField')(max_length=6)),
            ))
            db.send_create_signal(u'djangocms_flash', ['Flash'])

    def backwards(self, orm):
        # Deleting model 'Flash'
        db.delete_table(u'djangocms_flash_flash')

    models = {
        'cms.cmsplugin': {
            'Meta': {'object_name': 'CMSPlugin'},
            'changed_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
            'creation_date': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'language': ('django.db.models.fields.CharField', [], {'max_length': '15', 'db_index': 'True'}),
            'level': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
            'lft': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
            'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.CMSPlugin']", 'null': 'True', 'blank': 'True'}),
            'placeholder': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['cms.Placeholder']", 'null': 'True'}),
            'plugin_type': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
            'position': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True', 'blank': 'True'}),
            'rght': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'}),
            'tree_id': ('django.db.models.fields.PositiveIntegerField', [], {'db_index': 'True'})
        },
        'cms.placeholder': {
            'Meta': {'object_name': 'Placeholder'},
            'default_width': ('django.db.models.fields.PositiveSmallIntegerField', [], {'null': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'slot': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'})
        },
        u'djangocms_flash.flash': {
            'Meta': {'object_name': 'Flash', '_ormbases': ['cms.CMSPlugin']},
            u'cmsplugin_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['cms.CMSPlugin']", 'unique': 'True', 'primary_key': 'True'}),
            'file': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
            'height': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            'width': ('django.db.models.fields.CharField', [], {'max_length': '6'})
        }
    }

    complete_apps = ['djangocms_flash']


text:
package org.apache.pig.backend.hadoop.executionengine.tez.operators;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.tez.TezInput;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.library.api.KeyValueReader;

/**
 * POShuffledValueInputTez is used read tuples from a Tez Intermediate output from a shuffle edge
 * To be used with POValueOutputTez.
 * TODO: To be removed after PIG-3775 and TEZ-661
 */
public class POShuffledValueInputTez extends PhysicalOperator implements TezInput {

    private static final long serialVersionUID = 1L;
    private static final Log LOG = LogFactory.getLog(POShuffledValueInputTez.class);
    private Set<String> inputKeys = new HashSet<String>();
    private transient boolean finished = false;
    private transient Iterator<KeyValueReader> readers;
    private transient KeyValueReader currentReader;
    protected static final TupleFactory mTupleFactory = TupleFactory.getInstance();
    private transient Configuration conf;

    public POShuffledValueInputTez(OperatorKey k) {
        super(k);
    }

    @Override
    public String[] getTezInputs() {
        return inputKeys.toArray(new String[inputKeys.size()]);
    }

    @Override
    public void replaceInput(String oldInputKey, String newInputKey) {
        if (inputKeys.remove(oldInputKey)) {
            inputKeys.add(newInputKey);
        }
    }

    @Override
    public void addInputsToSkip(Set<String> inputsToSkip) {
    }

    @Override
    public void attachInputs(Map<String, LogicalInput> inputs,
            Configuration conf) throws ExecException {
        this.conf = conf;
        List<KeyValueReader> readersList = new ArrayList<KeyValueReader>();
        try {
            for (String inputKey : inputKeys) {
                LogicalInput input = inputs.get(inputKey);
                if (input == null) {
                    throw new ExecException("Input from vertex " + inputKey
                            + " is missing");
                }

                KeyValueReader reader = (KeyValueReader) input.getReader();
                readersList.add(reader);
                LOG.info("Attached input from vertex " + inputKey + " : input="
                        + input + ", reader=" + reader);
            }
            readers = readersList.iterator();
            currentReader = readers.next();
        } catch (Exception e) {
            throw new ExecException(e);
        }
    }

    @Override
    public Result getNextTuple() throws ExecException {
        try {
            if (finished) {
                return RESULT_EOP;
            }

            do {
                if (currentReader.next()) {
                    Tuple origTuple = (Tuple) currentReader.getCurrentValue();
                    Tuple copy = mTupleFactory.newTuple(origTuple.getAll());
                    return new Result(POStatus.STATUS_OK, copy);
                }
                currentReader = readers.hasNext() ? readers.next() : null;
            } while (currentReader != null);

            finished = true;
            // For certain operators (such as STREAM), we could still have some work
            // to do even after seeing the last input. These operators set a flag that
            // says all input has been sent and to run the pipeline one more time.
            if (Boolean.valueOf(conf.get(JobControlCompiler.END_OF_INP_IN_MAP, "false"))) {
                this.parentPlan.endOfAllInput = true;
            }
            return RESULT_EOP;
        } catch (IOException e) {
            throw new ExecException(e);
        }
    }

    public void addInputKey(String inputKey) {
        this.inputKeys.add(inputKey);
    }

    @Override
    public Tuple illustratorMarkup(Object in, Object out, int eqClassIndex) {
        // TODO Auto-generated method stub
        return null;
    }

    @Override
    public void visit(PhyPlanVisitor v) throws VisitorException {
        v.visit(this);
    }

    @Override
    public boolean supportsMultipleInputs() {
        return false;
    }

    @Override
    public boolean supportsMultipleOutputs() {
        return false;
    }

    @Override
    public String name() {
        return "POShuffledValueInputTez - " + mKey.toString() + "\t<-\t " + inputKeys;
    }
}


text:
package org.apache.hadoop.hive.ql.parse;

import java.io.IOException;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.sql.Date;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.LineageInfo;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;

/**
 * BaseSemanticAnalyzer.
 *
 */
public abstract class BaseSemanticAnalyzer {
  protected static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName());
  protected final Hive db;
  protected final HiveConf conf;
  protected List<Task<? extends Serializable>> rootTasks;
  protected FetchTask fetchTask;
  protected final Log LOG;
  protected final LogHelper console;

  protected Context ctx;
  protected HashMap<String, String> idToTableNameMap;
  protected QueryProperties queryProperties;

  /**
   * A set of FileSinkOperators being written to in an ACID compliant way.  We need to remember
   * them here because when we build them we don't yet know the transaction id.  We need to go
   * back and set it once we actually start running the query.
   */
  protected Set<FileSinkDesc> acidFileSinks = new HashSet<FileSinkDesc>();

  public static int HIVE_COLUMN_ORDER_ASC = 1;
  public static int HIVE_COLUMN_ORDER_DESC = 0;

  /**
   * ReadEntitites that are passed to the hooks.
   */
  protected HashSet<ReadEntity> inputs;
  /**
   * List of WriteEntities that are passed to the hooks.
   */
  protected HashSet<WriteEntity> outputs;
  /**
   * Lineage information for the query.
   */
  protected LineageInfo linfo;
  protected TableAccessInfo tableAccessInfo;
  protected ColumnAccessInfo columnAccessInfo;
  /**
   * Columns accessed by updates
   */
  protected ColumnAccessInfo updateColumnAccessInfo;

  public boolean skipAuthorization() {
    return false;
  }

  class RowFormatParams {
    String fieldDelim = null;
    String fieldEscape = null;
    String collItemDelim = null;
    String mapKeyDelim = null;
    String lineDelim = null;
    String nullFormat = null;

    protected void analyzeRowFormat(ASTNode child) throws SemanticException {
      child = (ASTNode) child.getChild(0);
      int numChildRowFormat = child.getChildCount();
      for (int numC = 0; numC < numChildRowFormat; numC++) {
        ASTNode rowChild = (ASTNode) child.getChild(numC);
        switch (rowChild.getToken().getType()) {
        case HiveParser.TOK_TABLEROWFORMATFIELD:
          fieldDelim = unescapeSQLString(rowChild.getChild(0)
              .getText());
          if (rowChild.getChildCount() >= 2) {
            fieldEscape = unescapeSQLString(rowChild
                .getChild(1).getText());
          }
          break;
        case HiveParser.TOK_TABLEROWFORMATCOLLITEMS:
          collItemDelim = unescapeSQLString(rowChild
              .getChild(0).getText());
          break;
        case HiveParser.TOK_TABLEROWFORMATMAPKEYS:
          mapKeyDelim = unescapeSQLString(rowChild.getChild(0)
              .getText());
          break;
        case HiveParser.TOK_TABLEROWFORMATLINES:
          lineDelim = unescapeSQLString(rowChild.getChild(0)
              .getText());
          if (!lineDelim.equals("\n")
              && !lineDelim.equals("10")) {
            throw new SemanticException(SemanticAnalyzer.generateErrorMessage(rowChild,
                ErrorMsg.LINES_TERMINATED_BY_NON_NEWLINE.getMsg()));
          }
          break;
        case HiveParser.TOK_TABLEROWFORMATNULL:
          nullFormat = unescapeSQLString(rowChild.getChild(0)
                    .getText());
          break;
        default:
          throw new AssertionError("Unkown Token: " + rowChild);
        }
      }
    }
  }

  public BaseSemanticAnalyzer(HiveConf conf) throws SemanticException {
   this(conf, createHiveDB(conf));
  }

  public BaseSemanticAnalyzer(HiveConf conf, Hive db) throws SemanticException {
    try {
      this.conf = conf;
      this.db = db;
      rootTasks = new ArrayList<Task<? extends Serializable>>();
      LOG = LogFactory.getLog(this.getClass().getName());
      console = new LogHelper(LOG);
      idToTableNameMap = new HashMap<String, String>();
      inputs = new LinkedHashSet<ReadEntity>();
      outputs = new LinkedHashSet<WriteEntity>();
    } catch (Exception e) {
      throw new SemanticException(e);
    }
  }

  protected static Hive createHiveDB(HiveConf conf) throws SemanticException {
    try {
      return Hive.get(conf);
    } catch (HiveException e) {
      throw new SemanticException(e);
    }
  }

  public HashMap<String, String> getIdToTableNameMap() {
    return idToTableNameMap;
  }

  public abstract void analyzeInternal(ASTNode ast) throws SemanticException;
  public void init(boolean clearPartsCache) {
    //no-op
  }

  public void initCtx(Context ctx) {
    this.ctx = ctx;
  }

  public void analyze(ASTNode ast, Context ctx) throws SemanticException {
    initCtx(ctx);
    init(true);
    analyzeInternal(ast);
  }

  public void validate() throws SemanticException {
    // Implementations may choose to override this
  }

  public List<Task<? extends Serializable>> getRootTasks() {
    return rootTasks;
  }

  /**
   * @return the fetchTask
   */
  public FetchTask getFetchTask() {
    return fetchTask;
  }

  /**
   * @param fetchTask
   *          the fetchTask to set
   */
  public void setFetchTask(FetchTask fetchTask) {
    this.fetchTask = fetchTask;
  }

  protected void reset(boolean clearPartsCache) {
    rootTasks = new ArrayList<Task<? extends Serializable>>();
  }

  public static String stripIdentifierQuotes(String val) {
    if ((val.charAt(0) == '`' && val.charAt(val.length() - 1) == '`')) {
      val = val.substring(1, val.length() - 1);
    }
    return val;
  }

  public static String stripQuotes(String val) {
    return PlanUtils.stripQuotes(val);
  }

  public static String charSetString(String charSetName, String charSetString)
      throws SemanticException {
    try {
      // The character set name starts with a _, so strip that
      charSetName = charSetName.substring(1);
      if (charSetString.charAt(0) == '\'') {
        return new String(unescapeSQLString(charSetString).getBytes(),
            charSetName);
      } else // hex input is also supported
      {
        assert charSetString.charAt(0) == '0';
        assert charSetString.charAt(1) == 'x';
        charSetString = charSetString.substring(2);

        byte[] bArray = new byte[charSetString.length() / 2];
        int j = 0;
        for (int i = 0; i < charSetString.length(); i += 2) {
          int val = Character.digit(charSetString.charAt(i), 16) * 16
              + Character.digit(charSetString.charAt(i + 1), 16);
          if (val > 127) {
            val = val - 256;
          }
          bArray[j++] = (byte)val;
        }

        String res = new String(bArray, charSetName);
        return res;
      }
    } catch (UnsupportedEncodingException e) {
      throw new SemanticException(e);
    }
  }

  /**
   * Get dequoted name from a table/column node.
   * @param tableOrColumnNode the table or column node
   * @return for table node, db.tab or tab. for column node column.
   */
  public static String getUnescapedName(ASTNode tableOrColumnNode) {
    return getUnescapedName(tableOrColumnNode, null);
  }

  public static String getUnescapedName(ASTNode tableOrColumnNode, String currentDatabase) {
    int tokenType = tableOrColumnNode.getToken().getType();
    if (tokenType == HiveParser.TOK_TABNAME) {
      // table node
      if (tableOrColumnNode.getChildCount() == 2) {
        String dbName = unescapeIdentifier(tableOrColumnNode.getChild(0).getText());
        String tableName = unescapeIdentifier(tableOrColumnNode.getChild(1).getText());
        return dbName + "." + tableName;
      }
      String tableName = unescapeIdentifier(tableOrColumnNode.getChild(0).getText());
      if (currentDatabase != null) {
        return currentDatabase + "." + tableName;
      }
      return tableName;
    } else if (tokenType == HiveParser.StringLiteral) {
      return unescapeSQLString(tableOrColumnNode.getText());
    }
    // column node
    return unescapeIdentifier(tableOrColumnNode.getText());
  }

  public static String[] getQualifiedTableName(ASTNode tabNameNode) throws SemanticException {
    if (tabNameNode.getType() != HiveParser.TOK_TABNAME ||
        (tabNameNode.getChildCount() != 1 && tabNameNode.getChildCount() != 2)) {
      throw new SemanticException(ErrorMsg.INVALID_TABLE_NAME.getMsg(tabNameNode));
    }
    if (tabNameNode.getChildCount() == 2) {
      String dbName = unescapeIdentifier(tabNameNode.getChild(0).getText());
      String tableName = unescapeIdentifier(tabNameNode.getChild(1).getText());
      return new String[] {dbName, tableName};
    }
    String tableName = unescapeIdentifier(tabNameNode.getChild(0).getText());
    return Utilities.getDbTableName(tableName);
  }

  public static String getDotName(String[] qname) throws SemanticException {
    String genericName = StringUtils.join(qname, ".");
    if (qname.length != 2) {
      throw new SemanticException(ErrorMsg.INVALID_TABLE_NAME, genericName);
    }
    return genericName;
  }

  /**
   * Get the unqualified name from a table node.
   *
   * This method works for table names qualified with their schema (e.g., "db.table")
   * and table names without schema qualification. In both cases, it returns
   * the table name without the schema.
   *
   * @param node the table node
   * @return the table name without schema qualification
   *         (i.e., if name is "db.table" or "table", returns "table")
   */
  public static String getUnescapedUnqualifiedTableName(ASTNode node) {
    assert node.getChildCount() <= 2;

    if (node.getChildCount() == 2) {
      node = (ASTNode) node.getChild(1);
    }

    return getUnescapedName(node);
  }

  /**
   * Remove the encapsulating "`" pair from the identifier. We allow users to
   * use "`" to escape identifier for table names, column names and aliases, in
   * case that coincide with Hive language keywords.
   */
  public static String unescapeIdentifier(String val) {
    if (val == null) {
      return null;
    }
    if (val.charAt(0) == '`' && val.charAt(val.length() - 1) == '`') {
      val = val.substring(1, val.length() - 1);
    }
    return val;
  }

  /**
   * Converts parsed key/value properties pairs into a map.
   *
   * @param prop ASTNode parent of the key/value pairs
   *
   * @param mapProp property map which receives the mappings
   */
  public static void readProps(
    ASTNode prop, Map<String, String> mapProp) {

    for (int propChild = 0; propChild < prop.getChildCount(); propChild++) {
      String key = unescapeSQLString(prop.getChild(propChild).getChild(0)
          .getText());
      String value = null;
      if (prop.getChild(propChild).getChild(1) != null) {
        value = unescapeSQLString(prop.getChild(propChild).getChild(1).getText());
      }
      mapProp.put(key, value);
    }
  }

  private static final int[] multiplier = new int[] {1000, 100, 10, 1};

  @SuppressWarnings("nls")
  public static String unescapeSQLString(String b) {
    Character enclosure = null;

    // Some of the strings can be passed in as unicode. For example, the
    // delimiter can be passed in as \002 - So, we first check if the
    // string is a unicode number, else go back to the old behavior
    StringBuilder sb = new StringBuilder(b.length());
    for (int i = 0; i < b.length(); i++) {

      char currentChar = b.charAt(i);
      if (enclosure == null) {
        if (currentChar == '\'' || b.charAt(i) == '\"') {
          enclosure = currentChar;
        }
        // ignore all other chars outside the enclosure
        continue;
      }

      if (enclosure.equals(currentChar)) {
        enclosure = null;
        continue;
      }

      if (currentChar == '\\' && (i + 6 < b.length()) && b.charAt(i + 1) == 'u') {
        int code = 0;
        int base = i + 2;
        for (int j = 0; j < 4; j++) {
          int digit = Character.digit(b.charAt(j + base), 16);
          code += digit * multiplier[j];
        }
        sb.append((char)code);
        i += 5;
        continue;
      }

      if (currentChar == '\\' && (i + 4 < b.length())) {
        char i1 = b.charAt(i + 1);
        char i2 = b.charAt(i + 2);
        char i3 = b.charAt(i + 3);
        if ((i1 >= '0' && i1 <= '1') && (i2 >= '0' && i2 <= '7')
            && (i3 >= '0' && i3 <= '7')) {
          byte bVal = (byte) ((i3 - '0') + ((i2 - '0') * 8) + ((i1 - '0') * 8 * 8));
          byte[] bValArr = new byte[1];
          bValArr[0] = bVal;
          String tmp = new String(bValArr);
          sb.append(tmp);
          i += 3;
          continue;
        }
      }

      if (currentChar == '\\' && (i + 2 < b.length())) {
        char n = b.charAt(i + 1);
        switch (n) {
        case '0':
          sb.append("\0");
          break;
        case '\'':
          sb.append("'");
          break;
        case '"':
          sb.append("\"");
          break;
        case 'b':
          sb.append("\b");
          break;
        case 'n':
          sb.append("\n");
          break;
        case 'r':
          sb.append("\r");
          break;
        case 't':
          sb.append("\t");
          break;
        case 'Z':
          sb.append("\u001A");
          break;
        case '\\':
          sb.append("\\");
          break;
        // The following 2 lines are exactly what MySQL does TODO: why do we do this?
        case '%':
          sb.append("\\%");
          break;
        case '_':
          sb.append("\\_");
          break;
        default:
          sb.append(n);
        }
        i++;
      } else {
        sb.append(currentChar);
      }
    }
    return sb.toString();
  }

  /**
   * Escapes the string for AST; doesn't enclose it in quotes, however.
   */
  public static String escapeSQLString(String b) {
    // There's usually nothing to escape so we will be optimistic.
    String result = b;
    for (int i = 0; i < result.length(); ++i) {
      char currentChar = result.charAt(i);
      if (currentChar == '\\' && ((i + 1) < result.length())) {
        // TODO: do we need to handle the "this is what MySQL does" here?
        char nextChar = result.charAt(i + 1);
        if (nextChar == '%' || nextChar == '_') {
          ++i;
          continue;
        }
      }
      switch (currentChar) {
      case '\0': result = spliceString(result, i, "\\0"); ++i; break;
      case '\'': result = spliceString(result, i, "\\'"); ++i; break;
      case '\"': result = spliceString(result, i, "\\\""); ++i; break;
      case '\b': result = spliceString(result, i, "\\b"); ++i; break;
      case '\n': result = spliceString(result, i, "\\n"); ++i; break;
      case '\r': result = spliceString(result, i, "\\r"); ++i; break;
      case '\t': result = spliceString(result, i, "\\t"); ++i; break;
      case '\\': result = spliceString(result, i, "\\\\"); ++i; break;
      case '\u001A': result = spliceString(result, i, "\\Z"); ++i; break;
      default: {
        if (currentChar < ' ') {
          String hex = Integer.toHexString(currentChar);
          String unicode = "\\u";
          for (int j = 4; j > hex.length(); --j) {
            unicode += '0';
          }
          unicode += hex;
          result = spliceString(result, i, unicode);
          i += (unicode.length() - 1);
        }
        break; // if not a control character, do nothing
      }
      }
    }
    return result;
  }

  private static String spliceString(String str, int i, String replacement) {
    return spliceString(str, i, 1, replacement);
  }

  private static String spliceString(String str, int i, int length, String replacement) {
    return str.substring(0, i) + replacement + str.substring(i + length);
  }

  public HashSet<ReadEntity> getInputs() {
    return inputs;
  }

  public HashSet<WriteEntity> getOutputs() {
    return outputs;
  }

  /**
   * @return the schema for the fields which will be produced
   * when the statement is executed, or null if not known
   */
  public List<FieldSchema> getResultSchema() {
    return null;
  }

  protected List<FieldSchema> getColumns(ASTNode ast) throws SemanticException {
    return getColumns(ast, true);
  }

  /**
   * Get the list of FieldSchema out of the ASTNode.
   */
  public static List<FieldSchema> getColumns(ASTNode ast, boolean lowerCase) throws SemanticException {
    List<FieldSchema> colList = new ArrayList<FieldSchema>();
    int numCh = ast.getChildCount();
    for (int i = 0; i < numCh; i++) {
      FieldSchema col = new FieldSchema();
      ASTNode child = (ASTNode) ast.getChild(i);
      Tree grandChild = child.getChild(0);
      if(grandChild != null) {
        String name = grandChild.getText();
        if(lowerCase) {
          name = name.toLowerCase();
        }
        // child 0 is the name of the column
        col.setName(unescapeIdentifier(name));
        // child 1 is the type of the column
        ASTNode typeChild = (ASTNode) (child.getChild(1));
        col.setType(getTypeStringFromAST(typeChild));

        // child 2 is the optional comment of the column
        if (child.getChildCount() == 3) {
          col.setComment(unescapeSQLString(child.getChild(2).getText()));
        }
      }
      colList.add(col);
    }
    return colList;
  }

  public static List<String> getColumnNames(ASTNode ast) {
    List<String> colList = new ArrayList<String>();
    int numCh = ast.getChildCount();
    for (int i = 0; i < numCh; i++) {
      ASTNode child = (ASTNode) ast.getChild(i);
      colList.add(unescapeIdentifier(child.getText()).toLowerCase());
    }
    return colList;
  }

  protected List<Order> getColumnNamesOrder(ASTNode ast) {
    List<Order> colList = new ArrayList<Order>();
    int numCh = ast.getChildCount();
    for (int i = 0; i < numCh; i++) {
      ASTNode child = (ASTNode) ast.getChild(i);
      if (child.getToken().getType() == HiveParser.TOK_TABSORTCOLNAMEASC) {
        colList.add(new Order(unescapeIdentifier(child.getChild(0).getText()).toLowerCase(),
            HIVE_COLUMN_ORDER_ASC));
      } else {
        colList.add(new Order(unescapeIdentifier(child.getChild(0).getText()).toLowerCase(),
            HIVE_COLUMN_ORDER_DESC));
      }
    }
    return colList;
  }

  protected static String getTypeStringFromAST(ASTNode typeNode)
      throws SemanticException {
    switch (typeNode.getType()) {
    case HiveParser.TOK_LIST:
      return serdeConstants.LIST_TYPE_NAME + "<"
          + getTypeStringFromAST((ASTNode) typeNode.getChild(0)) + ">";
    case HiveParser.TOK_MAP:
      return serdeConstants.MAP_TYPE_NAME + "<"
          + getTypeStringFromAST((ASTNode) typeNode.getChild(0)) + ","
          + getTypeStringFromAST((ASTNode) typeNode.getChild(1)) + ">";
    case HiveParser.TOK_STRUCT:
      return getStructTypeStringFromAST(typeNode);
    case HiveParser.TOK_UNIONTYPE:
      return getUnionTypeStringFromAST(typeNode);
    default:
      return DDLSemanticAnalyzer.getTypeName(typeNode);
    }
  }

  private static String getStructTypeStringFromAST(ASTNode typeNode)
      throws SemanticException {
    String typeStr = serdeConstants.STRUCT_TYPE_NAME + "<";
    typeNode = (ASTNode) typeNode.getChild(0);
    int children = typeNode.getChildCount();
    if (children <= 0) {
      throw new SemanticException("empty struct not allowed.");
    }
    StringBuilder buffer = new StringBuilder(typeStr);
    for (int i = 0; i < children; i++) {
      ASTNode child = (ASTNode) typeNode.getChild(i);
      buffer.append(unescapeIdentifier(child.getChild(0).getText())).append(":");
      buffer.append(getTypeStringFromAST((ASTNode) child.getChild(1)));
      if (i < children - 1) {
        buffer.append(",");
      }
    }

    buffer.append(">");
    return buffer.toString();
  }

  private static String getUnionTypeStringFromAST(ASTNode typeNode)
      throws SemanticException {
    String typeStr = serdeConstants.UNION_TYPE_NAME + "<";
    typeNode = (ASTNode) typeNode.getChild(0);
    int children = typeNode.getChildCount();
    if (children <= 0) {
      throw new SemanticException("empty union not allowed.");
    }
    StringBuilder buffer = new StringBuilder(typeStr);
    for (int i = 0; i < children; i++) {
      buffer.append(getTypeStringFromAST((ASTNode) typeNode.getChild(i)));
      if (i < children - 1) {
        buffer.append(",");
      }
    }
    buffer.append(">");
    typeStr = buffer.toString();
    return typeStr;
  }

  /**
   * tableSpec.
   *
   */
  public static class tableSpec {
    public String tableName;
    public Table tableHandle;
    public Map<String, String> partSpec; // has to use LinkedHashMap to enforce order
    public Partition partHandle;
    public int numDynParts; // number of dynamic partition columns
    public List<Partition> partitions; // involved partitions in TableScanOperator/FileSinkOperator
    public static enum SpecType {TABLE_ONLY, STATIC_PARTITION, DYNAMIC_PARTITION};
    public SpecType specType;

    public tableSpec(Hive db, HiveConf conf, ASTNode ast)
        throws SemanticException {
      this(db, conf, ast, true, false);
    }

    public tableSpec(Hive db, HiveConf conf, String tableName, Map<String, String> partSpec)
        throws HiveException {
      this.tableName = tableName;
      this.partSpec = partSpec;
      this.tableHandle = db.getTable(tableName);
      if (partSpec != null) {
        this.specType = SpecType.STATIC_PARTITION;
        this.partHandle = db.getPartition(tableHandle, partSpec, false);
        this.partitions = Arrays.asList(partHandle);
      } else {
        this.specType = SpecType.TABLE_ONLY;
      }
    }

    public tableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartitionsSpec,
        boolean allowPartialPartitionsSpec) throws SemanticException {
      assert (ast.getToken().getType() == HiveParser.TOK_TAB
          || ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION
          || ast.getToken().getType() == HiveParser.TOK_TABTYPE
          || ast.getToken().getType() == HiveParser.TOK_CREATETABLE);
      int childIndex = 0;
      numDynParts = 0;

      try {
        // get table metadata
        tableName = getUnescapedName((ASTNode)ast.getChild(0));
        boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE);
        if (testMode) {
          tableName = conf.getVar(HiveConf.ConfVars.HIVETESTMODEPREFIX)
              + tableName;
        }
        if (ast.getToken().getType() != HiveParser.TOK_CREATETABLE) {
          tableHandle = db.getTable(tableName);
        }
      } catch (InvalidTableException ite) {
        throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(ast
            .getChild(0)), ite);
      } catch (HiveException e) {
        throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg(ast
            .getChild(childIndex), e.getMessage()), e);
      }

      // get partition metadata if partition specified
      if (ast.getChildCount() == 2 && ast.getToken().getType() != HiveParser.TOK_CREATETABLE) {
        childIndex = 1;
        ASTNode partspec = (ASTNode) ast.getChild(1);
        partitions = new ArrayList<Partition>();
        // partSpec is a mapping from partition column name to its value.
        Map<String, String> tmpPartSpec = new HashMap<String, String>(partspec.getChildCount());
        for (int i = 0; i < partspec.getChildCount(); ++i) {
          ASTNode partspec_val = (ASTNode) partspec.getChild(i);
          String val = null;
          String colName = unescapeIdentifier(partspec_val.getChild(0).getText().toLowerCase());
          if (partspec_val.getChildCount() < 2) { // DP in the form of T partition (ds, hr)
            if (allowDynamicPartitionsSpec) {
              ++numDynParts;
            } else {
              throw new SemanticException(ErrorMsg.INVALID_PARTITION
                                                       .getMsg(" - Dynamic partitions not allowed"));
            }
          } else { // in the form of T partition (ds="2010-03-03")
            val = stripQuotes(partspec_val.getChild(1).getText());
          }
          tmpPartSpec.put(colName, val);
        }

        // check if the columns, as well as value types in the partition() clause are valid
        validatePartSpec(tableHandle, tmpPartSpec, ast, conf, false);

        List<FieldSchema> parts = tableHandle.getPartitionKeys();
        partSpec = new LinkedHashMap<String, String>(partspec.getChildCount());
        for (FieldSchema fs : parts) {
          String partKey = fs.getName();
          partSpec.put(partKey, tmpPartSpec.get(partKey));
        }

        // check if the partition spec is valid
        if (numDynParts > 0) {
          int numStaPart = parts.size() - numDynParts;
          if (numStaPart == 0 &&
              conf.getVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equalsIgnoreCase("strict")) {
            throw new SemanticException(ErrorMsg.DYNAMIC_PARTITION_STRICT_MODE.getMsg());
          }

          // check the partitions in partSpec be the same as defined in table schema
          if (partSpec.keySet().size() != parts.size()) {
            ErrorPartSpec(partSpec, parts);
          }
          Iterator<String> itrPsKeys = partSpec.keySet().iterator();
          for (FieldSchema fs: parts) {
            if (!itrPsKeys.next().toLowerCase().equals(fs.getName().toLowerCase())) {
              ErrorPartSpec(partSpec, parts);
            }
          }

          // check if static partition appear after dynamic partitions
          for (FieldSchema fs: parts) {
            if (partSpec.get(fs.getName().toLowerCase()) == null) {
              if (numStaPart > 0) { // found a DP, but there exists ST as subpartition
                throw new SemanticException(
                    ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg(ast.getChild(childIndex)));
              }
              break;
            } else {
              --numStaPart;
            }
          }
          partHandle = null;
          specType = SpecType.DYNAMIC_PARTITION;
        } else {
          try {
            if (allowPartialPartitionsSpec) {
              partitions = db.getPartitions(tableHandle, partSpec);
            } else {
              // this doesn't create partition.
              partHandle = db.getPartition(tableHandle, partSpec, false);
              if (partHandle == null) {
                // if partSpec doesn't exists in DB, return a delegate one
                // and the actual partition is created in MoveTask
                partHandle = new Partition(tableHandle, partSpec, null);
              } else {
                partitions.add(partHandle);
              }
            }
          } catch (HiveException e) {
            throw new SemanticException(
                ErrorMsg.INVALID_PARTITION.getMsg(ast.getChild(childIndex)), e);
          }
          specType = SpecType.STATIC_PARTITION;
        }
      } else {
        specType = SpecType.TABLE_ONLY;
      }
    }

    public Map<String, String> getPartSpec() {
      return this.partSpec;
    }

    public void setPartSpec(Map<String, String> partSpec) {
      this.partSpec = partSpec;
    }

    @Override
    public String toString() {
      if (partHandle != null) {
        return partHandle.toString();
      } else {
        return tableHandle.toString();
      }
    }
  }

  /**
   * Gets the lineage information.
   *
   * @return LineageInfo associated with the query.
   */
  public LineageInfo getLineageInfo() {
    return linfo;
  }

  /**
   * Sets the lineage information.
   *
   * @param linfo The LineageInfo structure that is set in the optimization phase.
   */
  public void setLineageInfo(LineageInfo linfo) {
    this.linfo = linfo;
  }

  /**
   * Gets the table access information.
   *
   * @return TableAccessInfo associated with the query.
   */
  public TableAccessInfo getTableAccessInfo() {
    return tableAccessInfo;
  }

  /**
   * Sets the table access information.
   *
   * @param tableAccessInfo The TableAccessInfo structure that is set in the optimization phase.
   */
  public void setTableAccessInfo(TableAccessInfo tableAccessInfo) {
    this.tableAccessInfo = tableAccessInfo;
  }

  /**
   * Gets the column access information.
   *
   * @return ColumnAccessInfo associated with the query.
   */
  public ColumnAccessInfo getColumnAccessInfo() {
    return columnAccessInfo;
  }

  /**
   * Sets the column access information.
   *
   * @param columnAccessInfo The ColumnAccessInfo structure that is set immediately after
   * the optimization phase.
   */
  public void setColumnAccessInfo(ColumnAccessInfo columnAccessInfo) {
    this.columnAccessInfo = columnAccessInfo;
  }

  public ColumnAccessInfo getUpdateColumnAccessInfo() {
    return updateColumnAccessInfo;
  }

  public void setUpdateColumnAccessInfo(ColumnAccessInfo updateColumnAccessInfo) {
    this.updateColumnAccessInfo = updateColumnAccessInfo;
  }

  protected LinkedHashMap<String, String> extractPartitionSpecs(Tree partspec)
      throws SemanticException {
    LinkedHashMap<String, String> partSpec = new LinkedHashMap<String, String>();
    for (int i = 0; i < partspec.getChildCount(); ++i) {
      CommonTree partspec_val = (CommonTree) partspec.getChild(i);
      String val = stripQuotes(partspec_val.getChild(1).getText());
      partSpec.put(partspec_val.getChild(0).getText().toLowerCase(), val);
    }
    return partSpec;
  }

  /**
   * Checks if given specification is proper specification for prefix of
   * partition cols, for table partitioned by ds, hr, min valid ones are
   * (ds='2008-04-08'), (ds='2008-04-08', hr='12'), (ds='2008-04-08', hr='12', min='30')
   * invalid one is for example (ds='2008-04-08', min='30')
   * @param spec specification key-value map
   * @return true if the specification is prefix; never returns false, but throws
   * @throws HiveException
   */
  public final boolean isValidPrefixSpec(Table tTable, Map<String, String> spec)
      throws HiveException {

    // TODO - types need to be checked.
    List<FieldSchema> partCols = tTable.getPartitionKeys();
    if (partCols == null || (partCols.size() == 0)) {
      if (spec != null) {
        throw new HiveException(
            "table is not partitioned but partition spec exists: "
                + spec);
      } else {
        return true;
      }
    }

    if (spec == null) {
      throw new HiveException("partition spec is not specified");
    }

    Iterator<String> itrPsKeys = spec.keySet().iterator();
    for (FieldSchema fs: partCols) {
      if(!itrPsKeys.hasNext()) {
        break;
      }
      if (!itrPsKeys.next().toLowerCase().equals(
              fs.getName().toLowerCase())) {
        ErrorPartSpec(spec, partCols);
      }
    }

    if(itrPsKeys.hasNext()) {
      ErrorPartSpec(spec, partCols);
    }

    return true;
  }

  private static void ErrorPartSpec(Map<String, String> partSpec,
      List<FieldSchema> parts) throws SemanticException {
    StringBuilder sb =
        new StringBuilder(
            "Partition columns in the table schema are: (");
    for (FieldSchema fs : parts) {
      sb.append(fs.getName()).append(", ");
    }
    sb.setLength(sb.length() - 2); // remove the last ", "
    sb.append("), while the partitions specified in the query are: (");

    Iterator<String> itrPsKeys = partSpec.keySet().iterator();
    while (itrPsKeys.hasNext()) {
      sb.append(itrPsKeys.next()).append(", ");
    }
    sb.setLength(sb.length() - 2); // remove the last ", "
    sb.append(").");
    throw new SemanticException(ErrorMsg.PARTSPEC_DIFFER_FROM_SCHEMA
        .getMsg(sb.toString()));
  }

  public Hive getDb() {
    return db;
  }

  public QueryProperties getQueryProperties() {
    return queryProperties;
  }

  public Set<FileSinkDesc> getAcidFileSinks() {
    return acidFileSinks;
  }

  /**
   * Construct list bucketing context.
   *
   * @param skewedColNames
   * @param skewedValues
   * @param skewedColValueLocationMaps
   * @param isStoredAsSubDirectories
   * @return
   */
  protected ListBucketingCtx constructListBucketingCtx(List<String> skewedColNames,
      List<List<String>> skewedValues, Map<List<String>, String> skewedColValueLocationMaps,
      boolean isStoredAsSubDirectories, HiveConf conf) {
    ListBucketingCtx lbCtx = new ListBucketingCtx();
    lbCtx.setSkewedColNames(skewedColNames);
    lbCtx.setSkewedColValues(skewedValues);
    lbCtx.setLbLocationMap(skewedColValueLocationMaps);
    lbCtx.setStoredAsSubDirectories(isStoredAsSubDirectories);
    lbCtx.setDefaultKey(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY);
    lbCtx.setDefaultDirName(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME);
    return lbCtx;
  }

  /**
   * Given a ASTNode, return list of values.
   *
   * use case:
   *   create table xyz list bucketed (col1) with skew (1,2,5)
   *   AST Node is for (1,2,5)
   * @param ast
   * @return
   */
  protected List<String> getSkewedValueFromASTNode(ASTNode ast) {
    List<String> colList = new ArrayList<String>();
    int numCh = ast.getChildCount();
    for (int i = 0; i < numCh; i++) {
      ASTNode child = (ASTNode) ast.getChild(i);
      colList.add(stripQuotes(child.getText()).toLowerCase());
    }
    return colList;
  }

  /**
   * Retrieve skewed values from ASTNode.
   *
   * @param node
   * @return
   * @throws SemanticException
   */
  protected List<String> getSkewedValuesFromASTNode(Node node) throws SemanticException {
    List<String> result = null;
    Tree leafVNode = ((ASTNode) node).getChild(0);
    if (leafVNode == null) {
      throw new SemanticException(
          ErrorMsg.SKEWED_TABLE_NO_COLUMN_VALUE.getMsg());
    } else {
      ASTNode lVAstNode = (ASTNode) leafVNode;
      if (lVAstNode.getToken().getType() != HiveParser.TOK_TABCOLVALUE) {
        throw new SemanticException(
            ErrorMsg.SKEWED_TABLE_NO_COLUMN_VALUE.getMsg());
      } else {
        result = new ArrayList<String>(getSkewedValueFromASTNode(lVAstNode));
      }
    }
    return result;
  }

  /**
   * Analyze list bucket column names
   *
   * @param skewedColNames
   * @param child
   * @return
   * @throws SemanticException
   */
  protected List<String> analyzeSkewedTablDDLColNames(List<String> skewedColNames, ASTNode child)
      throws SemanticException {
  Tree nNode = child.getChild(0);
    if (nNode == null) {
      throw new SemanticException(ErrorMsg.SKEWED_TABLE_NO_COLUMN_NAME.getMsg());
    } else {
      ASTNode nAstNode = (ASTNode) nNode;
      if (nAstNode.getToken().getType() != HiveParser.TOK_TABCOLNAME) {
        throw new SemanticException(ErrorMsg.SKEWED_TABLE_NO_COLUMN_NAME.getMsg());
      } else {
        skewedColNames = getColumnNames(nAstNode);
      }
    }
    return skewedColNames;
  }

  /**
   * Handle skewed values in DDL.
   *
   * It can be used by both skewed by ... on () and set skewed location ().
   *
   * @param skewedValues
   * @param child
   * @throws SemanticException
   */
  protected void analyzeDDLSkewedValues(List<List<String>> skewedValues, ASTNode child)
      throws SemanticException {
  Tree vNode = child.getChild(1);
    if (vNode == null) {
      throw new SemanticException(ErrorMsg.SKEWED_TABLE_NO_COLUMN_VALUE.getMsg());
    }
    ASTNode vAstNode = (ASTNode) vNode;
    switch (vAstNode.getToken().getType()) {
      case HiveParser.TOK_TABCOLVALUE:
        for (String str : getSkewedValueFromASTNode(vAstNode)) {
          List<String> sList = new ArrayList<String>(Arrays.asList(str));
          skewedValues.add(sList);
        }
        break;
      case HiveParser.TOK_TABCOLVALUE_PAIR:
        ArrayList<Node> vLNodes = vAstNode.getChildren();
        for (Node node : vLNodes) {
          if ( ((ASTNode) node).getToken().getType() != HiveParser.TOK_TABCOLVALUES) {
            throw new SemanticException(
                ErrorMsg.SKEWED_TABLE_NO_COLUMN_VALUE.getMsg());
          } else {
            skewedValues.add(getSkewedValuesFromASTNode(node));
          }
        }
        break;
      default:
        break;
    }
  }

  /**
   * process stored as directories
   *
   * @param child
   * @return
   */
  protected boolean analyzeStoredAdDirs(ASTNode child) {
    boolean storedAsDirs = false;
    if ((child.getChildCount() == 3)
        && (((ASTNode) child.getChild(2)).getToken().getType()
            == HiveParser.TOK_STOREDASDIRS)) {
      storedAsDirs = true;
    }
    return storedAsDirs;
  }

  private static boolean getPartExprNodeDesc(ASTNode astNode,
      Map<ASTNode, ExprNodeDesc> astExprNodeMap) throws SemanticException {

    if (astNode == null) {
      return true;
    } else if ((astNode.getChildren() == null) || (astNode.getChildren().size() == 0)) {
      return astNode.getType() != HiveParser.TOK_PARTVAL;
    }

    TypeCheckCtx typeCheckCtx = new TypeCheckCtx(null);
    boolean result = true;
    for (Node childNode : astNode.getChildren()) {
      ASTNode childASTNode = (ASTNode)childNode;

      if (childASTNode.getType() != HiveParser.TOK_PARTVAL) {
        result = getPartExprNodeDesc(childASTNode, astExprNodeMap) && result;
      } else {
        boolean isDynamicPart = childASTNode.getChildren().size() <= 1;
        result = !isDynamicPart && result;
        if (!isDynamicPart) {
          ASTNode partVal = (ASTNode)childASTNode.getChildren().get(1);
          astExprNodeMap.put((ASTNode)childASTNode.getChildren().get(0),
            TypeCheckProcFactory.genExprNode(partVal, typeCheckCtx).get(partVal));
        }
      }
    }
    return result;
  }

  public static void validatePartSpec(Table tbl, Map<String, String> partSpec,
      ASTNode astNode, HiveConf conf, boolean shouldBeFull) throws SemanticException {
    tbl.validatePartColumnNames(partSpec, shouldBeFull);

    if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TYPE_CHECK_ON_INSERT)) {
      return;
    }

    Map<ASTNode, ExprNodeDesc> astExprNodeMap = new HashMap<ASTNode, ExprNodeDesc>();
    if (!getPartExprNodeDesc(astNode, astExprNodeMap)) {
      STATIC_LOG.warn("Dynamic partitioning is used; only validating "
          + astExprNodeMap.size() + " columns");
    }

    if (astExprNodeMap.isEmpty()) {
      return; // All columns are dynamic, nothing to do.
    }

    List<FieldSchema> parts = tbl.getPartitionKeys();
    Map<String, String> partCols = new HashMap<String, String>(parts.size());
    for (FieldSchema col : parts) {
      partCols.put(col.getName(), col.getType().toLowerCase());
    }
    for (Entry<ASTNode, ExprNodeDesc> astExprNodePair : astExprNodeMap.entrySet()) {
      String astKeyName = astExprNodePair.getKey().toString().toLowerCase();
      if (astExprNodePair.getKey().getType() == HiveParser.Identifier) {
        astKeyName = stripIdentifierQuotes(astKeyName);
      }
      String colType = partCols.get(astKeyName);
      ObjectInspector inputOI = astExprNodePair.getValue().getWritableObjectInspector();

      TypeInfo expectedType =
          TypeInfoUtils.getTypeInfoFromTypeString(colType);
      ObjectInspector outputOI =
          TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType);
      Object value = null;
      String colSpec = partSpec.get(astKeyName);
      try {
        value =
            ExprNodeEvaluatorFactory.get(astExprNodePair.getValue()).
            evaluate(colSpec);
      } catch (HiveException e) {
        throw new SemanticException(e);
      }
      Object convertedValue =
        ObjectInspectorConverters.getConverter(inputOI, outputOI).convert(value);
      if (convertedValue == null) {
        throw new SemanticException(ErrorMsg.PARTITION_SPEC_TYPE_MISMATCH, astKeyName,
            inputOI.getTypeName(), outputOI.getTypeName());
      }

    }
  }

  protected WriteEntity toWriteEntity(String location) throws SemanticException {
    return toWriteEntity(new Path(location));
  }

  protected WriteEntity toWriteEntity(Path location) throws SemanticException {
    try {
      Path path = tryQualifyPath(location);
      return new WriteEntity(path, FileUtils.isLocalFile(conf, path.toUri()));
    } catch (Exception e) {
      throw new SemanticException(e);
    }
  }

  protected ReadEntity toReadEntity(String location) throws SemanticException {
    return toReadEntity(new Path(location));
  }

  protected ReadEntity toReadEntity(Path location) throws SemanticException {
    try {
      Path path = tryQualifyPath(location);
      return new ReadEntity(path, FileUtils.isLocalFile(conf, path.toUri()));
    } catch (Exception e) {
      throw new SemanticException(e);
    }
  }

  private Path tryQualifyPath(Path path) throws IOException {
    try {
      return path.getFileSystem(conf).makeQualified(path);
    } catch (IOException e) {
      return path;  // some tests expected to pass invalid schema
    }
  }

  protected Database getDatabase(String dbName) throws SemanticException {
    return getDatabase(dbName, true);
  }

  protected Database getDatabase(String dbName, boolean throwException) throws SemanticException {
    Database database;
    try {
      database = db.getDatabase(dbName);
    } catch (Exception e) {
      throw new SemanticException(e.getMessage(), e);
    }
    if (database == null && throwException) {
      throw new SemanticException(ErrorMsg.DATABASE_NOT_EXISTS.getMsg(dbName));
    }
    return database;
  }

  protected Table getTable(String[] qualified) throws SemanticException {
    return getTable(qualified[0], qualified[1], true);
  }

  protected Table getTable(String[] qualified, boolean throwException) throws SemanticException {
    return getTable(qualified[0], qualified[1], throwException);
  }

  protected Table getTable(String tblName) throws SemanticException {
    return getTable(null, tblName, true);
  }

  protected Table getTable(String tblName, boolean throwException) throws SemanticException {
    return getTable(null, tblName, throwException);
  }

  protected Table getTable(String database, String tblName, boolean throwException)
      throws SemanticException {
    Table tab;
    try {
      tab = database == null ? db.getTable(tblName, false)
          : db.getTable(database, tblName, false);
    }
    catch (InvalidTableException e) {
      throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tblName), e);
    }
    catch (Exception e) {
      throw new SemanticException(e.getMessage(), e);
    }
    if (tab == null && throwException) {
      throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tblName));
    }
    return tab;
  }

  protected Partition getPartition(Table table, Map<String, String> partSpec,
      boolean throwException) throws SemanticException {
    Partition partition;
    try {
      partition = db.getPartition(table, partSpec, false);
    } catch (Exception e) {
      throw new SemanticException(toMessage(ErrorMsg.INVALID_PARTITION, partSpec), e);
    }
    if (partition == null && throwException) {
      throw new SemanticException(toMessage(ErrorMsg.INVALID_PARTITION, partSpec));
    }
    return partition;
  }

  protected List<Partition> getPartitions(Table table, Map<String, String> partSpec,
      boolean throwException) throws SemanticException {
    List<Partition> partitions;
    try {
      partitions = partSpec == null ? db.getPartitions(table) :
          db.getPartitions(table, partSpec);
    } catch (Exception e) {
      throw new SemanticException(toMessage(ErrorMsg.INVALID_PARTITION, partSpec), e);
    }
    if (partitions.isEmpty() && throwException) {
      throw new SemanticException(toMessage(ErrorMsg.INVALID_PARTITION, partSpec));
    }
    return partitions;
  }

  protected String toMessage(ErrorMsg message, Object detail) {
    return detail == null ? message.getMsg() : message.getMsg(detail.toString());
  }
}