text:
package org.apache.phoenix.end2end;

import static org.apache.phoenix.mapreduce.PhoenixJobCounters.INPUT_RECORDS;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_VALID_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT;
import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.hadoop.mapreduce.Job;
import org.apache.phoenix.compile.ExplainPlan;
import org.apache.phoenix.compile.ExplainPlanAttributes;
import org.apache.phoenix.end2end.index.IndexTestUtil;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository;
import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository;
import org.apache.phoenix.mapreduce.index.PhoenixIndexImportDirectMapper;
import org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters;
import org.apache.phoenix.mapreduce.index.PhoenixServerBuildIndexMapper;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.transaction.PhoenixTransactionProvider.Feature;
import org.apache.phoenix.transaction.TransactionFactory;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.BeforeParam;
import org.junit.runners.Parameterized.Parameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@Category(NeedsOwnMiniClusterTest.class)
@RunWith(Parameterized.class)
public class IndexToolIT extends BaseTest {
    private static final Logger LOGGER = LoggerFactory.getLogger(IndexToolIT.class);

    private static String tmpDir;

    private final boolean localIndex;
    private final boolean mutable;
    private final boolean transactional;
    private final String tableDDLOptions;
    private final String indexDDLOptions;
    private final boolean useSnapshot;
    private final boolean useTenantId;
    private final boolean namespaceMapped;

    public IndexToolIT(String transactionProvider, boolean mutable, boolean localIndex,
            boolean useSnapshot, boolean useTenantId, boolean namespaceMapped) {
        this.localIndex = localIndex;
        this.mutable = mutable;
        this.transactional = transactionProvider != null;
        this.useSnapshot = useSnapshot;
        this.useTenantId = useTenantId;
        this.namespaceMapped = namespaceMapped;
        StringBuilder optionBuilder = new StringBuilder();
        if (!mutable) {
            optionBuilder.append(" IMMUTABLE_ROWS=true ");
        }
        if (transactional) {
            if (!(optionBuilder.length() == 0)) {
                optionBuilder.append(",");
            }
            optionBuilder.append(" TRANSACTIONAL=true,TRANSACTION_PROVIDER='" + transactionProvider + "'");
        }
        optionBuilder.append(" SPLIT ON(1,2)");
        this.tableDDLOptions = optionBuilder.toString();
        StringBuilder indexOptionBuilder = new StringBuilder();
        if (!localIndex && transactionProvider == null) {
            if (!(optionBuilder.length() == 0)) {
                optionBuilder.append(",");
            }
            optionBuilder.append(" COLUMN_ENCODED_BYTES=0");
            indexOptionBuilder.append(" IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS,COLUMN_ENCODED_BYTES=2");
        }
        this.indexDDLOptions = indexOptionBuilder.toString();
    }

    @BeforeParam
    public static synchronized void setup(
            String transactionProvider, boolean mutable, boolean localIndex,
            boolean useSnapshot, boolean useTenantId, boolean namespaceMapped)
                    throws Exception {
        if (clusterInitialized && Boolean.valueOf(namespaceMapped).equals(utility.getConfiguration()
                .getBoolean(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, true))) {
            //Perf optimization: no need to re-initialize the minicluster
            return;
        } else {
            //This has been known to cause flakeyness
            //Maybe we're gonna have to refactor the namespace
            if(clusterInitialized) {
                tearDownMiniCluster(0);
                System.setProperty("java.io.tmpdir", tmpDir);
            } else {
                tmpDir = System.getProperty("java.io.tmpdir");
            }
            Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
            serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
            serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5));
            serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
                QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
            serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8));
            serverProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
            serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
                Boolean.toString(namespaceMapped));
            Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
            clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true));
            clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
            clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
            clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString());
            clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
                Boolean.toString(namespaceMapped));
            setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),
                new ReadOnlyProps(clientProps.entrySet().iterator()));
        }
    }

    @Parameters(
            name = "transactionProvider={0},mutable={1},localIndex={2}," +
                "useSnapshot={3},useTenant={4},namespaceMapped={5}")
    public static synchronized Collection<Object[]> data() {
        List<Object[]> list = Lists.newArrayListWithExpectedSize(48);
        boolean[] Booleans = new boolean[] { false, true };
        for (boolean namespaceMapped : Booleans) {
            for (boolean useSnapshot : Booleans) {
                for (String transactionProvider : new String[] {"TEPHRA", "OMID", null}) {
                    if(transactionProvider !=null &&
                            !TransactionFactory.Provider.valueOf(transactionProvider).runTests()) {
                        continue;
                    }
                    for (boolean mutable : Booleans) {
                        for (boolean localIndex : Booleans) {
                            if(localIndex && useSnapshot) {
                                continue;
                            }
                            if (!localIndex
                                    || transactionProvider == null
                                    || !TransactionFactory.getTransactionProvider(
                                            TransactionFactory.Provider.valueOf(transactionProvider))
                                        .isUnsupported(Feature.ALLOW_LOCAL_INDEX)) {
                                list.add(new Object[]{transactionProvider, mutable, localIndex,
                                        useSnapshot, false, namespaceMapped});
                            }
                        }
                    }
                }
            }
        }
        // Add the usetenantId
        list.add(new Object[] { null, false, true, false, true, false});
        return TestUtil.filterTxParamData(list,0);
    }

    protected static void setEveryNthRowWithNull(int nrows, int nthRowNull, PreparedStatement stmt) throws Exception {
        for (int i = 1; i <= nrows; i++) {
            stmt.setInt(1, i);
            stmt.setInt(2, i + 1);
            if (i % nthRowNull != 0) {
                stmt.setInt(3, i * i);
            } else {
                stmt.setNull(3, Types.INTEGER);
            }
            stmt.execute();
        }
    }

    @Test
    public void testSecondaryIndex() throws Exception {
        String schemaName = generateUniqueName();
        String dataTableName = generateUniqueName();
        String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
        String indexTableName = generateUniqueName();
        String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName);
        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
        PhoenixConnection conn = (PhoenixConnection) DriverManager.getConnection(getUrl(), props);
        try {
            if(namespaceMapped) {
                conn.createStatement().execute("CREATE SCHEMA " + schemaName);
            }
            String stmString1 =
                    "CREATE TABLE " + dataTableFullName
                            + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
                            + tableDDLOptions;
            conn.createStatement().execute(stmString1);
            String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
            PreparedStatement upsertStmt = conn.prepareStatement(upsertQuery);

            // insert two rows
            upsertRow(upsertStmt, 1);
            upsertRow(upsertStmt, 2);
            conn.commit();

            if (transactional) {
                // insert two rows in another connection without committing so that they are not
                // visible to other transactions
                try (Connection conn2 = DriverManager.getConnection(getUrl(), props)) {
                    conn2.setAutoCommit(false);
                    PreparedStatement stmt2 = conn2.prepareStatement(upsertQuery);
                    upsertRow(stmt2, 5);
                    upsertRow(stmt2, 6);
                    ResultSet rs =
                            conn.createStatement()
                                    .executeQuery("SELECT count(*) from " + dataTableFullName);
                    assertTrue(rs.next());
                    assertEquals("Unexpected row count ", 2, rs.getInt(1));
                    assertFalse(rs.next());
                    rs =
                            conn2.createStatement()
                                    .executeQuery("SELECT count(*) from " + dataTableFullName);
                    assertTrue(rs.next());
                    assertEquals("Unexpected row count ", 4, rs.getInt(1));
                    assertFalse(rs.next());
                }
            }

            String stmtString2 =
                    String.format(
                        "CREATE %s INDEX %s ON %s  (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ASYNC " + this.indexDDLOptions,
                        (localIndex ? "LOCAL" : ""), indexTableName, dataTableFullName);
            conn.createStatement().execute(stmtString2);

            // verify rows are fetched from data table.
            String selectSql =
                    String.format(
                        "SELECT ID FROM %s WHERE LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz' = 'xxUNAME2_xyz'",
                        dataTableFullName);

            // assert we are pulling from data table.
            ExplainPlan plan = conn.prepareStatement(selectSql)
                .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
                .getExplainPlan();
            ExplainPlanAttributes explainPlanAttributes =
                plan.getPlanStepsAsAttributes();
            assertEquals("PARALLEL 1-WAY",
                explainPlanAttributes.getIteratorTypeAndScanSize());
            assertEquals("FULL SCAN ",
                explainPlanAttributes.getExplainScanType());
            assertEquals(dataTableFullName,
                explainPlanAttributes.getTableName().replaceAll(":", "."));
            assertEquals("SERVER FILTER BY (LPAD(UPPER(NAME, 'en_US'), 8, 'x') || '_xyz') = 'xxUNAME2_xyz'",
                explainPlanAttributes.getServerWhereFilter().replaceAll(":", "."));

            ResultSet rs = conn.createStatement().executeQuery(selectSql);
            assertTrue(rs.next());
            assertEquals(2, rs.getInt(1));
            assertFalse(rs.next());
            conn.commit();

            // run the index MR job.
            runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName);

            if (!localIndex) {
                IndexTestUtil.assertRowsForEmptyColValue(conn, indexTableFullName,
                    QueryConstants.VERIFIED_BYTES);
            }
            // insert two more rows
            upsertRow(upsertStmt, 3);
            upsertRow(upsertStmt, 4);
            conn.commit();

            if (!localIndex) {
                IndexTestUtil.assertRowsForEmptyColValue(conn, indexTableFullName,
                    QueryConstants.VERIFIED_BYTES);
            }
            // assert we are pulling from index table.
            plan = conn.prepareStatement(selectSql)
                .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
                .getExplainPlan();
            explainPlanAttributes = plan.getPlanStepsAsAttributes();
            assertEquals("RANGE SCAN ",
                explainPlanAttributes.getExplainScanType());
            final String expectedTableName;
            if (localIndex) {
                expectedTableName = dataTableFullName;
            } else {
                expectedTableName = indexTableFullName;
            }
            assertEquals(expectedTableName,
                explainPlanAttributes.getTableName().replaceAll(":", "."));

            rs = conn.createStatement().executeQuery(selectSql);
            assertTrue(rs.next());
            assertEquals(2, rs.getInt(1));
            assertFalse(rs.next());
            if (localIndex || transactional || useTenantId || useSnapshot) {
                return;
            }
            // Run the index MR job and verify that the global index table is built correctly
            IndexTool indexTool = runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.BEFORE, new String[0]);
            assertEquals(4, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue());
            assertEquals(4, indexTool.getJob().getCounters().findCounter(SCANNED_DATA_ROW_COUNT).getValue());
            assertEquals(0, indexTool.getJob().getCounters().findCounter(REBUILT_INDEX_ROW_COUNT).getValue());
            assertEquals(4, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_VALID_INDEX_ROW_COUNT).getValue());
            assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_EXPIRED_INDEX_ROW_COUNT).getValue());
            assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT).getValue());
            assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue());
            assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue());
            assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue());
            assertEquals(0, indexTool.getJob().getCounters().findCounter(
                    BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_EXTRA_CELLS).getValue());
            assertEquals(0, indexTool.getJob().getCounters().findCounter(
                    BEFORE_REBUILD_INVALID_INDEX_ROW_COUNT_COZ_MISSING_CELLS).getValue());
        } finally {
            conn.close();
        }
    }

    protected static void dropIndexToolTables(Connection conn) throws Exception {
        Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
        TableName indexToolOutputTable =
            TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES);
        admin.disableTable(indexToolOutputTable);
        admin.deleteTable(indexToolOutputTable);
        TableName indexToolResultTable = TableName.valueOf(IndexVerificationResultRepository.RESULT_TABLE_NAME_BYTES);
        admin.disableTable(indexToolResultTable);
        admin.deleteTable(indexToolResultTable);
    }

    public static class MutationCountingRegionObserver extends SimpleRegionObserver {
        public static AtomicInteger mutationCount = new AtomicInteger(0);

        public static void setMutationCount(int value) {
            mutationCount.set(0);
        }

        public static int getMutationCount() {
            return mutationCount.get();
        }

        @Override
        public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
                                   MiniBatchOperationInProgress<Mutation> miniBatchOp) throws HBaseIOException {
            mutationCount.addAndGet(miniBatchOp.size());
        }
    }

    private static void verifyIndexTableRowKey(byte[] rowKey, String indexTableFullName) {
        // The row key for the output table : timestamp | index table name | data row key
        // The row key for the result table : timestamp | index table name | datable table region name |
        //                                    scan start row | scan stop row

        // This method verifies the common prefix, i.e., "timestamp | index table name | ", since the rest of the
        // fields may include the separator key
        int offset = Bytes.indexOf(rowKey, IndexVerificationResultRepository.ROW_KEY_SEPARATOR_BYTE);
        offset++;
        byte[] indexTableFullNameBytes = Bytes.toBytes(indexTableFullName);
        assertEquals(Bytes.compareTo(rowKey, offset, indexTableFullNameBytes.length, indexTableFullNameBytes, 0,
                indexTableFullNameBytes.length), 0);
        assertEquals(rowKey[offset + indexTableFullNameBytes.length],
            IndexVerificationResultRepository.ROW_KEY_SEPARATOR_BYTE[0]);
    }

    public static Cell getErrorMessageFromIndexToolOutputTable(Connection conn, String dataTableFullName, String indexTableFullName)
            throws Exception {
        byte[] indexTableFullNameBytes = Bytes.toBytes(indexTableFullName);
        byte[] dataTableFullNameBytes = Bytes.toBytes(dataTableFullName);
        Table hIndexTable = conn.unwrap(PhoenixConnection.class).getQueryServices()
                .getTable(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES);
        Scan scan = new Scan();
        ResultScanner scanner = hIndexTable.getScanner(scan);
        boolean dataTableNameCheck = false;
        boolean indexTableNameCheck = false;
        Cell errorMessageCell = null;
        for (Result result = scanner.next(); result != null; result = scanner.next()) {
            for (Cell cell : result.rawCells()) {
                assertTrue(Bytes.compareTo(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
                    IndexVerificationOutputRepository.OUTPUT_TABLE_COLUMN_FAMILY, 0,
                    IndexVerificationOutputRepository.OUTPUT_TABLE_COLUMN_FAMILY.length) == 0);
                if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
                    IndexVerificationOutputRepository.DATA_TABLE_NAME_BYTES, 0, IndexVerificationOutputRepository.DATA_TABLE_NAME_BYTES.length) == 0) {
                    dataTableNameCheck = true;
                    assertTrue("Value was different! Expected: " + Bytes.toString(dataTableFullNameBytes)
                            + " Actual: " + Bytes.toString(cell.getValueArray(),
                        cell.getValueOffset(), cell.getValueLength()),
                        Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(),
                            dataTableFullNameBytes, 0, dataTableFullNameBytes.length) == 0);
                } else if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
                    IndexVerificationOutputRepository.INDEX_TABLE_NAME_BYTES, 0, IndexVerificationOutputRepository.INDEX_TABLE_NAME_BYTES.length) == 0) {
                    indexTableNameCheck = true;
                    assertTrue(Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(),
                            indexTableFullNameBytes, 0, indexTableFullNameBytes.length) == 0);
                } else if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
                    IndexVerificationOutputRepository.ERROR_MESSAGE_BYTES, 0, IndexVerificationOutputRepository.ERROR_MESSAGE_BYTES.length) == 0) {
                    errorMessageCell = cell;
                }
            }
        }
        assertTrue( "DataTableNameCheck was false", dataTableNameCheck);
        assertTrue("IndexTableNameCheck was false", indexTableNameCheck);
        assertTrue("Error message cell was null", errorMessageCell != null);
        verifyIndexTableRowKey(CellUtil.cloneRow(errorMessageCell), indexTableFullName);
        hIndexTable = conn.unwrap(PhoenixConnection.class).getQueryServices()
                .getTable(IndexVerificationResultRepository.RESULT_TABLE_NAME_BYTES);
        scan = new Scan();
        scanner = hIndexTable.getScanner(scan);
        Result result = scanner.next();
        if (result != null) {
            verifyIndexTableRowKey(CellUtil.cloneRow(result.rawCells()[0]), indexTableFullName);
        }

        return errorMessageCell;
    }

    @Test
    public void testIndexToolWithTenantId() throws Exception {
        if (!useTenantId) { return;}
        String tenantId = generateUniqueName();
        String schemaName = generateUniqueName();
        String dataTableName = generateUniqueName();
        String viewTenantName = generateUniqueName();
        String indexNameGlobal = generateUniqueName();
        String indexNameTenant = generateUniqueName();
        String viewIndexTableName = "_IDX_" + dataTableName;

        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
        Connection connGlobal = DriverManager.getConnection(getUrl(), props);
        props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
        Connection connTenant = DriverManager.getConnection(getUrl(), props);
        String createTblStr = "CREATE TABLE %s (TENANT_ID VARCHAR(15) NOT NULL,ID INTEGER NOT NULL"
                + ", NAME VARCHAR, CONSTRAINT PK_1 PRIMARY KEY (TENANT_ID, ID)) MULTI_TENANT=true";
        String createViewStr = "CREATE VIEW %s AS SELECT * FROM %s";

        String upsertQueryStr = "UPSERT INTO %s (TENANT_ID, ID, NAME) VALUES('%s' , %d, '%s')";
        String createIndexStr = "CREATE INDEX %s ON %s (NAME) " + this.indexDDLOptions;

        try {
            String tableStmtGlobal = String.format(createTblStr, dataTableName);
            connGlobal.createStatement().execute(tableStmtGlobal);

            String viewStmtTenant = String.format(createViewStr, viewTenantName, dataTableName);
            connTenant.createStatement().execute(viewStmtTenant);

            String idxStmtTenant = String.format(createIndexStr, indexNameTenant, viewTenantName);
            connTenant.createStatement().execute(idxStmtTenant);

            connTenant.createStatement()
                    .execute(String.format(upsertQueryStr, viewTenantName, tenantId, 1, "x"));
            connTenant.commit();

            runIndexTool(false, "", viewTenantName, indexNameTenant,
                    tenantId, 0, new String[0]);

            String selectSql = String.format("SELECT ID FROM %s WHERE NAME='x'", viewTenantName);
            ExplainPlan plan = connTenant.prepareStatement(selectSql)
                .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
                .getExplainPlan();
            ExplainPlanAttributes explainPlanAttributes =
                plan.getPlanStepsAsAttributes();
            assertEquals("PARALLEL 1-WAY",
                explainPlanAttributes.getIteratorTypeAndScanSize());
            assertEquals("RANGE SCAN ",
                explainPlanAttributes.getExplainScanType());
            assertEquals(viewIndexTableName,
                explainPlanAttributes.getTableName());

            ResultSet rs = connTenant.createStatement().executeQuery(selectSql);
            assertTrue(rs.next());
            assertEquals(1, rs.getInt(1));
            assertFalse(rs.next());

            // Remove from tenant view index and build.
            ConnectionQueryServices queryServices = connGlobal.unwrap(PhoenixConnection.class).getQueryServices();
            Admin admin = queryServices.getAdmin();
            TableName tableName = TableName.valueOf(viewIndexTableName);
            admin.disableTable(tableName);
            admin.truncateTable(tableName, false);

            runIndexTool(false, "", viewTenantName, indexNameTenant,
                    tenantId, 0, new String[0]);
            Table htable= queryServices.getTable(Bytes.toBytes(viewIndexTableName));
            int count = getUtility().countRows(htable);
            // Confirm index has rows
            assertTrue(count == 1);

            selectSql = String.format("SELECT /*+ INDEX(%s) */ COUNT(*) FROM %s",
                    indexNameTenant, viewTenantName);
            rs = connTenant.createStatement().executeQuery(selectSql);
            assertTrue(rs.next());
            assertEquals(1, rs.getInt(1));
            assertFalse(rs.next());

            String idxStmtGlobal =
                    String.format(createIndexStr, indexNameGlobal, dataTableName);
            connGlobal.createStatement().execute(idxStmtGlobal);

            // run the index MR job this time with tenant id.
            // We expect it to return -1 because indexTable is not correct for this tenant.
            runIndexTool(false, schemaName, dataTableName, indexNameGlobal,
                    tenantId, -1, new String[0]);

        } finally {
            connGlobal.close();
            connTenant.close();
        }
    }

    @Test
    public void testSaltedVariableLengthPK() throws Exception {
        if (!mutable) return;
        if (transactional) return;
        String schemaName = generateUniqueName();
        String dataTableName = generateUniqueName();
        String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
        String indexTableName = generateUniqueName();
        try (Connection conn =
                DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
            if(namespaceMapped) {
                conn.createStatement().execute("CREATE SCHEMA " + schemaName);
            }
            String dataDDL =
                    "CREATE TABLE " + dataTableFullName + "(\n"
                            + "ID VARCHAR NOT NULL PRIMARY KEY,\n"
                            + "\"info\".CAR_NUM VARCHAR(18) NULL,\n"
                            + "\"info\".CAP_DATE VARCHAR NULL,\n" + "\"info\".ORG_ID BIGINT NULL,\n"
                            + "\"info\".ORG_NAME VARCHAR(255) NULL\n" + ") SALT_BUCKETS=3";
            conn.createStatement().execute(dataDDL);

            String upsert =
                    "UPSERT INTO " + dataTableFullName
                            + "(ID,CAR_NUM,CAP_DATE,ORG_ID,ORG_NAME) VALUES('1','car1','2016-01-01 00:00:00',11,'orgname1')";
            conn.createStatement().execute(upsert);
            conn.commit();

            String indexDDL =
                    String.format(
                        "CREATE %s INDEX %s on %s (\"info\".CAR_NUM,\"info\".CAP_DATE) ASYNC " + this.indexDDLOptions,
                        (localIndex ? "LOCAL" : ""), indexTableName, dataTableFullName);
            conn.createStatement().execute(indexDDL);

            runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName);

            ResultSet rs =
                    conn.createStatement().executeQuery(
                        "SELECT ORG_ID,CAP_DATE,CAR_NUM,ORG_NAME FROM " + dataTableFullName
                                + " WHERE CAR_NUM='car1' AND CAP_DATE>='2016-01-01' AND CAP_DATE<='2016-05-02' LIMIT 10");
            assertTrue(rs.next());
            int orgId = rs.getInt(1);
            assertEquals(11, orgId);
        }
    }

    /**
     * Test presplitting an index table
     */
    @Test
    public void testSplitIndex() throws Exception {
        if (localIndex) return; // can't split local indexes
        if (!mutable) return;
        if (transactional) return;
        String schemaName = generateUniqueName();
        String dataTableName = generateUniqueName();
        String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
        String indexTableName = generateUniqueName();
        try (Connection conn =
                DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
                Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
            if(namespaceMapped) {
                conn.createStatement().execute("CREATE SCHEMA " + schemaName);
            }
            String dataDDL =
                    "CREATE TABLE " + dataTableFullName + "(\n"
                            + "ID VARCHAR NOT NULL PRIMARY KEY,\n"
                            + "\"info\".CAR_NUM VARCHAR(18) NULL,\n"
                            + "\"test\".CAR_NUM VARCHAR(18) NULL,\n"
                            + "\"info\".CAP_DATE VARCHAR NULL,\n" + "\"info\".ORG_ID BIGINT NULL,\n"
                            + "\"info\".ORG_NAME VARCHAR(255) NULL\n" + ") COLUMN_ENCODED_BYTES = 0";
            conn.createStatement().execute(dataDDL);

            String[] carNumPrefixes = new String[] {"a", "b", "c", "d"};

            // split the data table, as the tool splits the index table to have the same # of regions
            // doesn't really matter what the split points are, we just want a target # of regions
            int numSplits = carNumPrefixes.length;
            int targetNumRegions = numSplits + 1;
            byte[][] splitPoints = new byte[numSplits][];
            for (String prefix : carNumPrefixes) {
                splitPoints[--numSplits] = Bytes.toBytes(prefix);
            }
            TableName hDataName = TableName.valueOf(
                    SchemaUtil.getPhysicalHBaseTableName(schemaName, dataTableName, namespaceMapped)
                    .getBytes());
            HTableDescriptor dataTD = admin.getTableDescriptor(hDataName);
            admin.disableTable(hDataName);
            admin.deleteTable(hDataName);
            admin.createTable(dataTD, splitPoints);
            assertEquals(targetNumRegions, admin.getTableRegions(hDataName).size());

            // insert data where index column values start with a, b, c, d
            int idCounter = 1;
            try (PreparedStatement ps = conn.prepareStatement("UPSERT INTO " + dataTableFullName
                + "(ID,\"info\".CAR_NUM,\"test\".CAR_NUM,CAP_DATE,ORG_ID,ORG_NAME) VALUES(?,?,?,'2016-01-01 00:00:00',11,'orgname1')")){
                for (String carNum : carNumPrefixes) {
                    for (int i = 0; i < 100; i++) {
                        ps.setString(1, idCounter++ + "");
                        ps.setString(2, carNum + "_" + i);
                        ps.setString(3, "test-" + carNum + "_ " + i);
                        ps.addBatch();
                    }
                }
                ps.executeBatch();
                conn.commit();
            }

            String indexDDL =
                    String.format(
                        "CREATE INDEX %s on %s (\"info\".CAR_NUM,\"test\".CAR_NUM,\"info\".CAP_DATE) ASYNC " + this.indexDDLOptions,
                        indexTableName, dataTableFullName);
            conn.createStatement().execute(indexDDL);

            // run with 50% sampling rate, split if data table more than 3 regions
            runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName, "-sp", "50", "-spa", "3");

            TableName hIndexName = TableName.valueOf(
                SchemaUtil.getPhysicalHBaseTableName(schemaName, indexTableName, namespaceMapped)
                .getBytes());
            assertEquals(targetNumRegions, admin.getTableRegions(hIndexName).size());
            List<Cell> values = new ArrayList<>();
            // every index region should have been written to, if the index table was properly split uniformly
            for (HRegion region : getUtility().getHBaseCluster().getRegions(hIndexName)) {
                values.clear();
                RegionScanner scanner = region.getScanner(new Scan());
                scanner.next(values);
                if (values.isEmpty()) {
                    fail("Region did not have any results: " + region.getRegionInfo());
                }
            }
        }
    }

    @Test
    public void testCaseSensitiveNames() throws Exception {
        // We have three references to the "same" table name. e.g. for `my_table`
        // The original name, unquoted: `my_table`
        // The quoted name to prevent Phoenix from auto-capitalizing it (s): `"my_table"`
        // The qualified named with schema (q): `"my_schema":"my_table"`
        String schemaName = generateUniqueName().toLowerCase();
        String dataTableName = generateUniqueName().toLowerCase();
        String sSchemaName = SchemaUtil.getCaseSensitiveColumnDisplayName(null, schemaName);
        String sDataTableName = SchemaUtil.getCaseSensitiveColumnDisplayName(null, dataTableName);
        String qDataTableName = SchemaUtil.getCaseSensitiveColumnDisplayName(schemaName, dataTableName);
        // The explain plan doesn't re-quote the qualified table name: `my_schema.my_table`
        String dataTableNameForExplain = SchemaUtil.getTableName(schemaName, dataTableName);

        String indexTableName = generateUniqueName().toLowerCase();
        String sIndexTableName = SchemaUtil.getCaseSensitiveColumnDisplayName(null, indexTableName);
        String indexTableNameForExplain = SchemaUtil.getTableName(schemaName, indexTableName);

        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
        Connection conn = DriverManager.getConnection(getUrl(), props);
        try {
            if(namespaceMapped) {
                conn.createStatement().execute("CREATE SCHEMA " + sSchemaName);
            }
            String stmString1 =
                    "CREATE TABLE " + qDataTableName
                         + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) "
                         + tableDDLOptions;
            conn.createStatement().execute(stmString1);
            String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", qDataTableName);
            PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);

            // insert two rows
            upsertRow(stmt1, 1);
            upsertRow(stmt1, 2);
            conn.commit();

            if (transactional) {
                // insert two rows in another connection without committing so that they are not
                // visible to other transactions
                try (Connection conn2 = DriverManager.getConnection(getUrl(), props)) {
                    conn2.setAutoCommit(false);
                    PreparedStatement stmt2 = conn2.prepareStatement(upsertQuery);
                    upsertRow(stmt2, 5);
                    upsertRow(stmt2, 6);
                    ResultSet rs =
                            conn.createStatement()
                                    .executeQuery("SELECT count(*) from " + qDataTableName);
                    assertTrue(rs.next());
                    assertEquals("Unexpected row count ", 2, rs.getInt(1));
                    assertFalse(rs.next());
                    rs =
                            conn2.createStatement()
                                    .executeQuery("SELECT count(*) from " + qDataTableName);
                    assertTrue(rs.next());
                    assertEquals("Unexpected row count ", 4, rs.getInt(1));
                    assertFalse(rs.next());
                }
            }

            String stmtString2 =
                    String.format(
                        "CREATE %s INDEX %s ON %s  (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ASYNC ",
                        (localIndex ? "LOCAL" : ""), sIndexTableName, qDataTableName);
            conn.createStatement().execute(stmtString2);

            // verify rows are fetched from data table.
            String selectSql =
                    String.format(
                        "SELECT ID FROM %s WHERE LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz' = 'xxUNAME2_xyz'",
                        qDataTableName);

            ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql);
            String actualExplainPlan = QueryUtil.getExplainPlan(rs);

            // assert we are pulling from data table.
            assertEquals(String.format(
                "CLIENT PARALLEL 1-WAY FULL SCAN OVER %s\n"
                     + "    SERVER FILTER BY (LPAD(UPPER(NAME, 'en_US'), 8, 'x') || '_xyz') = 'xxUNAME2_xyz'",
                        dataTableNameForExplain), actualExplainPlan.replaceAll(":", "."));

            rs = stmt1.executeQuery(selectSql);
            assertTrue(rs.next());
            assertEquals(2, rs.getInt(1));
            assertFalse(rs.next());
            conn.commit();

            // run the index MR job.
            runIndexTool(useSnapshot, sSchemaName, sDataTableName, sIndexTableName);

            // insert two more rows
            upsertRow(stmt1, 3);
            upsertRow(stmt1, 4);
            conn.commit();

            // assert we are pulling from index table.
            rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql);
            actualExplainPlan = QueryUtil.getExplainPlan(rs);
            // Because the explain plan doesn't include double-quotes around case-sensitive table names,
            // we need to tell assertExplainPlan to not normalize our table names.
            assertExplainPlan(localIndex, actualExplainPlan, dataTableNameForExplain, indexTableNameForExplain, false);

            rs = conn.createStatement().executeQuery(selectSql);
            assertTrue(rs.next());
            assertEquals(2, rs.getInt(1));
            assertFalse(rs.next());
        } finally {
            conn.close();
        }
    }

    public static void assertExplainPlan(boolean localIndex, String actualExplainPlan,
            String dataTableFullName, String indexTableFullName) {
        assertExplainPlan(localIndex, actualExplainPlan, dataTableFullName, indexTableFullName, true);
    }

    public static void assertExplainPlan(boolean localIndex, String actualExplainPlan,
            String dataTableFullName, String indexTableFullName, boolean normalizeTableNames) {
        String expectedExplainPlan;
        if (localIndex) {
            expectedExplainPlan = String.format(" RANGE SCAN OVER %s [1,",
                normalizeTableNames ? SchemaUtil.normalizeIdentifier(dataTableFullName)
                        : dataTableFullName);
        } else {
            expectedExplainPlan = String.format(" RANGE SCAN OVER %s",
                normalizeTableNames ? SchemaUtil.normalizeIdentifier(indexTableFullName)
                        : indexTableFullName);
        }
        assertTrue(actualExplainPlan + "\n expected to contain \n" + expectedExplainPlan,
            actualExplainPlan.replaceAll(":", ".").contains(expectedExplainPlan));
    }

    public static CounterGroup getMRJobCounters(IndexTool indexTool) throws IOException {
        return indexTool.getJob().getCounters().getGroup(PhoenixIndexToolJobCounters.class.getName());
    }

    private static List<String> getArgList (boolean useSnapshot, String schemaName,
                                            String dataTable, String indxTable, String tenantId,
                                            IndexTool.IndexVerifyType verifyType, Long startTime,
                                            Long endTime, Long incrementalVerify) {
        return getArgList(useSnapshot, schemaName, dataTable, indxTable, tenantId,
            verifyType, startTime, endTime, IndexTool.IndexDisableLoggingType.NONE, incrementalVerify, false);
    }

    private static List<String> getArgList (boolean useSnapshot, String schemaName,
                            String dataTable, String indxTable, String tenantId,
                            IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime,
                                            IndexTool.IndexDisableLoggingType disableLoggingType,
                                            Long incrementalVerify, boolean useIndexTableAsSource) {
        List<String> args = Lists.newArrayList();
        if (schemaName != null) {
            args.add("--schema=" + schemaName);
        }
        // Work around CLI-254. The long-form arg parsing doesn't strip off double-quotes
        args.add("--data-table=" + dataTable);
        args.add("--index-table=" + indxTable);
        args.add("-v");
        args.add(verifyType.getValue()); // verify index rows inline

        // Need to run this job in foreground for the test to be deterministic
        args.add("-runfg");
        if (useSnapshot) {
            args.add("-snap");
        }

        if (tenantId != null) {
            args.add("-tenant");
            args.add(tenantId);
        }
        if(startTime != null) {
            args.add("-st");
            args.add(String.valueOf(startTime));
        }
        if(endTime != null) {
            args.add("-et");
            args.add(String.valueOf(endTime));
        }

        if (verifyType != IndexTool.IndexVerifyType.NONE && disableLoggingType != null) {
            args.add("-dl");
            args.add(disableLoggingType.getValue());
        }

        if(incrementalVerify!=null) {
            args.add("-rv");
            args.add(String.valueOf(incrementalVerify));
        }

        if (useIndexTableAsSource) {
            args.add("-fi");
        }

        args.add("-op");
        args.add("/tmp/" + UUID.randomUUID().toString());
        return args;
    }

    public static String[] getArgValues( boolean useSnapshot, String schemaName,
            String dataTable, String indexTable, String tenantId, IndexTool.IndexVerifyType verifyType) {
        List<String> args = getArgList(useSnapshot, schemaName, dataTable, indexTable,
                tenantId, verifyType, null, null, null);
        return args.toArray(new String[0]);
    }

    public static String[] getArgValues(boolean useSnapshot, String schemaName,
                                        String dataTable, String indexTable, String tenantId, IndexTool.IndexVerifyType verifyType,
                                        IndexTool.IndexDisableLoggingType disableLoggingType) {
        List<String> args = getArgList(useSnapshot, schemaName, dataTable, indexTable,
                tenantId, verifyType, null, null, disableLoggingType, null, false);
        return args.toArray(new String[0]);
    }

    public static String [] getArgValues(boolean useSnapshot, String schemaName,
            String dataTable, String indexTable, String tenantId,
            IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime) {
        List<String> args = getArgList(useSnapshot, schemaName, dataTable, indexTable,
                tenantId, verifyType, startTime, endTime, null);
        return args.toArray(new String[0]);
    }

    public static String [] getArgValues(boolean useSnapshot, String schemaName,
                                         String dataTable, String indexTable, String tenantId,
                                         IndexTool.IndexVerifyType verifyType, Long startTime,
                                         Long endTime,
                                         IndexTool.IndexDisableLoggingType disableLoggingType,
                                         Long incrementalVerify ) {
        List<String> args = getArgList(useSnapshot, schemaName, dataTable, indexTable,
            tenantId, verifyType, startTime, endTime, disableLoggingType, incrementalVerify, false);
        return args.toArray(new String[0]);
    }

    public static String [] getArgValues(boolean directApi, boolean useSnapshot, String schemaName,
        String dataTable, String indexTable, String tenantId,
        IndexTool.IndexVerifyType verifyType, Long startTime,
        Long endTime,
        IndexTool.IndexDisableLoggingType disableLoggingType,
        Long incrementalVerify,
        boolean useIndexTableAsSource) {
        List<String> args = getArgList(useSnapshot, schemaName, dataTable, indexTable,
            tenantId, verifyType, startTime, endTime, disableLoggingType, incrementalVerify, useIndexTableAsSource);
        return args.toArray(new String[0]);
    }

    public static void upsertRow(PreparedStatement stmt, int i) throws SQLException {
        // insert row
        stmt.setInt(1, i);
        stmt.setString(2, "uname" + String.valueOf(i));
        stmt.setInt(3, 95050 + i);
        stmt.executeUpdate();
    }

    public static void runIndexTool(boolean useSnapshot, String schemaName,
            String dataTableName, String indexTableName) throws Exception {
        runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName, new String[0]);
    }

    private static void verifyMapper(Job job, boolean useSnapshot, String schemaName,
                                  String dataTableName, String indexTableName, String tenantId) throws Exception {
        Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
        if (tenantId != null) {
            props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
        }

        try (Connection conn =
                     DriverManager.getConnection(getUrl(), props)) {
            PTable indexTable = PhoenixRuntime.getTableNoCache(conn,
                   SchemaUtil.normalizeFullTableName(SchemaUtil.getTableName(schemaName, indexTableName)));
            PTable dataTable = PhoenixRuntime.getTableNoCache(conn, SchemaUtil.normalizeFullTableName(SchemaUtil.getTableName(schemaName, dataTableName)));
            boolean transactional = dataTable.isTransactional();
            boolean localIndex = PTable.IndexType.LOCAL.equals(indexTable.getIndexType());
            if ((localIndex || !transactional) && !useSnapshot) {
                assertEquals(job.getMapperClass(), PhoenixServerBuildIndexMapper.class);
            } else {
                assertEquals(job.getMapperClass(), PhoenixIndexImportDirectMapper.class);
            }
        }
    }

    public static void runIndexTool(boolean useSnapshot, String schemaName,
            String dataTableName, String indexTableName, String... additionalArgs) throws Exception {
        runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName, null, 0, additionalArgs);
    }

    public static IndexTool runIndexTool(boolean useSnapshot, String schemaName,
            String dataTableName, String indexTableName, String tenantId, int expectedStatus,
            String... additionalArgs) throws Exception {
        return runIndexTool(useSnapshot, schemaName, dataTableName, indexTableName, tenantId, expectedStatus,
                IndexTool.IndexVerifyType.NONE, additionalArgs);
    }

    public static IndexTool runIndexTool( boolean useSnapshot, String schemaName,
                                         String dataTableName, String indexTableName, String tenantId,
                                         int expectedStatus, IndexTool.IndexVerifyType verifyType,
                                         String... additionalArgs) throws Exception {
        Configuration conf = new Configuration(getUtility().getConfiguration());
        return runIndexTool(conf, useSnapshot, schemaName, dataTableName, indexTableName,
            tenantId, expectedStatus, verifyType, IndexTool.IndexDisableLoggingType.NONE, additionalArgs);
    }

    public static IndexTool runIndexTool(Configuration conf, boolean useSnapshot, String schemaName,
            String dataTableName, String indexTableName, String tenantId,
            int expectedStatus, IndexTool.IndexVerifyType verifyType, IndexTool.IndexDisableLoggingType disableLoggingType,
            String... additionalArgs) throws Exception {
        IndexTool indexingTool = new IndexTool();
        conf.set(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
        indexingTool.setConf(conf);
        final String[] cmdArgs = getArgValues(useSnapshot, schemaName, dataTableName,
            indexTableName, tenantId, verifyType, disableLoggingType);
        List<String> cmdArgList = new ArrayList<>(Arrays.asList(cmdArgs));
        cmdArgList.addAll(Arrays.asList(additionalArgs));
        LOGGER.info("Running IndexTool with {}", Arrays.toString(cmdArgList.toArray()), new Exception("Stack Trace"));
        int status = indexingTool.run(cmdArgList.toArray(new String[cmdArgList.size()]));

        if (expectedStatus == 0) {
            verifyMapper(indexingTool.getJob(), useSnapshot, schemaName, dataTableName, indexTableName, tenantId);
        }
        assertEquals(expectedStatus, status);
        return indexingTool;
    }
}

redpajama_set_name: RedPajamaGithub


text:
When to Grow Crown Vetch?
Crown vetch (Coronilla varia) is planted as a ground cover and grown on disturbed slopes to stop erosion. This vigorous, fast-growing perennial grows in 2-foot tall mounds. It is green from early spring through late fall and dies back in winter. When it blooms in summer, its white, purple and pink flowers attract bees and other pollinators. Crown vetch is hardy in U.S. Department of Agriculture plant hardiness zones 3 through 9 and can be invasive.
Fall or spring is the best time to plant crown vetch in frost-prone areas of its hardiness zones. In mild, frost-free areas, it can be planted in fall through winter and early spring. Crown vetch is planted in two ways, either by seeds sown directly in soil or by crown or root divisions taken from a mature plant and placed in soil. Start with a spacing of 2 feet between plants to create a dense ground cover. Once established, crown vetch can spread aggressively, choking out other plants in the area.
Good drainage is the key to growing crown vetch successfully. The plant grows well in nutritiously poor, rocky and dry soil conditions. In wet areas, however, its roots and crown can rot, killing the plant. Plant crown vetch in a site that receives full to partial sun for its best growth and flowering. Crown vetch is seen growing naturally most often in disturbed areas, on embankments and along roadways. Once established in a suitable location, it takes care of itself and does not require watering or fertilizing.
Seed Planting
Crown vetch seeds germinate best in soil that is between 68 and 77 degrees Fahrenheit. Pretreat the seeds in order to get them to germinate. Without their pretreatment, you could wait several years for their germination. Pretreat crown vetch seeds by placing them in water that is about 190 F for four hours. You also can treat the seeds prior to planting them by rubbing them between two pieces of sandpaper to damage each seed's coat. Plant the seeds directly in a prepared garden bed, and barely cover them with soil.
Division Planting
Propagate and plant crown vetch from sections of its roots simply by digging up an established plant and cutting 4-inch long sections of its roots. Once replanted in soil, the root sections send up shoots. Propagation from crown division is a simple process. Just dig up a plant, and divide its roots into sections. Replant each section in pots or back in the landscape at the same depth in the soil as the plant was originally so that the base of the stalks are at the soil line.
U.S. Department of Agriculture, Forest Service: Coronilla Varia
University of Missouri Extension: Crown Vetch
Ohio State University: Coronilla Varia
Eulalia Palomo has been a professional writer since 2009. Prior to taking up writing full time she has worked as a landscape artist and organic gardener. Palomo holds a Bachelor of Arts in liberal studies from Boston University. She travels widely and has spent over six years living abroad.
Palomo, Eulalia. "When to Grow Crown Vetch?" Home Guides | SF Gate, http://homeguides.sfgate.com/grow-crown-vetch-67937.html. Accessed 18 July 2019.
Palomo, Eulalia. (n.d.). When to Grow Crown Vetch? Home Guides | SF Gate. Retrieved from http://homeguides.sfgate.com/grow-crown-vetch-67937.html
Palomo, Eulalia. "When to Grow Crown Vetch?" accessed July 18, 2019. http://homeguides.sfgate.com/grow-crown-vetch-67937.html
When to Transplant Liriope Monkey Grass?
The Best Time of Year to Plant Yuccas
Information About Yarrow Plant Seeds
Can Hen and Chicks Plants Be Kept Inside?
The Difference Between Self-Pollination & Cross-Pollination
Trim a Boston Fern
When Does the Annual Sweet Pea Bloom?
Start Morning Glory Seeds Indoors
Transplant a Dwarf Ruellia
How Deep Do You Plant Bleeding Hearts?
The Propagation of Cyclamen

redpajama_set_name: RedPajamaCommonCrawl


text:
Laberinto (La Trampa) – Uruguayan Music
by Emilio Pérez Miguel on December 19, 2009
"Laberinto" Was La Trampa's Fifth Studio Album. It Was Issued In 2005 To Good Sales And (Mostly) Positive Reviews.
"Laberinto" [Labyrinth] constitutes the fifth album of original material released by Uruguayan rockers La Trampa. Alongside La Vela Puerca, No Te Va Gustar and Buitres the band possesses immense popularity in the country; and like the aforementioned bands, it has been making inroads in the Argentinean market.
This time around there is a strong presence of both Uruguayan and Argentinean folk music in the compositions (all penned by guitarist Garo Arakelian), which coupled with the traditional approach of the band results in a captivating listening experience. The entrance to this particular labyrinth is not that unusual, though: "Puente De Estrellas" [Bridge Of Stars], "Las Décimas" [The Ten-line Stanzas] and "El Poeta Dice La Verdad" [The Poet Tells The Truth] are classic La Trampa, competing with the best moments of preceding albums such as "Caída Libre" [Free Fall] (2002).
However, come the fourth selection, a "zamba" (an Argentinean rhythm, not to be confused with the Brazilian "samba") titled "Ronda De Lenguas" [Round Of Tongues] and the thread abruptly disappears, leaving the listener wondering how to proceed. The bands playfully leads him through a maze of gentle songs interspersed with harder-sounding offerings, until "Vagos Recuerdos" [Vague Memories] signalizes the way out. Along the way he is bound to find "Canciones Al Viento" [Songs To The Wind], which combines a measured introduction with an energetic conclusion, emphasizing the will to transcend of the lyrics ("Quiero una canción que vuele cuando ya no vuele yo/Que se haga viento en el aire cuando no respire yo/Quiero un viento que te cante cuando ya no cante yo") [I want a song that flies when I no longer fly/That becomes wind in the air when I no longer breathe/I want a wind that sings for you when I no longer sing] and the salient "Pensares" [Thoughts], bearing the most realized chorus of the whole album, alongside a pensive guitar solo that complements the lyrics perfectly.
Several songs are dominated by a wistful mood ("Araucaria", "Alta Mar" [High Sea]), which at times clashes with the defiant attitude La Trampa had exhibited in previous albums. But it is difficult to significantly fault the compositions – the lyrics in particular evince a maturity and dexterity that shows they truly have come of age.
Both "Las Décimas" and "El Poeta Dice La Verdad" (the first single to be taken from the album, and the only track whose lyrics were not composed by Arakelian; they are taken from a sonnet by the Spanish poet Federico García Lorca) received ample exposure in Uruguayan radio, effectively becoming the songs the public readily associate with this album. But those willing to immerse themselves in the labyrinth are going to relish the experience. Ultimately, "Laberinto" stands as another quality offering from a band that has set a standard of excellence, and that has become an indelible part of the Uruguayan scene.
The band's official website can be reached here.
Tagged as: Alejandro Spuntone, Alvaro Pintos, Carlos Ráfols, Diego Varela, el poeta dice la verdad, folclore, Folk, Garo Arakelian, hard rock, Irvin Carballo, la trampa, laberinto, las decimas, Music, musica, Rock, Uruguay, uruguayan
{ 1 trackback }
Días Lúcidos (Lapso) – Uruguayan Music | MusicKO
Previous post: Cinco Estrellas (Astroboy) – Uruguayan Music
Next post: The 15 Best Music Startups To Be Highlighted At MidemNet Lab


100% Polyester Therma-FIT double piqué, 7.4oz. Therma-FIT double knit double piqué for ultimate stretch, warmth, and mobility. Traditional ½ zip silhouette with mock neck to protect against the elements. Interior Nike Golf tape. Front pockets provide storage and hand warmth. Heat transfer twill swoosh on left sleeve. Brushed fleece interior provides warmth and coverage.


text:
Q: upload image to mysql in android studio
09-01 17:36:36.244 10323-10323/com.example.lar_pc.bottomnavdemo
  E/AndroidRuntime: FATAL EXCEPTION: main
  Process: com.example.lar_pc.bottomnavdemo, PID: 10323
  java.lang.NullPointerException: Attempt to invoke virtual method
  'java.lang.String java.lang.String.toString()' on a null object
  reference
  at
  com.example.lar_pc.bottomnavdemo.AlbumsFragment.onTaskCompleted(AlbumsFragment.java:103)
  at
  com.example.lar_pc.bottomnavdemo.MultiPartRequester$AsyncHttpRequest.onPostExecute(MultiPartRequester.java:119)
  at
  com.example.lar_pc.bottomnavdemo.MultiPartRequester$AsyncHttpRequest.onPostExecute(MultiPartRequester.java:52)
  at android.os.AsyncTask.finish(AsyncTask.java:674)
  at android.os.AsyncTask.-wrap1(AsyncTask.java)
  at
  android.os.AsyncTask$InternalHandler.handleMessage(AsyncTask.java:691)
  at android.os.Handler.dispatchMessage(Handler.java:111)
  at android.os.Looper.loop(Looper.java:207)
  at android.app.ActivityThread.main(ActivityThread.java:5728)
  at java.lang.reflect.Method.invoke(Native Method)
  at
  com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:789)
  at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:679)

redpajama_set_name: RedPajamaStackExchange


text:
Hundreds of Liberians legally in U.S. face deportation following Trump decision
Alan Gomez
The Trump administration is facing mounting pressure from lawmakers, human rights activists and civil rights lawyers to extend deportation protections for thousands of Liberians who may become undocumented immigrants within a week.
President Donald Trump announced in March 2018 that hundreds of Liberians who had been legally living and working in the U.S. for more than two decades would have their status revoked and would no longer be protected from deportation.
Through a presidential memorandum, the president gave them one year to settle their affairs, but now he's facing pressure to save that group before their status expires on Sunday.
Democrats in Congress have sent letters to Trump to extend the Deferred Enforced Departure program, or DED, for the Liberians, arguing that conditions in their home country remain dangerous and that they've become contributing members of their U.S. communities.
Hennepin County, Minnesota, home to Minneapolis, has the greatest number of Liberians — an estimated 8,285 — among all counties in the nation, according to the Census Bureau's American Community Survey covering 2013-2017. The International Institute of Minnesota, a nonprofit devoted to helping new immigrants settle in the state, estimated 35,000 Liberians lived in the state.
Another large community of Liberians is in Philadelphia and its suburban Pennsylvania counties — nearly 11,000, according to ACS estimates. Providence County, Rhode Island; Montgomery and Prince George's Counties, Maryland and Mercer County, New Jersey also have large communities of Liberian immigrants.
On Monday, attorneys general in 10 states — including Minnesota Attorney General Keith Ellison — jumped in, submitting a filing in federal court supporting a lawsuit that's trying to overturn Trump's decision.
The attorneys general wrote that Liberians have contributed to their economies, especially in the field of health care, and that forcing their deportation would hurt their U.S.-born children who are U.S. citizens at birth.
"Liberians protected by DED are hardworking colleagues and valued contributors to our communities," they wrote. Their children, "have already suffered because of the fear that their parents may be removed from the country."
Since taking office, the Trump administration has been systematically ending similar programs that have temporarily protected select groups of undocumented immigrants from deportation.
The administration ended the Deferred Action for Childhood Arrivals (DACA) program, which has protected from deportation more than 800,000 undocumented immigrants brought to the country as children.
But federal judges have blocked that decision, and the Justice Department is appealing.
The administration has also been phasing out the Temporary Status Program (TPS), which has protected more than 300,000 people from 10 countries that have been ravaged by violence and natural disasters.
A federal judge also has blocked that decision, and the Justice Department is appealing.
About 10,000 Liberians were first granted TPS status in March 1991 under former President George H.W. Bush when a civil war broke out in that country. Ever since, Liberians have been shuffled between the TPS and DED programs, but their protected status has endured, according to a report from the Congressional Research Service.
Fleeing the war: How an Iowa couple from Liberia nurtures refugee children
They have been extended by former Presidents Bill Clinton, George W. Bush and Barack Obama as the country grappled with a civil war, ongoing armed conflicts, and an outbreak of the Ebola virus. But last year, after consulting with the departments of State and Homeland Security, Trump concluded that those conditions had been largely resolved and that it was time for them to return home.
"Liberia is no longer experiencing armed conflict and has made significant progress in restoring stability and democratic governance," Trump wrote. "The 2014 outbreak of Ebola Virus Disease caused a tragic loss of life and economic damage to the country, but Liberia has made tremendous progress in its ability to diagnose and contain future outbreaks of the disease."
Many have disputed that conclusion, arguing that conditions in Liberia remain just as dire. They believe that forcing hundreds of Liberian citizens to return now would be hurtful not only to the returning Liberians, but the country as a whole as it continues recovering. That's why on Friday, 11 Democratic members of Congress, led by House Majority Leader Rep. Steny Hoyer, D-Md., sent a letter to Trump asking him to reconsider his decision.
"Given the dangers of deadly disease outbreak and violence in Liberia and its neighboring countries, it would be irresponsible to force these individuals and their families to return at this time," the group wrote. "Liberians legally residing in the United States, if sent back, could overburden the country's limited infrastructure and reserve the advances that the Liberian people and government have made."
The Liberians are part of the American Dream and Promise Act introduced by House Democrats, which would extend protections for people in all three programs: DACA, TPS and DED. The bill faces a difficult road in the Republican-led Senate, meaning the only remaining hope for Liberians in the program are a federal court ruling extending their protections, or an unexpected change of heart from Trump.
ICE sets record for arrests of undocumented immigrants with no criminal record Texas flight attendant detained by ICE for weeks after airline's 'administrative error' Supreme Court to decide if states can prosecute undocumented immigrants for identity theft Why once-supportive Senate Republicans may sink House Dreamer bill Commerce Secretary Wilbur Ross defends citizenship question on 2020 census


Share the burden. Your organization is ready to help, and you want to inspire others! The Homeless & Hungry Logo Poster features the stark image of an individual in need of help, and allows you to share an important message. If you have sponsors or partners sharing in that message, you can feature them on the template in one of eleven spaces for custom messages.
TicketRiver.com.au helps you help others.


text:
Chikhaldara (o Chhikaldara) è una città dell'India di 4.718 abitanti, situata nel distretto di Amravati, nello stato federato del Maharashtra. In base al numero di abitanti la città rientra nella classe VI (meno di 5.000 persone).

Geografia fisica
La città è situata a 21° 25' 04 N e 77° 17' 44 E.

Società

Evoluzione demografica
Al censimento del 2001 la popolazione di Chikhaldara assommava a 4.718 persone, delle quali 2.717 maschi e 2.001 femmine. I bambini di età inferiore o uguale ai sei anni assommavano a 588, dei quali 292 maschi e 296 femmine. Infine, coloro che erano in grado di saper almeno leggere e scrivere erano 3.783, dei quali 2.338 maschi e 1.445 femmine.

Note

Centri abitati del Maharashtra

redpajama_set_name: RedPajamaWikipedia


text:
Is WikiLeaks founder Julian Assange a hero or a villain? | The Tylt
Is WikiLeaks founder Julian Assange a hero or a villain?
WikiLeaks founder Julian Assange was arrested in London after living in the Ecuadorian embassy for seven years. Assange sought asylum in 2012 after being accused of sexual misconduct in Sweden. Authorities in the United States have been attempting to extradite Assange for years in connection with hacks carried out by Chelsea Manning and accusations that he worked with Russia to meddle in the 2016 election. Supporters say Assange is a hero who has fought to reveal dangerous state secrets. Others say he is a megalomaniac who put lives at risk. What do you think?
By Cait Bladt
#AssangeIsAHero
#AssangeIsBadNews
Assange's myriad supporters say his indictment and possible extradition to the United States represents a "grave threat" to freedom of the press. Assange has long claimed the mantle of journalist, claiming he is protected by freedom of the press. Glenn Greenwald of the Intercept claims the activities Assange is being charged with—encouraging Chelsea Manning to hack into government agencies, providing her with a username that was not her own—are activities any good journalist would engage in.
Whatever else is true about the indictment, substantial parts of the document explicitly characterize as criminal exactly the actions that journalists routinely engage in with their sources, and thus constitutes a dangerous attempt to criminalize investigative journalism.
The indictment, for instance, places great emphasis on Assange's alleged encouragement that Manning – after she already turned over hundreds of thousands of classified documents – try to get more documents for WikiLeaks to publish. The indictment claims that "discussions also reflect Assange actively encouraging Manning to provide more information. During an exchange, Manning told Assange that 'after this upload, that's all I really have got left.' To which Assange replied, 'curious eyes never run dry in my experience.'"
But encouraging sources to obtain more information is something journalists do routinely. Indeed, it would be a breach of one's journalistic duties not to ask vital sources with access to classified information if they could provide even more information so as to allow more complete reporting. If a source comes to a journalist with information, it is entirely common and expected that the journalist would reply: can you also get me X, Y and Z to complete the story or to make it better? As Edward Snowden said this morning, "Bob Woodward stated publicly he would have advised me to remain in place and act as a mole."
https://theintercept.com/2019/04/11/the-u-s-governments-indictment-of-julian-assange-poses-grave-threats-to-press-freedoms/
Writer Michael Weiss is uncompromising in his read of Assange for the Atlantic. Weiss, who refers to Assange both as a "powdered-sugar Saddam" and a "sex pest," believes Assange has used the mantle of a noble whistleblower to hide his own more nefarious, self-interested motives. While Assange frequently claims to be a journalist, he rarely follows journalistic standards and has shown a disconcerting lack of concern for the safety of others.
Releasing U.S. diplomatic communiques which named foreigners living in conflict zones or authoritarian states and liaising with American officials was always going to require thorough vetting and redaction, lest those foreigners be put in harm's way. Assange did not care—he wanted their names published, according to Luke Harding and David Leigh in WikiLeaks: Inside Julian Assange's War on Secrecy. As they recount the story, when Guardian journalists working with WikiLeaks to disseminate its tranche of U.S. secrets tried to explain to Assange why it was morally reprehensible to publish the names of Afghans working with American troops, Assange replied: "Well, they're informants. So, if they get killed, they've got it coming to them. They deserve it." (Assange denied the account; the names, in the end, were not published.)
...Fish and guests might begin to stink after three days, but Assange has reeked from long before he stepped foot in his hideaway cubby across from Harrod's. He has put innocent people's lives in danger; he has defamed and tormented a poor family whose son was murdered; he has seemingly colluded with foreign regimes not simply to out American crimes but to help them carry off their own; and he otherwise made that honorable word "transparency" in as much of a need of delousing as he is.
https://www.theatlantic.com/ideas/archive/2019/04/julian-assange-got-what-he-deserved/587008/
The Guardian, which has worked closely with Assange and published many of the documents WikiLeaks has procured, published an opinion piece penned by its editorial board, arguing Assange should not be extradited to the U.S. While the editors acknowledge Assange is not a blameless person, they believe sending him to America to answer for his actions with regard to publishing state secrets is a blow to the freedom of the press.
From first to last, the Assange case is a morally tangled web. He believes in publishing things that should not always be published – this has long been a difficult divide between the Guardian and him. But he has also shone a light on things that should never have been hidden. When he first entered the Ecuadorian embassy he was trying to avoid extradition to Sweden over allegations of rape and molestation. That was wrong. But those cases have now been closed. He still faces the English courts for skipping bail. If he leaves the embassy, and is arrested, he should answer for that, perhaps in ways that might result in deportation to his own country, Australia. Nothing about this is easy, least of all Mr Assange himself. But when the call comes from Washington, it requires a firm and principled no. It would neither be safe nor right for the UK to extradite Mr Assange to Mr Trump's America.
https://www.theguardian.com/commentisfree/2019/apr/09/the-guardian-view-on-julian-assange-it-would-be-wrong-to-extradite-him
While it is undeniable that WikiLeaks played a critical role in exposing violence and horrible misconduct of the American military in Afghanistan, Assange is not solely responsible for the organization's actions. As Paul Blest argues for the Outline, Assange's behavior has long been abhorrent and his recent pivot towards hard-right bigotry and fear-mongering should preclude him from any praise WikiLeaks itself has garnered.
Assange has long been a dirtbag. Since 2012, Assange has infamously taken refuge in the Ecuadorian embassy in London to avoid extradition to Sweden, where he faced charges for sexual assault. The case was dropped earlier this year because, the prosecutor said, Assange's refuge in the embassy "exhausted" "all possibilities to conduct the investigation." The allegations, which prosecutors said they would resume investigating if Assange ever leaves the embassy, included an accusation that Assange had sex with a woman while she was sleeping. One of the women wrote in 2013 that she had faced harassment and even threats after coming forward.
...[L]ately, Assange has begun to indulge some of the most insidious far-right crankery. Assange and Wikileaks have more than encouraged the theory that murdered DNC staffer Seth Rich was Wikileaks' source for the DNC hack, and even once implied that Rich's parents were bought off. In August of last year, they offered a $20,000 reward for information leading to his killer; Rich's own family, however, has dismissed all of this as a conspiracy theory.
https://theoutline.com/post/2493/julian-assange-sucks
Ashley Hord/The Tylt
Tell Me Everything: How to Protect Voting Rights in the Age of COVID-19
The Road to Super Tuesday: Day Five
The Road to Iowa: Caucus Day
the road to super tuesday
Is Chelsea Manning a whistleblower or a traitor?
Do you still trust the media?
Do you trust WikiLeaks?
Is it wrong for journalists to use anonymous sources?
Should ExxonMobil be held responsible for misleading the public about climate change?
Is Wikileaks' Julian Assange a hero or a villain?
Is Chelsea Manning a hero or a traitor?
Should news outlets publish stories based on anonymous sources?


Termites are a small bug that likes to eat wood. This does not sound so bad now does it? Termites are not a bad bug, if they are not eating your home. If termites are eating your home then they are a big problem. You do not want termites eating your home that is for sure.
Termites cause billions of dollars worth of damages each year just in the United States alone.
One of the biggest problems with termites is that they can eat away at your home for years and you not even know it.
To a termite colony your home is like a gigantic plate of food. The termites favorite food is wood and so just what is your home mostly made of? Yes, that is correct, it is made of wood. Termites are not just some small problem that you do not have to worry about. They are a big problem because they come in colonies of millions to eat your home. If you want to protect your home from being damaged you need to have it checked for termites yearly. Having your house checked every year can help prevent termites from destroying your home and that should give you peace of mind.


text:
Welcome to our beta site. We appreciate any feedback you have.
Policymaker Resource Library
The resources below are devoted to MSMEs and regional trade agreements and are chosen for policymakers and researchers. They appear in the reverse order in which they were uploaded.
Search the Policymaker Resource Library
Need Something More Specific?
Filter by Sub-Topics Africa Asia Dispute Settlement Export Potential Export Readiness Exporting Government Procurement Importing Intellectual Property Latin America and the Caribbean Middle East Non-Tariff Measures Rules of Origin Sanitary and Phytosanitary Measures (SPS) Services Tariffs Technical Barriers to Trade (TBT) Trade Agreements Trade Facilitation Trade Finance Trade Remedies World Trade Organization (WTO)
Value Added Tax Rates (VAT) By Country
This page lists the applicable valued added tax (VAT) rates by countries. It also provides links to related resources and a dedicated e-mail newsletter.
Go directly to website
FedEx: How duties and taxes are calculated
This webpage provides a brief summary and video about how duties and taxes are calculated in international trade. It also notes other costs that businesses should consider.
The General Agreement on Tariffs and Trade (GATT 1947)
This page features the text of the General Agreement on Tariffs and Trade, the treaty which was the predecessor to the World Trade Organization and which still informs its rules on goods trade.
Principles of the trading system
This page provides an overview of the core principles of the multilateral trading system, including non-discrimination, increasingly liberalized trade, fair competition, and predictability.
Tariffs (WTO)
This page provides an explanation for how tariffs works and links to several related resources, including databases, publications, legal texts, and more.
World Tariff Profiles
This annual publication features data for over 170 countries and customs territories on tariffs and non-tariff measures.
WTO Data: Information on trade and trade policy measures
This site provides links to various World Trade Organization databases on goods, services, intellectual property, trade agreements, government procurement, trade and environment, and more.
Regional trade agreements and the WTO
This page provides an overview of how regional trade agreements work and what these mean in the context of the World Trade Organization.
WTO Regional Trade Agreements Database
This database includes information related to regional trade agreements notified to the World Trade Organization.
WTO Integrated Trade Intelligence Portal (I-TIP)
This World Trade Organization database includes information on measures related to goods and services trade. It also covers information on trade agreements, public procurement, and more.
Harmonized System Database
This online tool is designed to help traders as they navigate tariffs, preferential rules of origin, and valuation. It incorporates product-specific information from regional trade agreements.