text:
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.alter;

import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.persist.ReplicaPersistInfo;
import org.apache.doris.persist.ReplicaPersistInfo.ReplicaOperationType;
import org.apache.doris.task.AgentBatchTask;
import org.apache.doris.task.AgentTask;
import org.apache.doris.task.AgentTaskExecutor;
import org.apache.doris.task.AgentTaskQueue;
import org.apache.doris.task.ClearAlterTask;
import org.apache.doris.task.SchemaChangeTask;
import org.apache.doris.thrift.TKeysType;
import org.apache.doris.thrift.TResourceInfo;
import org.apache.doris.thrift.TStorageType;
import org.apache.doris.thrift.TTabletInfo;
import org.apache.doris.thrift.TTaskType;

import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multiset;
import com.google.common.collect.Sets;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

public class SchemaChangeJob extends AlterJob {
    private static final Logger LOG = LogManager.getLogger(SchemaChangeJob.class);

    private String tableName;

    // indexId -> changedColumns
    private Map<Long, List<Column>> changedIndexIdToSchema;
    // indexId -> new schema version
    private Map<Long, Integer> changedIndexIdToSchemaVersion;
    // indexId -> new schema hash
    private Map<Long, Integer> changedIndexIdToSchemaHash;
    // indexId -> new short key column count
    private Map<Long, Short> changedIndexIdToShortKeyColumnCount;

    private TResourceInfo resourceInfo;

    // partition id -> list of replica infos
    private Multimap<Long, ReplicaPersistInfo> replicaInfos;

    /*
     * infos bellow do not need to persist
     */
    // total unfinished replicas
    private List<Long> unfinishedReplicaIds;
    // for calculating process of schemaChange of an index
    // indexId -> totalReplicaNum
    private Multimap<Long, Long> indexIdToTotalReplicaNum;
    // use index id to count finished replica num
    private Multiset<Long> indexIdToFinishedReplicaNum;

    private Multimap<Long, Long> partitionIdToFinishedIndexIds;

    // bloom filter columns
    private boolean hasBfChange;
    private Set<String> bfColumns;
    private double bfFpp;

    // Init as null, to be compatible with former schema change job.
    // If this is set to null, storage type will remain what it was.
    // This can only set to COLUMN
    private TStorageType newStorageType = null;

    private SchemaChangeJob() {
        this(-1, -1, null, null, -1);
    }

    public SchemaChangeJob(long dbId, long tableId, TResourceInfo resourceInfo, String tableName, long transactionId) {
        super(JobType.SCHEMA_CHANGE, dbId, tableId, resourceInfo);

        this.tableName = tableName;

        this.changedIndexIdToSchema = new HashMap<Long, List<Column>>();
        this.changedIndexIdToSchemaVersion = new HashMap<Long, Integer>();
        this.changedIndexIdToSchemaHash = new HashMap<Long, Integer>();
        this.changedIndexIdToShortKeyColumnCount = new HashMap<Long, Short>();

        this.replicaInfos = LinkedHashMultimap.create();

        this.unfinishedReplicaIds = new LinkedList<Long>();
        this.indexIdToTotalReplicaNum = HashMultimap.create();
        this.indexIdToFinishedReplicaNum = HashMultiset.create();

        this.partitionIdToFinishedIndexIds = HashMultimap.create();

        this.hasBfChange = false;
        this.bfColumns = null;
        this.bfFpp = 0;

        this.transactionId = transactionId;
    }

    public final String getTableName() {
        return tableName;
    }

    public synchronized int getTotalReplicaNumByIndexId(long indexId) {
        return this.indexIdToTotalReplicaNum.get(indexId).size();
    }

    // schema
    public void putToChangedIndexSchemaMap(long indexId, List<Column> alterSchema) {
        this.changedIndexIdToSchema.put(indexId, alterSchema);
    }

    public Map<Long, List<Column>> getChangedIndexToSchema() {
        return changedIndexIdToSchema;
    }

    // schema info
    public void setNewSchemaInfo(long indexId, int newSchemaVersion, int newSchemaHash,
            short newShortKeyColumnCount) {
        this.changedIndexIdToSchemaVersion.put(indexId, newSchemaVersion);
        this.changedIndexIdToSchemaHash.put(indexId, newSchemaHash);
        this.changedIndexIdToShortKeyColumnCount.put(indexId, newShortKeyColumnCount);
    }

    // schema version
    public int getSchemaVersionByIndexId(long indexId) {
        if (changedIndexIdToSchemaVersion.containsKey(indexId)) {
            return changedIndexIdToSchemaVersion.get(indexId);
        }
        return -1;
    }

    // schema hash
    public int getSchemaHashByIndexId(long indexId) {
        if (changedIndexIdToSchemaHash.containsKey(indexId)) {
            return changedIndexIdToSchemaHash.get(indexId);
        }
        return -1;
    }

    // short key column count
    public short getShortKeyColumnCountByIndexId(long indexId) {
        if (changedIndexIdToShortKeyColumnCount.containsKey(indexId)) {
            return changedIndexIdToShortKeyColumnCount.get(indexId);
        }
        return (short) -1;
    }

    // bloom filter info
    public void setTableBloomFilterInfo(boolean hasBfChange, Set<String> bfColumns, double bfFpp) {
        this.hasBfChange = hasBfChange;
        this.bfColumns = bfColumns;
        this.bfFpp = bfFpp;
    }

    public void setNewStorageType(TStorageType newStorageType) {
        Preconditions.checkState(newStorageType == TStorageType.COLUMN);
        this.newStorageType = newStorageType;
    }

    public boolean isSchemaHashRelated(int schemaHash) {
        return changedIndexIdToSchemaHash.values().contains(schemaHash);
    }

    public synchronized int getFinishedReplicaNumByIndexId(long indexId) {
        return this.indexIdToFinishedReplicaNum.count(indexId);
    }

    public void deleteAllTableHistorySchema() {
        Database db = Catalog.getInstance().getDb(dbId);
        if (db == null) {
            LOG.warn("db[{}] does not exist", dbId);
            return;
        }

        db.readLock();
        try {
            OlapTable olapTable = (OlapTable) db.getTable(tableId);
            if (olapTable == null) {
                LOG.warn("table[{}] does not exist in db[{}]", tableId, dbId);
                return;
            }
            // drop all replicas with old schemaHash
            for (Partition partition : olapTable.getPartitions()) {
                long partitionId = partition.getId();
                for (Long indexId : this.changedIndexIdToSchema.keySet()) {
                    MaterializedIndex materializedIndex = partition.getIndex(indexId);
                    if (materializedIndex == null) {
                        LOG.warn("index[{}] does not exist in partition[{}-{}-{}]",
                                 indexId, dbId, tableId, partitionId);
                        continue;
                    }

                    // delete schema hash
                    // the real drop task is handled by report process
                    // we call 'deleteNewSchemaHash' but we delete old one actually.
                    // cause schema hash is switched when job is finished.
                    Catalog.getCurrentInvertedIndex().deleteNewSchemaHash(partitionId, indexId);
                    LOG.info("delete old schema. db[{}], table[{}], partition[{}], index[{}]",
                             dbId, tableId, partitionId, indexId);
                }
            } // end for partitions
        } finally {
            db.readUnlock();
        }
        return;
    }

    @Override
    public void addReplicaId(long parentId, long replicaId, long backendId) {
        // here parent id is index id
        this.unfinishedReplicaIds.add(replicaId);
        this.indexIdToTotalReplicaNum.put(parentId, replicaId);
        // this.backendIdToReplicaIds.put(backendId, replicaId);
    }

    @Override
    public synchronized void setReplicaFinished(long parentId, long replicaId) {
        // here parent id is index id
        if (!this.unfinishedReplicaIds.remove(replicaId)) {
            // this replica is already removed
            return;
        }

        if (parentId == -1L) {
            // find out which table is replica belongs to
            for (long oneIndexId : indexIdToTotalReplicaNum.keySet()) {
                Collection<Long> replicaIds = indexIdToTotalReplicaNum.get(oneIndexId);
                if (replicaIds.contains(replicaId)) {
                    this.indexIdToFinishedReplicaNum.add(oneIndexId);
                    break;
                }
            }
        } else {
            this.indexIdToFinishedReplicaNum.add(parentId);
        }
    }

    /*
     * return
     * 0:  sending clear tasks
     * 1:  all clear tasks are finished, the job is done normally.
     * -1: job meet some fatal error, like db or table is missing.
     */
    public int checkOrResendClearTasks() {
        Preconditions.checkState(this.state == JobState.FINISHING);
        // 1. check if all task finished
        boolean clearFailed = false;
        if (batchClearAlterTask != null) {
            List<AgentTask> allTasks = batchClearAlterTask.getAllTasks();
            for (AgentTask oneClearAlterTask : allTasks) {
                ClearAlterTask clearAlterTask = (ClearAlterTask) oneClearAlterTask;
                if (!clearAlterTask.isFinished()) {
                    clearFailed = true;
                }
                AgentTaskQueue.removeTask(clearAlterTask.getBackendId(),
                        TTaskType.CLEAR_ALTER_TASK, clearAlterTask.getSignature());
                // not remove the task from batch task, remove it by gc
            }
        }
        if (!clearFailed && batchClearAlterTask != null) {
            return 1;
        }

        Database db = Catalog.getInstance().getDb(dbId);
        if (db == null) {
            cancelMsg = "db[" + dbId + "] does not exist";
            LOG.warn(cancelMsg);
            return -1;
        }

        batchClearAlterTask = new AgentBatchTask();
        db.readLock();
        try {
            OlapTable olapTable = (OlapTable) db.getTable(tableId);
            if (olapTable == null) {
                cancelMsg = "could not find table[" + tableId + "] in db [" + dbId + "]";
                LOG.warn(cancelMsg);
                return -1;
            }

            boolean allAddSuccess = true;
            LOG.info("sending clear schema change job tasks for table [{}]", tableId);
            OUTER_LOOP:
            for (Partition partition : olapTable.getPartitions()) {
                long partitionId = partition.getId();
                for (MaterializedIndex index : partition.getMaterializedIndices()) {
                    for (Tablet tablet : index.getTablets()) {
                        List<Replica> replicas = tablet.getReplicas();
                        for (Replica replica : replicas) {
                            long backendId = replica.getBackendId();
                            ClearAlterTask clearAlterTask = new ClearAlterTask(backendId, dbId, tableId,
                                    partitionId, index.getId(), tablet.getId(),
                                    olapTable.getSchemaHashByIndexId(index.getId()));
                            if (AgentTaskQueue.addTask(clearAlterTask)) {
                                batchClearAlterTask.addTask(clearAlterTask);
                            } else {
                                allAddSuccess = false;
                                break OUTER_LOOP;
                            }
                        } // end for rollupReplicas
                    } // end for rollupTablets
                } // end for index
            } // end for partition
            if (!allAddSuccess) {
                for (AgentTask task : batchClearAlterTask.getAllTasks()) {
                    AgentTaskQueue.removeTask(task.getBackendId(), task.getTaskType(), task.getSignature());
                }
                batchClearAlterTask = null;
            }
        } finally {
            db.readUnlock();
        }

        LOG.info("successfully sending clear schema change job [{}]", tableId);
        return 0;
    }

    @Override
    public boolean sendTasks() {
        Preconditions.checkState(this.state == JobState.PENDING);
        // here we just sending tasks to AgentTaskQueue.
        // task report process will later resend this task

        LOG.info("sending schema change job {}, start txn id: {}", tableId, transactionId);

        Database db = Catalog.getInstance().getDb(dbId);
        if (db == null) {
            String msg = "db[" + dbId + "] does not exist";
            setMsg(msg);
            LOG.warn(msg);
            return false;
        }

        db.readLock();
        try {
            synchronized (this) {
                OlapTable olapTable = (OlapTable) db.getTable(tableId);
                if (olapTable == null) {
                    cancelMsg = "table[" + tableId + "] does not exist";
                    LOG.warn(cancelMsg);
                    return false;
                }

                Preconditions.checkNotNull(this.unfinishedReplicaIds);

                List<AgentTask> tasks = new LinkedList<AgentTask>();
                for (Partition partition : olapTable.getPartitions()) {
                    long partitionId = partition.getId();
                    short replicationNum = olapTable.getPartitionInfo().getReplicationNum(partitionId);
                    for (Long indexId : this.changedIndexIdToSchema.keySet()) {
                        MaterializedIndex alterIndex = partition.getIndex(indexId);
                        if (alterIndex == null) {
                            cancelMsg = "index[" + indexId + "] does not exist in partition[" + partitionId + "]";
                            LOG.warn(cancelMsg);
                            return false;
                        }

                        List<Column> alterSchema = this.changedIndexIdToSchema.get(indexId);
                        int newSchemaHash = this.changedIndexIdToSchemaHash.get(indexId);
                        Preconditions.checkState(newSchemaHash != -1);
                        int baseSchemaHash = olapTable.getSchemaHashByIndexId(indexId);
                        short newShortKeyColumnCount = this.changedIndexIdToShortKeyColumnCount.get(indexId);
                        Preconditions.checkState(newShortKeyColumnCount != (short) -1);
                        KeysType keysType = olapTable.getKeysType();
                        TKeysType schemaChangeKeysType;
                        if (keysType == KeysType.DUP_KEYS) {
                            schemaChangeKeysType = TKeysType.DUP_KEYS;
                        } else if (keysType == KeysType.UNIQUE_KEYS) {
                            schemaChangeKeysType = TKeysType.UNIQUE_KEYS;
                        } else {
                            schemaChangeKeysType = TKeysType.AGG_KEYS;
                        }

                        TStorageType storageType = newStorageType == null ? olapTable.getStorageTypeByIndexId(indexId)
                                : newStorageType;
                        for (Tablet tablet : alterIndex.getTablets()) {
                            long tabletId = tablet.getId();
                            short replicaSendNum = 0;
                            for (Replica replica : tablet.getReplicas()) {
                                if (replica.getState() != ReplicaState.SCHEMA_CHANGE) {
                                    // for now, all replica should be in SCHEMA_CHANGE,
                                    // because we don't allow tablet repair and balance during schema change.
                                    // but in case some edge cases are not took into consideration, we cancel
                                    // the schema change job here.
                                    cancelMsg = String.format(
                                            "replica %d of tablet %d in backend %d state is invalid: %s [send]",
                                            replica.getId(), tablet.getId(), replica.getBackendId(),
                                            replica.getState().name());
                                    LOG.warn(cancelMsg);
                                    return false;
                                }

                                long backendId = replica.getBackendId();
                                long replicaId = replica.getId();
                                SchemaChangeTask schemaChangeTask =
                                        new SchemaChangeTask(resourceInfo, backendId, dbId, tableId,
                                                             partitionId, indexId, tabletId, replicaId,
                                                             alterSchema, newSchemaHash,
                                                             baseSchemaHash, newShortKeyColumnCount,
                                                             storageType,
                                                             bfColumns, bfFpp, schemaChangeKeysType);
                                addReplicaId(indexId, replicaId, backendId);
                                tasks.add(schemaChangeTask);
                                replicaSendNum++;
                            }

                            if (replicaSendNum < replicationNum / 2 + 1) {
                                // In the case that quorum num of non-NORMAL replica(probably CLONE)
                                // in this tablet, schema change job can not finish.
                                // So cancel it.
                                cancelMsg = String.format("num of normal replica in tablet %d is less than quorum num",
                                                          tabletId);
                                LOG.warn(cancelMsg);
                                return false;
                            }
                        } // end for tablets
                    } // end for alter indices
                } // end for partitions

                AgentBatchTask batchTask = new AgentBatchTask();
                // add all schemaChangeTask to AgentTaskQueue
                for (AgentTask task : tasks) {
                    if (!AgentTaskQueue.addTask(task)) {
                        cancelMsg = "failed add schema change task[" + task.getTabletId()
                                + ":" + task.getBackendId() + "]";
                        LOG.warn(cancelMsg);
                        return false;
                    } else {
                        batchTask.addTask(task);
                    }
                }

                if (batchTask.getTaskNum() > 0) {
                    AgentTaskExecutor.submit(batchTask);
                }
                // change schemaChangeJob's status
                this.state = JobState.RUNNING;
            } // end synchronized block
        } finally {
            db.readUnlock();
        }

        Preconditions.checkState(this.state == JobState.RUNNING);
        LOG.info("successfully sending schema change job[{}]", tableId);
        return true;
    }

    @Override
    public synchronized void cancel(OlapTable olapTable, String msg) {
        // make sure to get db write lock before calling this

        if (olapTable != null) {
            // 1. remove all task and set state
            for (Partition partition : olapTable.getPartitions()) {
                if (partition.getState() == PartitionState.NORMAL) {
                    continue;
                }
                long partitionId = partition.getId();
                for (Long indexId : this.changedIndexIdToSchema.keySet()) {
                    MaterializedIndex index = partition.getIndex(indexId);
                    if (index == null || index.getState() == IndexState.NORMAL) {
                        continue;
                    }
                    for (Tablet tablet : index.getTablets()) {
                        long tabletId = tablet.getId();
                        for (Replica replica : tablet.getReplicas()) {
                            if (replica.getState() == ReplicaState.CLONE || replica.getState() == ReplicaState.NORMAL) {
                                continue;
                            }
                            Preconditions.checkState(replica.getState() == ReplicaState.SCHEMA_CHANGE);
                            replica.setState(ReplicaState.NORMAL);
                            AgentTaskQueue.removeTask(replica.getBackendId(), TTaskType.SCHEMA_CHANGE, tabletId);
                        } // end for replicas
                    } // end for tablets

                    // delete schema hash in inverted index
                    Catalog.getCurrentInvertedIndex().deleteNewSchemaHash(partitionId, indexId);
                    Preconditions.checkArgument(index.getState() == IndexState.SCHEMA_CHANGE);
                    index.setState(IndexState.NORMAL);
                } // end for indices
                partition.setState(PartitionState.NORMAL);
            } // end for partitions
            olapTable.setState(OlapTableState.NORMAL);
        }

        this.state = JobState.CANCELLED;
        if (Strings.isNullOrEmpty(cancelMsg) && !Strings.isNullOrEmpty(msg)) {
            this.cancelMsg = msg;
        }

        this.finishedTime = System.currentTimeMillis();

        // 2. log
        Catalog.getInstance().getEditLog().logCancelSchemaChange(this);
        LOG.info("cancel schema change job[{}] finished, because: {}",
                olapTable == null ? -1 : olapTable.getId(), cancelMsg);
    }

    @Override
    public synchronized void removeReplicaRelatedTask(long parentId, long tabletId, long replicaId, long backendId) {
        // parentId is unused here
        setReplicaFinished(-1, replicaId);

        // this.backendIdToReplicaIds.get(backendId).remove(replicaId);
        // remove task
        AgentTaskQueue.removeTask(backendId, TTaskType.SCHEMA_CHANGE, tabletId);
    }

    @Override
    public void handleFinishedReplica(AgentTask task, TTabletInfo finishTabletInfo, long reportVersion)
            throws MetaNotFoundException {
        Preconditions.checkArgument(task instanceof SchemaChangeTask);
        SchemaChangeTask schemaChangeTask = (SchemaChangeTask) task;

        // check schema hash to avoid former schema change task try finishing current task
        int finishTabletInfoSchemaHash = finishTabletInfo.getSchema_hash();
        int taskSchemaHash = schemaChangeTask.getSchemaHash();
        if (finishTabletInfoSchemaHash != taskSchemaHash) {
            throw new MetaNotFoundException("Schema hash is not equal[" + finishTabletInfoSchemaHash + "-"
                    + taskSchemaHash + "], tablet: " + schemaChangeTask.getTabletId());
        }

        long dbId = schemaChangeTask.getDbId();
        long partitionId = schemaChangeTask.getPartitionId();
        long indexId = schemaChangeTask.getIndexId();
        long tabletId = schemaChangeTask.getTabletId();
        long replicaId = schemaChangeTask.getReplicaId();

        // update replica's info
        Database db = Catalog.getInstance().getDb(dbId);
        if (db == null) {
            throw new MetaNotFoundException("Cannot find db[" + dbId + "]");
        }
        db.writeLock();
        try {
            OlapTable olapTable = (OlapTable) db.getTable(tableId);
            if (olapTable == null) {
                throw new MetaNotFoundException("Cannot find table[" + tableId + "]");
            }
            Preconditions.checkState(olapTable.getState() == OlapTableState.SCHEMA_CHANGE);

            Partition partition = olapTable.getPartition(partitionId);
            if (partition == null) {
                throw new MetaNotFoundException("Cannot find partition[" + partitionId + "]");
            }
            Preconditions.checkState(partition.getState() == PartitionState.SCHEMA_CHANGE);

            MaterializedIndex materializedIndex = partition.getIndex(indexId);
            if (materializedIndex == null) {
                throw new MetaNotFoundException("Cannot find index[" + indexId + "]");
            }
            Preconditions.checkState(materializedIndex.getState() == IndexState.SCHEMA_CHANGE);

            Preconditions.checkArgument(finishTabletInfo.getTablet_id() == tabletId);
            Tablet tablet = materializedIndex.getTablet(tabletId);
            if (tablet == null) {
                throw new MetaNotFoundException("Cannot find tablet[" + tabletId + "]");
            }

            Replica replica = tablet.getReplicaById(replicaId);
            if (replica == null) {
                throw new MetaNotFoundException("Cannot find replica[" + replicaId + "]");
            }
            // replica's state may be NORMAL(due to clone), so no need to check

            long version = finishTabletInfo.getVersion();
            long versionHash = finishTabletInfo.getVersion_hash();
            long dataSize = finishTabletInfo.getData_size();
            long rowCount = finishTabletInfo.getRow_count();
            // do not need check version > replica.getVersion, because the new replica's version is first set by sc
            replica.updateVersionInfo(version, versionHash, dataSize, rowCount);
        } finally {
            db.writeUnlock();
        }

        Catalog.getCurrentSystemInfo().updateBackendReportVersion(schemaChangeTask.getBackendId(),
                                                                    reportVersion, dbId);
        setReplicaFinished(indexId, replicaId);

        LOG.info("finish schema change replica[{}]. index[{}]. tablet[{}], backend[{}]",
                 replicaId, indexId, tabletId, task.getBackendId());
    }

    /**
     * should consider following cases:
     * 1. replica is removed from this tablet.
     * 2. backend is dead or is dropped from system
     *
     * we make new schema visible, but keep table's state as SCHEMA_CHANGE.
     * 1. Make the new schema visible, because we want that the following load jobs will only load
     * data to the new tablet.
     * 2. keep the table's state in SCHEMA_CHANGE, because we don't want another alter job being processed.
     */
    @Override
    public int tryFinishJob() {
        if (this.state != JobState.RUNNING) {
            LOG.info("schema change job[{}] is not running.", tableId);
            return 0;
        }

        Database db = Catalog.getInstance().getDb(dbId);
        if (db == null) {
            cancelMsg = String.format("database %d does not exist", dbId);
            LOG.warn(cancelMsg);
            return -1;
        }

        db.writeLock();
        try {
            synchronized (this) {
                Table table = db.getTable(tableId);
                if (table == null) {
                    cancelMsg = String.format("table %d does not exist", tableId);
                    LOG.warn(cancelMsg);
                    return -1;
                }

                boolean hasUnfinishedPartition = false;
                OlapTable olapTable = (OlapTable) table;
                for (Partition partition : olapTable.getPartitions()) {
                    long partitionId = partition.getId();
                    short expectReplicationNum = olapTable.getPartitionInfo().getReplicationNum(partition.getId());
                    boolean hasUnfinishedIndex = false;
                    for (long indexId : this.changedIndexIdToSchema.keySet()) {
                        MaterializedIndex materializedIndex = partition.getIndex(indexId);
                        if (materializedIndex == null) {
                            // this should not happen, we do not allow dropping rollup during schema change.
                            // cancel the job here.
                            cancelMsg = String.format("index %d is missing", indexId);
                            LOG.warn(cancelMsg);
                            return -1;
                        }

                        for (Tablet tablet : materializedIndex.getTablets()) {
                            List<Replica> replicas = tablet.getReplicas();
                            List<Replica> errorReplicas = Lists.newArrayList();
                            int healthNum = replicas.size();
                            for (Replica replica : replicas) {
                                if (replica.getState() != ReplicaState.SCHEMA_CHANGE) {
                                    // all replicas should be in state SCHEMA_CHANGE
                                    cancelMsg = String.format(
                                            "replica %d of tablet %d in backend %d state is invalid: %s [try finish]",
                                            replica.getId(), tablet.getId(), replica.getBackendId(),
                                            replica.getState().name());
                                    LOG.warn(cancelMsg);
                                    return -1;
                                }

                                if (!checkBackendState(replica)) {
                                    LOG.warn("backend {} state is abnormal, set replica {} of tablet {} as bad",
                                            replica.getBackendId(), tablet.getId(), replica.getId());
                                    errorReplicas.add(replica);
                                    --healthNum;
                                    continue;
                                }

                                if (replica.getLastFailedVersion() > 0 && System.currentTimeMillis()
                                        - replica.getLastFailedTimestamp() > Config.max_backend_down_time_second
                                                * 1000) {
                                    LOG.warn("replica {} of tablet {} last failed version > 0, "
                                            + "and last for an hour, set it as bad", replica, tablet.getId());
                                    --healthNum;
                                    continue;
                                }
                            }

                            if (healthNum < (expectReplicationNum / 2 + 1)) {
                                cancelMsg = String.format("schema change job[%d] cancelled. "
                                        + "tablet[%d] has few health replica."
                                        + " num: %d", tableId, tablet.getId(), healthNum);
                                LOG.warn(cancelMsg);
                                return -1;
                            }

                            for (Replica errReplica : errorReplicas) {
                                // For now, err replicas are those replicas which the backends they belong to is dead.
                                // We need to set these replicas as finished to let the schema change job
                                // finished.
                                setReplicaFinished(indexId, errReplica.getId());
                                // remove the replica from backend to replica map
                                // backendIdToReplicaIds.get(errReplica.getBackendId()).remove(errReplica.getId());
                                // remove error replica related task
                                AgentTaskQueue.removeTask(errReplica.getBackendId(), TTaskType.SCHEMA_CHANGE,
                                                          tablet.getId());
                            }
                        } // end for tablets

                        // check if index is finished
                        if (!this.partitionIdToFinishedIndexIds.containsKey(partitionId)
                                || !this.partitionIdToFinishedIndexIds.get(partitionId).contains(indexId)) {
                            int finishedReplicaNum = indexIdToFinishedReplicaNum.count(indexId);
                            int totalReplicaNum = indexIdToTotalReplicaNum.get(indexId).size();
                            if (finishedReplicaNum < totalReplicaNum) {
                                LOG.debug("index[{}] has unfinished replica. {}/{}", indexId,
                                          finishedReplicaNum, totalReplicaNum);
                                hasUnfinishedIndex = true;
                                continue;
                            }
                        }

                        // index is finished
                        // remove task for safety
                        // task may be left if some backends are down during schema change
                        if (!this.partitionIdToFinishedIndexIds.containsKey(partitionId)
                                || !this.partitionIdToFinishedIndexIds.get(partitionId).contains(indexId)) {
                            for (Tablet tablet : materializedIndex.getTablets()) {
                                for (Replica replica : tablet.getReplicas()) {
                                    AgentTaskQueue.removeTask(replica.getBackendId(), TTaskType.SCHEMA_CHANGE,
                                                              tablet.getId());
                                }
                            }

                            Preconditions.checkState(materializedIndex.getState() == IndexState.SCHEMA_CHANGE);
                            this.partitionIdToFinishedIndexIds.put(partitionId, indexId);
                            LOG.debug("schema change tasks finished in table[{}]", materializedIndex.getId());
                        }
                    } // end for indices

                    if (hasUnfinishedIndex) {
                        hasUnfinishedPartition = true;
                    }

                    // all table finished in this partition
                    LOG.info("schema change finished in partition {}, table: {}", partition.getId(), table.getId());
                } // end for partitions

                if (hasUnfinishedPartition) {
                    return 0;
                }

                Preconditions.checkState(unfinishedReplicaIds.isEmpty());

                // all partitions are finished
                // update state and save replica info
                Preconditions.checkState(olapTable.getState() == OlapTableState.SCHEMA_CHANGE);
                for (Partition partition : olapTable.getPartitions()) {
                    Preconditions.checkState(partition.getState() == PartitionState.SCHEMA_CHANGE);
                    long partitionId = partition.getId();
                    for (long indexId : this.changedIndexIdToSchema.keySet()) {
                        MaterializedIndex materializedIndex = partition.getIndex(indexId);
                        int schemaHash = changedIndexIdToSchemaHash.get(indexId);
                        Preconditions.checkState(materializedIndex.getState() == IndexState.SCHEMA_CHANGE);
                        for (Tablet tablet : materializedIndex.getTablets()) {
                            long tabletId = tablet.getId();
                            for (Replica replica : tablet.getReplicas()) {
                                if (replica.getState() != ReplicaState.SCHEMA_CHANGE) {
                                    // all replicas should be in state SCHEMA_CHANGE
                                    cancelMsg = String.format(
                                            "replica %d of tablet %d in backend %d state is invalid: %s [finish]",
                                            replica.getId(), tablet.getId(), replica.getBackendId(),
                                            replica.getState().name());
                                    LOG.warn(cancelMsg);
                                }

                                ReplicaPersistInfo replicaInfo = ReplicaPersistInfo.createForSchemaChange(partitionId,
                                        indexId, tabletId,
                                        replica.getBackendId(),
                                        replica.getVersion(),
                                        replica.getVersionHash(),
                                        schemaHash,
                                        replica.getDataSize(),
                                        replica.getRowCount(),
                                        replica.getLastFailedVersion(),
                                        replica.getLastFailedVersionHash(),
                                        replica.getLastSuccessVersion(),
                                        replica.getLastSuccessVersionHash());
                                this.replicaInfos.put(partitionId, replicaInfo);

                                replica.setState(ReplicaState.NORMAL);
                                replica.setSchemaHash(schemaHash);

                                // remove tasks for safety
                                AgentTaskQueue.removeTask(replica.getBackendId(), TTaskType.SCHEMA_CHANGE,
                                        tabletId);
                            } // end for replicas
                        } // end for tablets

                        // update schema hash
                        Catalog.getCurrentInvertedIndex().updateToNewSchemaHash(partitionId, indexId);
                        materializedIndex.setState(IndexState.NORMAL);
                    } // end for indices
                    partition.setState(PartitionState.NORMAL);
                } // end for partitions

                // 2. update to new schema for each index
                for (Map.Entry<Long, List<Column>> entry : changedIndexIdToSchema.entrySet()) {
                    Long indexId = entry.getKey();
                    int schemaVersion = changedIndexIdToSchemaVersion.get(indexId);
                    int schemaHash = changedIndexIdToSchemaHash.get(indexId);
                    short shortKeyColumnCount = changedIndexIdToShortKeyColumnCount.get(indexId);
                    olapTable.setIndexSchemaInfo(indexId, null, entry.getValue(), schemaVersion, schemaHash,
                                                 shortKeyColumnCount);

                    if (newStorageType != null) {
                        olapTable.setIndexStorageType(indexId, newStorageType);
                    }
                }

                // 3. update base schema if changed
                if (this.changedIndexIdToSchema.containsKey(olapTable.getBaseIndexId())) {
                    table.setNewBaseSchema(this.changedIndexIdToSchema.get(olapTable.getBaseIndexId()));
                }

                // 4. update table bloom filter columns
                if (hasBfChange) {
                    olapTable.setBloomFilterInfo(bfColumns, bfFpp);
                }

                this.state = JobState.FINISHING;
                this.transactionId = Catalog.getCurrentGlobalTransactionMgr().getTransactionIDGenerator().getNextTransactionId();
            }
        } finally {
            db.writeUnlock();
        }

        Catalog.getInstance().getEditLog().logFinishingSchemaChange(this);
        LOG.info("schema change job is finishing. finishing txn id: {} table {}", transactionId, tableId);
        return 1;
    }

    @Override
    public void finishJob() {
        Database db = Catalog.getInstance().getDb(dbId);
        if (db == null) {
            cancelMsg = String.format("database %d does not exist", dbId);
            LOG.warn(cancelMsg);
            return;
        }

        db.writeLock();
        try {
            OlapTable olapTable = (OlapTable) db.getTable(tableId);
            if (olapTable == null) {
                cancelMsg = String.format("table %d does not exist", tableId);
                LOG.warn(cancelMsg);
                return;
            }
            olapTable.setState(OlapTableState.NORMAL);
        } finally {
            db.writeUnlock();
        }

        this.finishedTime = System.currentTimeMillis();
        LOG.info("finished schema change job: {}", tableId);
    }

    @Override
    public synchronized void clear() {
        changedIndexIdToSchema = null;
        resourceInfo = null;
        replicaInfos = null;
        unfinishedReplicaIds = null;
        indexIdToTotalReplicaNum = null;
        indexIdToFinishedReplicaNum = null;
        partitionIdToFinishedIndexIds = null;
    }

    @Override
    public void replayInitJob(Database db) {
        db.writeLock();
        try {
            OlapTable olapTable = (OlapTable) db.getTable(tableId);

            // change the state of table/partition and replica, then add object to related List and Set
            for (Partition partition : olapTable.getPartitions()) {
                for (Map.Entry<Long, Integer> entry : changedIndexIdToSchemaHash.entrySet()) {
                    MaterializedIndex index = partition.getIndex(entry.getKey());
                    // set state to SCHEMA_CHANGE
                    for (Tablet tablet : index.getTablets()) {
                        for (Replica replica : tablet.getReplicas()) {
                            if (replica.getState() == ReplicaState.CLONE) {
                                // add log here, because there should no more CLONE replica when processing alter jobs.
                                LOG.warn(String.format(
                                        "replica %d of tablet %d in backend %d state is invalid: %s",
                                        replica.getId(), tablet.getId(), replica.getBackendId(),
                                        replica.getState().name()));
                                continue;
                            }
                            replica.setState(ReplicaState.SCHEMA_CHANGE);
                        }
                    }
                    index.setState(IndexState.SCHEMA_CHANGE);

                    Catalog.getCurrentInvertedIndex().setNewSchemaHash(partition.getId(), entry.getKey(),
                                                                       entry.getValue());
                }

                partition.setState(PartitionState.SCHEMA_CHANGE);
            } // end for partitions

            olapTable.setState(OlapTableState.SCHEMA_CHANGE);

            // reset status to PENDING for resending the tasks in polling thread
            this.state = JobState.PENDING;
        } finally {
            db.writeUnlock();
        }
    }

    @Override
    public void replayFinishing(Database db) {
        db.writeLock();
        try {
            OlapTable olapTable = (OlapTable) db.getTable(tableId);

            // set the status to normal
            for (Partition partition : olapTable.getPartitions()) {
                long partitionId = partition.getId();
                for (Map.Entry<Long, Integer> entry : changedIndexIdToSchemaHash.entrySet()) {
                    MaterializedIndex index = partition.getIndex(entry.getKey());
                    for (Tablet tablet : index.getTablets()) {
                        for (Replica replica : tablet.getReplicas()) {
                            replica.setState(ReplicaState.NORMAL);
                        }
                    }

                    index.setState(IndexState.NORMAL);

                    // update to new schema hash in inverted index
                    Catalog.getCurrentInvertedIndex().updateToNewSchemaHash(partitionId, index.getId());
                    Catalog.getCurrentInvertedIndex().deleteNewSchemaHash(partitionId, index.getId());
                }
                partition.setState(PartitionState.NORMAL);

                // update replica info
                Collection<ReplicaPersistInfo> replicaInfo = replicaInfos.get(partition.getId());
                if (replicaInfo != null) {
                    for (ReplicaPersistInfo info : replicaInfo) {
                        MaterializedIndex mIndex = (MaterializedIndex) partition.getIndex(info.getIndexId());
                        int schemaHash = info.getSchemaHash();
                        // for compatibility
                        if (schemaHash == -1) {
                            schemaHash = getSchemaHashByIndexId(info.getIndexId());
                        }
                        Tablet tablet = mIndex.getTablet(info.getTabletId());
                        if (info.getOpType() == ReplicaOperationType.SCHEMA_CHANGE) {
                            Replica replica = tablet.getReplicaByBackendId(info.getBackendId());
                            replica.updateVersionInfo(info.getVersion(), info.getVersionHash(),
                                                      info.getLastFailedVersion(),
                                                      info.getLastFailedVersionHash(),
                                                      info.getLastSuccessVersion(),
                                                      info.getLastSuccessVersionHash());
                            replica.setSchemaHash(schemaHash);
                        } else if (info.getOpType() == ReplicaOperationType.DELETE) {
                            // remove the replica from replica group
                            tablet.deleteReplicaByBackendId(info.getBackendId());
                        }
                    }
                }
            } // end for partitions

            // update schema
            for (Map.Entry<Long, List<Column>> entry : changedIndexIdToSchema.entrySet()) {
                long indexId = entry.getKey();
                int schemaVersion = getSchemaVersionByIndexId(indexId);
                int schemaHash = getSchemaHashByIndexId(indexId);
                short shortKeyColumnCount = getShortKeyColumnCountByIndexId(indexId);
                olapTable.setIndexSchemaInfo(indexId, null, entry.getValue(), schemaVersion, schemaHash,
                                             shortKeyColumnCount);

                if (newStorageType != null) {
                    olapTable.setIndexStorageType(indexId, newStorageType);
                }
                if (indexId == olapTable.getBaseIndexId()) {
                    olapTable.setNewBaseSchema(entry.getValue());
                }
            }

            // bloom filter columns
            if (hasBfChange) {
                olapTable.setBloomFilterInfo(bfColumns, bfFpp);
            } // end for partitions
        } finally {
            db.writeUnlock();
        }

        LOG.info("replay finishing schema change job: {}", tableId);
    }

    @Override
    public void replayFinish(Database db) {
        // if this is an old job, then should also update table or replica state
        if (transactionId < 0) {
            replayFinishing(db);
        }

        db.writeLock();
        try {
            OlapTable olapTable = (OlapTable) db.getTable(tableId);
            olapTable.setState(OlapTableState.NORMAL);
        } finally {
            db.writeUnlock();
        }

        LOG.info("replay finish schema change job: {}", tableId);
    }

    @Override
    public void replayCancel(Database db) {
        db.writeLock();
        try {
            // restore partition's state
            OlapTable olapTable = (OlapTable) db.getTable(tableId);
            if (olapTable == null) {
                return;
            }
            for (Partition partition : olapTable.getPartitions()) {
                long partitionId = partition.getId();
                for (Long indexId : this.changedIndexIdToSchema.keySet()) {
                    MaterializedIndex index = partition.getIndex(indexId);
                    if (index == null) {
                        continue;
                    }
                    for (Tablet tablet : index.getTablets()) {
                        for (Replica replica : tablet.getReplicas()) {
                            if (replica.getState() == ReplicaState.CLONE || replica.getState() == ReplicaState.NORMAL) {
                                continue;
                            }
                            replica.setState(ReplicaState.NORMAL);
                        } // end for replicas
                    } // end for tablets

                    Preconditions.checkState(index.getState() == IndexState.SCHEMA_CHANGE, index.getState());
                    index.setState(IndexState.NORMAL);

                    // delete new schema hash in invered index
                    Catalog.getCurrentInvertedIndex().deleteNewSchemaHash(partitionId, indexId);
                } // end for indices

                Preconditions.checkState(partition.getState() == PartitionState.SCHEMA_CHANGE,
                                         partition.getState());
                partition.setState(PartitionState.NORMAL);
            } // end for partitions

            olapTable.setState(OlapTableState.NORMAL);
        } finally {
            db.writeUnlock();
        }
    }

    @Override
    public void getJobInfo(List<List<Comparable>> jobInfos, OlapTable tbl) {
        if (changedIndexIdToSchemaVersion == null) {
            // for compatibility
            if (state == JobState.FINISHED || state == JobState.CANCELLED) {
                List<Comparable> jobInfo = new ArrayList<Comparable>();
                jobInfo.add(tableId); // job id
                jobInfo.add(tbl.getName()); // table name
                jobInfo.add(TimeUtils.longToTimeString(createTime));
                jobInfo.add(TimeUtils.longToTimeString(finishedTime));
                jobInfo.add("N/A"); // index name
                jobInfo.add("N/A"); // index id
                jobInfo.add("N/A"); // schema version
                jobInfo.add("N/A"); // index state
                jobInfo.add(-1); // transaction id
                jobInfo.add(state.name()); // job state
                jobInfo.add("N/A"); // progress
                jobInfo.add(cancelMsg);
                jobInfos.add(jobInfo);
                return;
            }

            // in previous version, changedIndexIdToSchema is set to null
            // when job is finished or cancelled.
            // so if changedIndexIdToSchema == null, the job'state must be FINISHED or CANCELLED
            return;
        }

        Map<Long, String> indexProgress = new HashMap<Long, String>();
        Map<Long, String> indexState = new HashMap<Long, String>();

        // calc progress and state for each table
        for (Long indexId : changedIndexIdToSchemaVersion.keySet()) {
            if (tbl.getIndexNameById(indexId) == null) {
                // this index may be dropped, and this should be a FINISHED job, just use a dummy info to show
                indexState.put(indexId, IndexState.NORMAL.name());
                indexProgress.put(indexId, "100%");
            } else {
                int totalReplicaNum = 0;
                int finishedReplicaNum = 0;
                String idxState = IndexState.NORMAL.name();
                for (Partition partition : tbl.getPartitions()) {
                    MaterializedIndex index = partition.getIndex(indexId);
                    if (state == JobState.RUNNING) {
                        int tableReplicaNum = getTotalReplicaNumByIndexId(indexId);
                        int tableFinishedReplicaNum = getFinishedReplicaNumByIndexId(indexId);
                        Preconditions.checkState(!(tableReplicaNum == 0 && tableFinishedReplicaNum == -1));
                        Preconditions.checkState(tableFinishedReplicaNum <= tableReplicaNum,
                                tableFinishedReplicaNum + "/" + tableReplicaNum);
                        totalReplicaNum += tableReplicaNum;
                        finishedReplicaNum += tableFinishedReplicaNum;
                    }

                    if (index.getState() != IndexState.NORMAL) {
                        idxState = index.getState().name();
                    }
                }

                indexState.put(indexId, idxState);

                if (Catalog.getInstance().isMaster() && state == JobState.RUNNING && totalReplicaNum != 0) {
                    indexProgress.put(indexId, (finishedReplicaNum * 100 / totalReplicaNum) + "%");
                } else {
                    indexProgress.put(indexId, "0%");
                }
            }
        }

        for (Long indexId : changedIndexIdToSchemaVersion.keySet()) {
            List<Comparable> jobInfo = new ArrayList<Comparable>();

            jobInfo.add(tableId);
            jobInfo.add(tbl.getName());
            jobInfo.add(TimeUtils.longToTimeString(createTime));
            jobInfo.add(TimeUtils.longToTimeString(finishedTime));
            jobInfo.add(tbl.getIndexNameById(indexId) == null ? "N/A" : tbl.getIndexNameById(indexId)); // index name
            jobInfo.add(indexId);
            // index schema version and schema hash
            jobInfo.add(changedIndexIdToSchemaVersion.get(indexId) + "-" + changedIndexIdToSchemaHash.get(indexId));
            jobInfo.add(indexState.get(indexId)); // index state
            jobInfo.add(transactionId);
            jobInfo.add(state.name()); // job state

            if (state == JobState.RUNNING) {
                jobInfo.add(indexProgress.get(indexId) == null ? "N/A" : indexProgress.get(indexId)); // progress
            } else {
                jobInfo.add("N/A");
            }

            jobInfo.add(cancelMsg);

            jobInfos.add(jobInfo);
        } // end for indexIds
    }

    @Override
    public synchronized void write(DataOutput out) throws IOException {
        super.write(out);

        Text.writeString(out, tableName);

        // 'unfinishedReplicaIds', 'indexIdToTotalReplicaNum' and 'indexIdToFinishedReplicaNum'
        // don't need persist. build it when send tasks

        // columns
        if (changedIndexIdToSchema != null) {
            out.writeBoolean(true);
            out.writeInt(changedIndexIdToSchema.size());
            for (Entry<Long, List<Column>> entry : changedIndexIdToSchema.entrySet()) {
                out.writeLong(entry.getKey());
                out.writeInt(entry.getValue().size());
                for (Column column : entry.getValue()) {
                    column.write(out);
                }
            }
        } else {
            out.writeBoolean(false);
        }

        // schema version and hash, and short key
        if (changedIndexIdToSchemaVersion != null) {
            out.writeBoolean(true);
            out.writeInt(changedIndexIdToSchemaVersion.size());
            for (Entry<Long, Integer> entry : changedIndexIdToSchemaVersion.entrySet()) {
                out.writeLong(entry.getKey());
                // schema version
                out.writeInt(entry.getValue());
                // schema hash
                out.writeInt(changedIndexIdToSchemaHash.get(entry.getKey()));
                // short key column count
                out.writeShort(changedIndexIdToShortKeyColumnCount.get(entry.getKey()));
            }
        }

        // replicaInfos is saving for restoring schemaChangeJobFinished
        if (replicaInfos != null) {
            out.writeBoolean(true);
            out.writeInt(replicaInfos.keySet().size());
            for (long partitionId : replicaInfos.keySet()) {
                Collection<ReplicaPersistInfo> infos = replicaInfos.get(partitionId);
                out.writeLong(partitionId);
                out.writeInt(infos.size());
                for (ReplicaPersistInfo replicaPersistInfo : infos) {
                    replicaPersistInfo.write(out);
                }
            }
        } else {
            out.writeBoolean(false);
        }

        // bloom filter columns
        out.writeBoolean(hasBfChange);
        if (bfColumns != null) {
            out.writeBoolean(true);
            out.writeInt(bfColumns.size());
            for (String bfColumn : bfColumns) {
                Text.writeString(out, bfColumn);
            }
            out.writeDouble(bfFpp);
        } else {
            out.writeBoolean(false);
        }

        // storage type
        if (newStorageType == null) {
            out.writeBoolean(false);
        } else {
            out.writeBoolean(true);
        }
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        super.readFields(in);

        tableName = Text.readString(in);

        if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_48) {
            if (in.readBoolean()) {
                int count = in.readInt();
                for (int i = 0; i < count; i++) {
                    long indexId = in.readLong();
                    int columnNum = in.readInt();
                    List<Column> columns = new LinkedList<Column>();
                    for (int j = 0; j < columnNum; j++) {
                        Column column = Column.read(in);
                        columns.add(column);
                    }
                    changedIndexIdToSchema.put(indexId, columns);
                    // schema version
                    changedIndexIdToSchemaVersion.put(indexId, in.readInt());
                    // schema hash
                    changedIndexIdToSchemaHash.put(indexId, in.readInt());
                    // short key column count
                    changedIndexIdToShortKeyColumnCount.put(indexId, in.readShort());
                }
            }
        } else {
            // columns
            if (in.readBoolean()) {
                int count = in.readInt();
                for (int i = 0; i < count; i++) {
                    long indexId = in.readLong();
                    int columnNum = in.readInt();
                    List<Column> columns = new LinkedList<Column>();
                    for (int j = 0; j < columnNum; j++) {
                        Column column = Column.read(in);
                        columns.add(column);
                    }
                    changedIndexIdToSchema.put(indexId, columns);
                }
            }

            // schema version and hash, and short key
            if (in.readBoolean()) {
                int count = in.readInt();
                for (int i = 0; i < count; i++) {
                    long indexId = in.readLong();
                    // schema version
                    changedIndexIdToSchemaVersion.put(indexId, in.readInt());
                    // schema hash
                    changedIndexIdToSchemaHash.put(indexId, in.readInt());
                    // short key column count
                    changedIndexIdToShortKeyColumnCount.put(indexId, in.readShort());
                }
            }
        }

        if (in.readBoolean()) {
            int count = in.readInt();
            for (int i = 0; i < count; ++i) {
                long partitionId = in.readLong();
                int infoSize = in.readInt();
                for (int j = 0; j < infoSize; j++) {
                    ReplicaPersistInfo info = ReplicaPersistInfo.read(in);
                    replicaInfos.put(partitionId, info);
                }
            }
        }

        // bloom filter columns
        if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_9) {
            hasBfChange = in.readBoolean();
            if (in.readBoolean()) {
                int bfColumnCount = in.readInt();
                bfColumns = Sets.newHashSet();
                for (int i = 0; i < bfColumnCount; i++) {
                    bfColumns.add(Text.readString(in));
                }

                bfFpp = in.readDouble();
            }
        }

        if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_39) {
            if (in.readBoolean()) {
                newStorageType = TStorageType.COLUMN;
            }
        }
    }

    public static SchemaChangeJob read(DataInput in) throws IOException {
        SchemaChangeJob schemaChangeJob = new SchemaChangeJob();
        schemaChangeJob.readFields(in);
        return schemaChangeJob;
    }

    public boolean equals(Object obj) {
        return true;
    }

    @Override
    public String toString() {
        return "SchemaChangeJob [tableName=" + tableName + ", type=" + type + ", state=" + state + ", dbId=" + dbId
                + ", tableId=" + tableId + ", transactionId=" + transactionId + ", isPreviousLoadFinished="
                + isPreviousLoadFinished + ", createTime=" + createTime + ", finishedTime=" + finishedTime + "]";
    }
}


text:
package tests.javax_servlet_http.HttpServletResponse;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpServlet;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import java.io.IOException;
import java.io.PrintWriter;

/**
 *	A Test for sendError(int) method
 */

public class SendErrorTestServlet extends HttpServlet {

    public void service ( HttpServletRequest request, HttpServletResponse response ) throws ServletException, IOException {

        PrintWriter out = response.getWriter();
        response.sendError( HttpServletResponse.SC_GONE );
    }
}

redpajama_set_name: RedPajamaGithub


text:
<!DOCTYPE html>

        <meta charset="ISO-8859-1">
        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
        <title></title>
        <meta name="description" content="">
        <meta name="viewport" content="width=device-width">

        <link rel="stylesheet" href="css/normalize.min.css">
        <link rel="stylesheet" href="css/main.css">

GUATEMALA MEMORIA DEL SILENCIO

inicio

informe

article header h1

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam sodales urna non odio egestas tempor. Nunc vel vehicula ante. Etiam bibendum iaculis libero, eget molestie nisl pharetra in. In semper consequat est, eu porta velit mollis nec.

article section h2

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam sodales urna non odio egestas tempor. Nunc vel vehicula ante. Etiam bibendum iaculis libero, eget molestie nisl pharetra in. In semper consequat est, eu porta velit mollis nec. Curabitur posuere enim eget turpis feugiat tempor. Etiam ullamcorper lorem dapibus velit suscipit ultrices. Proin in est sed erat facilisis pharetra.

article section h2

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam sodales urna non odio egestas tempor. Nunc vel vehicula ante. Etiam bibendum iaculis libero, eget molestie nisl pharetra in. In semper consequat est, eu porta velit mollis nec. Curabitur posuere enim eget turpis feugiat tempor. Etiam ullamcorper lorem dapibus velit suscipit ultrices. Proin in est sed erat facilisis pharetra.

article footer h3

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam sodales urna non odio egestas tempor. Nunc vel vehicula ante. Etiam bibendum iaculis libero, eget molestie nisl pharetra in. In semper consequat est, eu porta velit mollis nec. Curabitur posuere enim eget turpis feugiat tempor.

                <searchbox>

                    <gcse:search></gcse:search>
                </searchbox>

Información

El informe

Organizaciones guatemaltecas que contibuyen a la memoria colectiva

Recursos

Otros informes en América Latina

                Esta página está bajo la licencia MIT de código abierto.

                Acerca de este proyecto de difusión del informe "Memoria del Silencio".


text:
MODULE_big = pg_buffercache
OBJS	= pg_buffercache_pages.o

DATA_built = pg_buffercache.sql
DATA = uninstall_pg_buffercache.sql

ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
else
subdir = contrib/pg_buffercache
top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif


text:
#region Copyright & License

// Copyright © 2012 - 2017 François Chabot, Yves Dierick
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#endregion

using System;
using System.Diagnostics.CodeAnalysis;
using System.ServiceModel;
using Microsoft.BizTalk.Adapter.Wcf.Config;

namespace Be.Stateless.BizTalk.Dsl.Binding.Adapter
{
	public abstract partial class WcfWSHttpAdapter
	{
		#region Nested Type: Outbound

		/// <summary>
		/// You can use the WCF-WSHttp adapter to do cross-computer communication with services and clients that can
		/// understand the next-generation Web service standards, using either the HTTP or HTTPS transport with text or
		/// Message Transmission Optimization Mechanism (MTOM) encoding. The WCF-WSHttp adapter provides full access to
		/// the SOAP security, reliability, and transaction features.
		/// </summary>
		/// <seealso href="https://msdn.microsoft.com/en-us/library/bb245971.aspx">What Is the WCF-WSHttp Adapter?</seealso>
		/// <seealso href="https://msdn.microsoft.com/en-us/library/bb245939.aspx">How to Configure a WCF-WSHttp Send Port</seealso>
		/// <seealso href="https://msdn.microsoft.com/en-us/library/bb226397.aspx">WCF-WSHttp Transport Properties Dialog Box, Send, Security Tab</seealso>
		/// <seealso href="https://msdn.microsoft.com/en-us/library/bb245991.aspx">WCF Adapters Property Schema and Properties</seealso>.
		[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Global", Justification = "Public API")]
		public class Outbound
			: WcfWSHttpAdapter<EndpointAddress, WSHttpTLConfig>,
				IOutboundAdapter,
				IAdapterConfigClientCertificate,
				IAdapterConfigOutboundAction,
				IAdapterConfigOutboundCredentials,
				IAdapterConfigOutboundPropagateFaultMessage,
				IAdapterConfigProxySettings,
				IAdapterConfigProxyToUse
		{
			public Outbound()
			{
				// Proxy Tab - General Settings
				ProxyToUse = ProxySelection.None;

				// Messages Tab - Error Handling Settings
				PropagateFaultMessage = true;
			}

			public Outbound(Action<Outbound> adapterConfigurator) : this()
			{
				adapterConfigurator(this);
			}

			#region IAdapterConfigClientCertificate Members

			/// <summary>
			/// Specify the thumbprint of the X.509 certificate for authenticating this send port to services. This
			/// property is required if the <see cref="WcfWSHttpAdapter{TAddress,TConfig}.MessageClientCredentialType"/>
			/// property is set to <see cref="MessageCredentialType.Certificate"/>.
			/// </summary>
			/// <remarks>
			/// <para>
			/// The certificate to be used for this property must be installed into the My store in the Current User
			/// location of the user account for the send handler hosting this send port.
			/// </para>
			/// <para>
			/// It defaults to an <see cref="string.Empty"/> string.
			/// </para>
			/// </remarks>
			public string ClientCertificate
			{
				get { return _adapterConfig.ClientCertificate; }
				set { _adapterConfig.ClientCertificate = value; }
			}

			#endregion

			#region IAdapterConfigOutboundAction Members

			public string StaticAction
			{
				get { return _adapterConfig.StaticAction; }
				set { _adapterConfig.StaticAction = value; }
			}

			#endregion

			#region IAdapterConfigOutboundCredentials Members

			public bool UseSSO
			{
				get { return _adapterConfig.UseSSO; }
				set { _adapterConfig.UseSSO = value; }
			}

			public string AffiliateApplicationName
			{
				get { return _adapterConfig.AffiliateApplicationName; }
				set { _adapterConfig.AffiliateApplicationName = value; }
			}

			public string UserName
			{
				get { return _adapterConfig.UserName; }
				set { _adapterConfig.UserName = value; }
			}

			public string Password
			{
				get { return _adapterConfig.Password; }
				set { _adapterConfig.Password = value; }
			}

			#endregion

			#region IAdapterConfigOutboundPropagateFaultMessage Members

			public bool PropagateFaultMessage
			{
				get { return _adapterConfig.PropagateFaultMessage; }
				set { _adapterConfig.PropagateFaultMessage = value; }
			}

			#endregion

			#region IAdapterConfigProxySettings Members

			/// <summary>
			/// Specify the address of the proxy server.
			/// </summary>
			/// <remarks>
			/// <para>
			/// Use the https or the http scheme depending on the security configuration. This address can be followed by a
			/// colon and the port number. For example, <c>http://127.0.0.1:8080</c>.
			/// </para>
			/// <para>
			/// It defaults to an <see cref="string.Empty"/> string.
			/// </para>
			/// </remarks>
			public string ProxyAddress
			{
				get { return _adapterConfig.ProxyAddress; }
				set { _adapterConfig.ProxyAddress = value; }
			}

			/// <summary>
			/// Specify the user name to use for the proxy.
			/// </summary>
			/// <remarks>
			/// <para>
			/// The WCF-WSHttp adapter leverages the <see cref="WSHttpBinding"/> in the buffered transfer mode to
			/// communicate with an endpoint. Proxy credentials of <see cref="WSHttpBinding"/> are applicable only when the
			/// <see cref="WcfWSHttpAdapter{TAddress,TConfig}.SecurityMode"/> is <see cref="SecurityMode.Transport"/> or
			/// <see cref="SecurityMode.None"/>. If you set the <see
			/// cref="WcfWSHttpAdapter{TAddress,TConfig}.SecurityMode"/> property to
			/// <see cref="SecurityMode.Message"/>
			/// or <see cref="SecurityMode.TransportWithMessageCredential"/>, the WCF-WSHttp adapter does not use the
			/// credential specified in the <see cref="ProxyUserName"/> and <see cref="ProxyPassword"/> properties for
			/// authentication against the proxy.
			/// </para>
			/// <para>
			/// The WCF-WSHttp send adapter uses Basic authentication for the proxy.
			/// </para>
			/// <para>
			/// It defaults to an <see cref="string.Empty"/> string.
			/// </para>
			/// </remarks>
			public string ProxyUserName
			{
				get { return _adapterConfig.ProxyUserName; }
				set { _adapterConfig.ProxyUserName = value; }
			}

			/// <summary>
			/// Specify the password to use for the proxy.
			/// </summary>
			/// <remarks>
			/// It defaults to an <see cref="string.Empty"/> string.
			/// </remarks>
			public string ProxyPassword
			{
				get { return _adapterConfig.ProxyPassword; }
				set { _adapterConfig.ProxyPassword = value; }
			}

			#endregion

			#region IAdapterConfigProxyToUse Members

			/// <summary>
			/// Specify which proxy server to use for outgoing HTTP traffic.
			/// </summary>
			/// <remarks>
			/// <list type="bullet">
			/// <item>
			/// <see cref="ProxySelection.None"/> — Do not use a proxy server for this send port.
			/// </item>
			/// <item>
			/// <see cref="ProxySelection.Default"/> — Use the proxy settings in the send handler hosting this send
			/// port.
			/// </item>
			/// <item>
			/// <see cref="ProxySelection.UserSpecified"/> — Use the proxy server specified in the <see cref="ProxyAddress"/>
			/// property.
			/// </item>
			/// </list>
			/// It defaults to <see cref="ProxySelection.None"/>.
			/// </remarks>
			public ProxySelection ProxyToUse
			{
				get { return _adapterConfig.ProxyToUse; }
				set { _adapterConfig.ProxyToUse = value; }
			}

			#endregion
		}

		#endregion
	}
}


text:
var utils = require('./_utils'),
  fs = require('fs'),
  uglify = require('uglify-js')

module.exports = function(esskOptions) {

    return function () {

        var jsSourcePath = esskOptions.jsDest + path.sep + esskOptions.library + '.js',
            jsOutputPath = esskOptions.jsDest + path.sep + esskOptions.library + '.min.js',
            outputJS = uglify.minify(jsSourcePath)

        /**
         * Create a promise based on the result of the uglify output
         */
        return new Promise(function (resolve, reject) {
            // write the result of the uglify script in the output folder
            fs.writeFile(jsOutputPath, outputJS.code, function (err) {
                if (err) {
                    utils.print(err, 'error')
                    reject(err)
                } else {
                    utils.print('JS minified', 'confirm')
                    utils.print('Created file: ' + jsOutputPath , 'cool')
                    resolve()
                }
            })
        })

    }
}


text:
**Purpose**

Compare keys of 2 arrays recursively in both directions.
Can be used for comparing documented JSON response of an API versus the actual response.

**Usage**

Run `test/test.php` to see it in action.


text:
from time import time

from numpy import pi
from numpy import array
from numpy.random import random
from numpy.random import randint

from numpy import linspace
from numpy import arange
from numpy import column_stack
from numpy import cos
from numpy import sin

import cairocffi as cairo
from sand import Sand
from ..lib.sand_spline import SandSpline
from ..lib.helpers import hex_to_rgb_decimal, SimpleLinearScale, get_colors, _rnd_interpolate

def guide_iterator(x, y):
    while True:
        yield array([[x, y]])

def generate(args):
    width = args.width
    height = args.height

    xscale = SimpleLinearScale(domain=array([0, width]), range=array([0, 1]))
    yscale = SimpleLinearScale(domain=array([0, height]), range=array([0, 1]))

    # Margin as a pixel value of total size.  Convert that margin to a number between 0..1
    # representing the percentage of total pixel size
    margin = args.margin
    margin_x = xscale(margin)
    margin_y = yscale(margin)

    # Output PNG gamma
    gamma = 1.5

    # What frame to write out
    save_frame = args.save_every

    # TODO: Step.  Appears to be jitter multiplier for points along the spline
    # Causes the sand to be more "windswept" towards the later points
    step = 0.0000003 * 0.15

    # The number of points along the spline.  More points means a denser-looking spline.
    point_count = 1000

    # Convert colors to RGB decimal
    sand_color = hex_to_rgb_decimal(args.color)
    bg_color = hex_to_rgb_decimal(args.bg_color)

    # Set alpha
    sand_color.append(0.1)
    bg_color.append(1)

    sand = Sand(width, height)
    sand.set_rgba(sand_color)
    sand.set_bg(bg_color)

    for i in range(0, 100):
        num_points = 1500
        angle = random() * (pi * 2) + linspace(0, (pi * 2), num_points)
        points = column_stack((cos(angle), sin(angle))) * (1.0 * random() * 0.4)
        path = array([[0.5, 0.5]]) + _rnd_interpolate(points, 1000, ordered=True)

        sand.paint_dots(path)

    file_name = '{}/{}.png'.format(
        args.out_dir,
        int(time()))

    sand.write_to_png(file_name)


text:
![testIT WebTester](images/logo-650x157.png)

## Welcome to the testIT WebTester documentation!

- [Getting Started](chapters/getting-started.md)
- [Browser](chapters/browser.md)
- [Configuration](chapters/configuration.md)
- [Event System](chapters/event-system.md)
- [Page Objects](chapters/page-object.md)
  - [Anatomy of Pages](chapters/page-object-anatomy.md)
  - [Functional Page Objects](chapters/page-object-functional.md)
  - [Generic Element](chapters/page-object-generic.md)
  - [Page Object Factory](chapters/page-object-factory.md)
- Utility Classes
  - [Conditions](chapters/conditions.md)
  - [Identifications](chapters/identifications.md)
  - [Mouse](chapters/mouse.md)
  - [Waits](chapters/waits.md)
- Support Modules
  - Assertions
    - [AssertJ](chapters/support-assertj.md)
    - [Hamcrest](chapters/support-hamcrest.md)
  - Browser
    - [Chrome](chapters/support-chrome.md)
    - [Firefox](chapters/support-firefox.md)
    - [Internet Explorer](chapters/support-ie.md)
  - Testing
    - [JUnit](chapters/support-junit.md)
    - [TestNG](chapters/support-testng.md)
  - Integration
    - [Spring 4](chapters/support-spring4.md)
- Advanced Topics
  - [JavaScript and Ajax Handling](chapters/javascript-ajax.md)
  - [Ad-Hoc finding of Page Objects](chapters/ad-hoc-find.md)


text:
Class: Social Problems

Year: Summer 2015

Team: Miltiadis Pistiolas, Nick Allevato, Gloria Zeng, Grant Giller

Theme Based on: [Grayscale](http://startbootstrap.com/template-overviews/grayscale/)

## Copyright and License

Copyright 2013-2015 Iron Summit Media Strategies, LLC. Code released under the [Apache 2.0](https://github.com/IronSummitMedia/startbootstrap-grayscale/blob/gh-pages/LICENSE) license.


text:
Node.js BLE Communication Connector Helper Module

This can be used to communicate with [PSoC4BLEComm](https://github.com/perusworld/PSoC4BLEComm "PSOC4 BLE Communication Library")


text:
{{define "title"}}Command Help{{end}}
{{define "scripts"}}
<link rel="stylesheet" href="/static/dialog-polyfill.css">

{{end}}
{{define "content"}}

Command help for

<select id="repo" onchange="redraw();"><option>all commands</option></select>

      Command
      Example
      Description
      Who Can Use
      Plugin

{{end}}
{{define "extra content"}}
<dialog id="dialog" class="mdl-dialog">

    <button type="button" class="mdl-button close">Close</button>

</dialog>
{{end}}

{{template "page" (settings mobileUnfriendly lightMode "command-help" .)}}


text:
{-
HAAP: Haskell Automated Assessment Platform

This module provides wrappers to the _tagsoup_ library (<https://hackage.haskell.org/package/tagsoup>) for unstructured processing of HTML/XML documents.
-}

{-# LANGUAGE FlexibleContexts #-}

module HAAP.Web.HTML.TagSoup where

import HAAP.Core
import HAAP.IO
import HAAP.Plugin
import HAAP.Core

import Text.HTML.TagSoup

import Data.Default
import Data.Generics

--import Debug.Trace

import Control.Monad.IO.Class

type TagHtml = [Tag String]

parseTagSoupHTML :: (HaapStack t m,MonadIO m) => FilePath -> Haap t m TagHtml
parseTagSoupHTML file = do
    str <- runBaseIO' $ readFile file
    return $ parseTags str

asTagSoupHTML :: (TagHtml -> TagHtml) -> String -> String
asTagSoupHTML f str = renderTags $ f $ parseTags str

tagSoupChangeLinkUrls :: (String -> String) -> TagHtml -> TagHtml
tagSoupChangeLinkUrls furl = map felem
    where
    felem :: Tag String -> Tag String
    felem (TagOpen x atts) = TagOpen x $ map fatt atts
    felem i = i
    fatt :: Attribute String -> Attribute String
    fatt ("href",url) = ("href",furl url)
    fatt att = att

injectHTMLHead :: TagHtml -> TagHtml -> TagHtml
injectHTMLHead newhead = aux
    where
    aux [] = []
    aux (tag@(TagOpen "head" atts):tags) = tag : newhead ++ tags
    aux (x:xs) = x : aux xs

injectHTMLBody :: TagHtml -> TagHtml -> TagHtml
injectHTMLBody newbody = aux
    where
    aux [] = []
    aux (tag@(TagOpen "body" atts):tags) = tag : newbody ++ tags
    aux (x:xs) = x : aux xs


text:
using ::testing::AssertionResult;
using std::string;
using base::StringPiece;

namespace net {
namespace test {
namespace {

const HpackEntryType kInvalidHeaderType = static_cast<HpackEntryType>(99);
const size_t kInvalidIndex = 99999999;

}  // namespace

HpackEntryCollector::HpackEntryCollector() {
  Clear();
}

HpackEntryCollector::HpackEntryCollector(const HpackEntryCollector& other)
    : header_type_(other.header_type_),
      index_(other.index_),
      name_(other.name_),
      value_(other.value_),
      started_(other.started_),
      ended_(other.ended_) {}

HpackEntryCollector::HpackEntryCollector(HpackEntryType type,
                                         size_t index_or_size)
    : header_type_(type), index_(index_or_size), started_(true), ended_(true) {}
HpackEntryCollector::HpackEntryCollector(HpackEntryType type,
                                         size_t index,
                                         bool value_huffman,
                                         const string& value)
    : header_type_(type),
      index_(index),
      value_(value, value_huffman),
      started_(true),
      ended_(true) {}
HpackEntryCollector::HpackEntryCollector(HpackEntryType type,
                                         bool name_huffman,
                                         const string& name,
                                         bool value_huffman,
                                         const string& value)
    : header_type_(type),
      index_(0),
      name_(name, name_huffman),
      value_(value, value_huffman),
      started_(true),
      ended_(true) {}

HpackEntryCollector::~HpackEntryCollector() {}

void HpackEntryCollector::OnIndexedHeader(size_t index) {
  ASSERT_FALSE(started_);
  ASSERT_TRUE(IsClear()) << ToString();
  Init(HpackEntryType::kIndexedHeader, index);
  ended_ = true;
}
void HpackEntryCollector::OnStartLiteralHeader(HpackEntryType header_type,
                                               size_t maybe_name_index) {
  ASSERT_FALSE(started_);
  ASSERT_TRUE(IsClear()) << ToString();
  Init(header_type, maybe_name_index);
}
void HpackEntryCollector::OnNameStart(bool huffman_encoded, size_t len) {
  ASSERT_TRUE(started_);
  ASSERT_FALSE(ended_);
  ASSERT_FALSE(IsClear());
  ASSERT_TRUE(LiteralNameExpected()) << ToString();
  name_.OnStringStart(huffman_encoded, len);
}
void HpackEntryCollector::OnNameData(const char* data, size_t len) {
  ASSERT_TRUE(started_);
  ASSERT_FALSE(ended_);
  ASSERT_TRUE(LiteralNameExpected()) << ToString();
  ASSERT_TRUE(name_.IsInProgress());
  name_.OnStringData(data, len);
}
void HpackEntryCollector::OnNameEnd() {
  ASSERT_TRUE(started_);
  ASSERT_FALSE(ended_);
  ASSERT_TRUE(LiteralNameExpected()) << ToString();
  ASSERT_TRUE(name_.IsInProgress());
  name_.OnStringEnd();
}
void HpackEntryCollector::OnValueStart(bool huffman_encoded, size_t len) {
  ASSERT_TRUE(started_);
  ASSERT_FALSE(ended_);
  if (LiteralNameExpected()) {
    ASSERT_TRUE(name_.HasEnded());
  }
  ASSERT_TRUE(LiteralValueExpected()) << ToString();
  ASSERT_TRUE(value_.IsClear()) << value_.ToString();
  value_.OnStringStart(huffman_encoded, len);
}
void HpackEntryCollector::OnValueData(const char* data, size_t len) {
  ASSERT_TRUE(started_);
  ASSERT_FALSE(ended_);
  ASSERT_TRUE(LiteralValueExpected()) << ToString();
  ASSERT_TRUE(value_.IsInProgress());
  value_.OnStringData(data, len);
}
void HpackEntryCollector::OnValueEnd() {
  ASSERT_TRUE(started_);
  ASSERT_FALSE(ended_);
  ASSERT_TRUE(LiteralValueExpected()) << ToString();
  ASSERT_TRUE(value_.IsInProgress());
  value_.OnStringEnd();
  ended_ = true;
}
void HpackEntryCollector::OnDynamicTableSizeUpdate(size_t size) {
  ASSERT_FALSE(started_);
  ASSERT_TRUE(IsClear()) << ToString();
  Init(HpackEntryType::kDynamicTableSizeUpdate, size);
  ended_ = true;
}

void HpackEntryCollector::Clear() {
  header_type_ = kInvalidHeaderType;
  index_ = kInvalidIndex;
  name_.Clear();
  value_.Clear();
  started_ = ended_ = false;
}
bool HpackEntryCollector::IsClear() const {
  return header_type_ == kInvalidHeaderType && index_ == kInvalidIndex &&
         name_.IsClear() && value_.IsClear() && !started_ && !ended_;
}
bool HpackEntryCollector::IsComplete() const {
  return started_ && ended_;
}
bool HpackEntryCollector::LiteralNameExpected() const {
  switch (header_type_) {
    case HpackEntryType::kIndexedLiteralHeader:
    case HpackEntryType::kUnindexedLiteralHeader:
    case HpackEntryType::kNeverIndexedLiteralHeader:
      return index_ == 0;
    default:
      return false;
  }
}
bool HpackEntryCollector::LiteralValueExpected() const {
  switch (header_type_) {
    case HpackEntryType::kIndexedLiteralHeader:
    case HpackEntryType::kUnindexedLiteralHeader:
    case HpackEntryType::kNeverIndexedLiteralHeader:
      return true;
    default:
      return false;
  }
}
AssertionResult HpackEntryCollector::ValidateIndexedHeader(
    size_t expected_index) const {
  VERIFY_TRUE(started_);
  VERIFY_TRUE(ended_);
  VERIFY_EQ(HpackEntryType::kIndexedHeader, header_type_);
  VERIFY_EQ(expected_index, index_);
  return ::testing::AssertionSuccess();
}
AssertionResult HpackEntryCollector::ValidateLiteralValueHeader(
    HpackEntryType expected_type,
    size_t expected_index,
    bool expected_value_huffman,
    StringPiece expected_value) const {
  VERIFY_TRUE(started_);
  VERIFY_TRUE(ended_);
  VERIFY_EQ(expected_type, header_type_);
  VERIFY_NE(0u, expected_index);
  VERIFY_EQ(expected_index, index_);
  VERIFY_TRUE(name_.IsClear());
  VERIFY_SUCCESS(value_.Collected(expected_value, expected_value_huffman));
  return ::testing::AssertionSuccess();
}
AssertionResult HpackEntryCollector::ValidateLiteralNameValueHeader(
    HpackEntryType expected_type,
    bool expected_name_huffman,
    StringPiece expected_name,
    bool expected_value_huffman,
    StringPiece expected_value) const {
  VERIFY_TRUE(started_);
  VERIFY_TRUE(ended_);
  VERIFY_EQ(expected_type, header_type_);
  VERIFY_EQ(0u, index_);
  VERIFY_SUCCESS(name_.Collected(expected_name, expected_name_huffman));
  VERIFY_SUCCESS(value_.Collected(expected_value, expected_value_huffman));
  return ::testing::AssertionSuccess();
}
AssertionResult HpackEntryCollector::ValidateDynamicTableSizeUpdate(
    size_t size) const {
  VERIFY_TRUE(started_);
  VERIFY_TRUE(ended_);
  VERIFY_EQ(HpackEntryType::kDynamicTableSizeUpdate, header_type_);
  VERIFY_EQ(index_, size);
  return ::testing::AssertionSuccess();
}

void HpackEntryCollector::AppendToHpackBlockBuilder(
    HpackBlockBuilder* hbb) const {
  ASSERT_TRUE(started_ && ended_) << *this;
  switch (header_type_) {
    case HpackEntryType::kIndexedHeader:
      hbb->AppendIndexedHeader(index_);
      return;

    case HpackEntryType::kDynamicTableSizeUpdate:
      hbb->AppendDynamicTableSizeUpdate(index_);
      return;

    case HpackEntryType::kIndexedLiteralHeader:
    case HpackEntryType::kUnindexedLiteralHeader:
    case HpackEntryType::kNeverIndexedLiteralHeader:
      ASSERT_TRUE(value_.HasEnded()) << *this;
      if (index_ != 0) {
        CHECK(name_.IsClear());
        hbb->AppendNameIndexAndLiteralValue(header_type_, index_,
                                            value_.huffman_encoded, value_.s);
      } else {
        CHECK(name_.HasEnded()) << *this;
        hbb->AppendLiteralNameAndValue(header_type_, name_.huffman_encoded,
                                       name_.s, value_.huffman_encoded,
                                       value_.s);
      }
      return;

    default:
      ADD_FAILURE() << *this;
  }
}

string HpackEntryCollector::ToString() const {
  string result("Type=");
  switch (header_type_) {
    case HpackEntryType::kIndexedHeader:
      result += "IndexedHeader";
      break;
    case HpackEntryType::kDynamicTableSizeUpdate:
      result += "DynamicTableSizeUpdate";
      break;
    case HpackEntryType::kIndexedLiteralHeader:
      result += "IndexedLiteralHeader";
      break;
    case HpackEntryType::kUnindexedLiteralHeader:
      result += "UnindexedLiteralHeader";
      break;
    case HpackEntryType::kNeverIndexedLiteralHeader:
      result += "NeverIndexedLiteralHeader";
      break;
    default:
      if (header_type_ == kInvalidHeaderType) {
        result += "<unset>";
      } else {
        std::stringstream ss;
        ss << header_type_;
        result.append(ss.str());
      }
  }
  if (index_ != 0) {
    result.append(" Index=");
    std::stringstream ss;
    ss << index_;
    result.append(ss.str());
  }
  if (!name_.IsClear()) {
    result.append(" Name");
    result.append(name_.ToString());
  }
  if (!value_.IsClear()) {
    result.append(" Value");
    result.append(value_.ToString());
  }
  if (!started_) {
    EXPECT_FALSE(ended_);
    result.append(" !started");
  } else if (!ended_) {
    result.append(" !ended");
  } else {
    result.append(" Complete");
  }
  return result;
}

void HpackEntryCollector::Init(HpackEntryType type, size_t maybe_index) {
  ASSERT_TRUE(IsClear()) << ToString();
  header_type_ = type;
  index_ = maybe_index;
  started_ = true;
}

bool operator==(const HpackEntryCollector& a, const HpackEntryCollector& b) {
  return a.name() == b.name() && a.value() == b.value() &&
         a.index() == b.index() && a.header_type() == b.header_type() &&
         a.started() == b.started() && a.ended() == b.ended();
}
bool operator!=(const HpackEntryCollector& a, const HpackEntryCollector& b) {
  return !(a == b);
}

std::ostream& operator<<(std::ostream& out, const HpackEntryCollector& v) {
  return out << v.ToString();
}

}  // namespace test
}  // namespace net


text:
import os
import time
import fuse
import errno
from .item import RelFuseItem
#------------------------------------------------------------------------------#
class FileView(RelFuseItem):
    # --------------------------------------------------------------------------
    def __init__(self, path_getter):
        RelFuseItem.__init__(self)
        self._path_getter = path_getter
        self._fd = None

    # --------------------------------------------------------------------------
    def _get_mode(self):
        return 0o100440

    # --------------------------------------------------------------------------
    def _get_size(self):
        if not self._fd:
            return os.stat(self._path_getter()).st_size
        return os.fstat(self._fd).st_size

    # --------------------------------------------------------------------------
    def open(self, flags):
        if flags & (os.O_CREAT | os.O_WRONLY | os.O_APPEND | os.O_TRUNC):
            raise fuse.FuseOSError(errno.EACCES)

        self._fd = os.open(self._path_getter(), (os.O_RDONLY | flags))
        return self._fd

    # --------------------------------------------------------------------------
    def read(self, length, offset):
        if not self._fd:
            raise fuse.FuseOSError(errno.EBADFD)

        os.lseek(self._fd, offset, os.SEEK_SET)
        return os.read(self._fd, length)

    # --------------------------------------------------------------------------
    def release(self):
        fd = self._fd
        self._fd = None
        return os.close(fd)

#------------------------------------------------------------------------------#


text:
<?xml version="1.0" encoding="UTF-8"?>
<extension type="library" version="2.5" method="upgrade">
	<name>FOF</name>
	<libraryname>fof</libraryname>
	<description>LIB_FOF_XML_DESCRIPTION</description>
	<creationDate>2013-12-14</creationDate>
	<author>Nicholas K. Dionysopoulos / Akeeba Ltd</author>
	<authorEmail>nicholas@akeebabackup.com</authorEmail>
	<authorUrl>https://www.akeebabackup.com</authorUrl>
	<copyright>(C)2011-2013 Nicholas K. Dionysopoulos</copyright>
	<license>GNU GPLv2 or later</license>
	<version>2.1.1</version>
	<packager>Akeeba Ltd</packager>
	<packagerurl>https://www.AkeebaBackup.com/download.html</packagerurl>

	<files folder="fof">
		<folder>autoloader</folder>
		<folder>config</folder>
		<folder>controller</folder>
		<folder>dispatcher</folder>
		<folder>encrypt</folder>
		<folder>form</folder>
		<folder>hal</folder>
		<folder>inflector</folder>
		<folder>input</folder>
		<folder>less</folder>
		<folder>model</folder>
		<folder>platform</folder>
		<folder>query</folder>
		<folder>render</folder>
		<folder>string</folder>
		<folder>table</folder>
		<folder>template</folder>
		<folder>toolbar</folder>
		<folder>view</folder>

		<file>LICENSE.txt</file>
		<file>include.php</file>
		<file>index.html</file>
		<file>version.txt</file>
	</files>
</extension>


text:
package com.github.libsml.math.linalg

/**
 * Created by huangyu on 15/9/7.
 */
object VectorType extends Enumeration {

  type VectorType = Value
  val DENSE, SPARSE, MAP = Value
}


text:
package org.jboss.weld.tests.producer.alternative;

import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.inject.Produces;

@ApplicationScoped
public class Bar {

    @Produces
    @ApplicationScoped
    public Foo createFoo() {
        return new Foo("produced");
    }
}


text:
object Sequencing9 {

  case class Box(var value: Int)

  sealed trait OptionMut {
    def isEmpty: Boolean = this match {
      case SomeMut(_) => false
      case NoneMut() => true
    }

    def head: Box = {
      require(!isEmpty)
      val SomeMut(v) = this
      v
    }

    def filter(p: Int => Boolean): OptionMut = {
      // Here, the normalizer phase of AntiAliasing will
      // introduce an extra binding to this.head (due to being of a mutable type)
      // However, that binding may not appear before the !isEmpty condition, because
      // && is short-circuiting.
      // That is, the following normalization would be incorrect:
      //   val targetBound = this.head // Not guarded by !isEmpty anymore!
      //   if (!isEmpty && p(targetBound.value)) SomeMut(head)
      // while the following is correct:
      //   if (!isEmpty && {val targetBound = this.head; p(targetBound.value)}) SomeMut(head)
      if (!isEmpty && p(head.value)) SomeMut(head)
      else NoneMut()
    }
  }

  case class SomeMut(var box: Box) extends OptionMut
  case class NoneMut() extends OptionMut
}


text:
/**
* @author Vera Y. Petrashkova
* @version $Revision$
*/

package java.security.interfaces;

import java.math.BigInteger;
import java.security.PrivateKey;

/**
 * @com.intel.drl.spec_ref
 *
 */
public interface ECPrivateKey extends PrivateKey, ECKey {
    /**
     * @com.intel.drl.spec_ref
     */
    public static final long serialVersionUID = -7896394956925609184L;

    /**
     * @com.intel.drl.spec_ref
     */
    public BigInteger getS();
}


text:
Please check [issue](https://github.com/krists/refile-postgres/issues/9) for more details.

1) Change Rails schema dump format to SQL in your `config/application.rb` file
   ```ruby
   # Use structure.sql instead of schema.rb
   config.active_record.schema_format = :sql
   ```
2) Change version number in Gemfile
   ```ruby
   gem 'refile-postgres', '~> 1.3.0'
   ```
3) Create Rails migration
   ```
   rails g migration refile_postgres_migration_to_1_3_0
   ```
4) Add content to migration
   ```ruby
   class RefilePostgresMigrationTo130 < ActiveRecord::Migration
     def up
       execute <<-SQL
         DROP INDEX index_refile_attachments_on_namespace;
         ALTER TABLE refile_attachments RENAME TO old_refile_attachments;
         ALTER TABLE ONLY old_refile_attachments RENAME CONSTRAINT refile_attachments_pkey TO old_refile_attachments_pkey;
         CREATE TABLE refile_attachments (
           id integer NOT NULL,
           oid oid NOT NULL,
           namespace character varying NOT NULL,
           created_at timestamp without time zone DEFAULT ('now'::text)::timestamp without time zone
         );
         ALTER TABLE ONLY refile_attachments ADD CONSTRAINT refile_attachments_pkey PRIMARY KEY (id);
         ALTER SEQUENCE refile_attachments_id_seq RESTART OWNED BY refile_attachments.id;
         ALTER TABLE ONLY refile_attachments ALTER COLUMN id SET DEFAULT nextval('refile_attachments_id_seq'::regclass);
         INSERT INTO refile_attachments (oid, namespace) SELECT id, namespace FROM old_refile_attachments;
         CREATE INDEX index_refile_attachments_on_namespace ON refile_attachments USING btree (namespace);
         CREATE INDEX index_refile_attachments_on_oid ON refile_attachments USING btree (oid);
         DROP TABLE old_refile_attachments;
       SQL
     end

     def down
       raise ActiveRecord::IrreversibleMigration
     end
   end
   ```
5) Now it is safe to run [vacuumlo](http://www.postgresql.org/docs/9.5/static/vacuumlo.html)