text:
package org.elasticsearch.snapshots;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.lucene.util.CollectionUtil;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.*;
import org.elasticsearch.cluster.metadata.*;
import org.elasticsearch.cluster.metadata.SnapshotMetaData.ShardSnapshotStatus;
import org.elasticsearch.cluster.metadata.SnapshotMetaData.State;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.snapshots.IndexShardSnapshotAndRestoreService;
import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.repositories.Repository;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;

import java.io.IOException;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;

import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Maps.newHashMapWithExpectedSize;
import static com.google.common.collect.Sets.newHashSet;

/**
 * Service responsible for creating snapshots
 *

 * A typical snapshot creating process looks like this:
 *

 *
On the master node the {@link #createSnapshot(SnapshotRequest, CreateSnapshotListener)} is called and makes sure that no snapshots is currently running
 * and registers the new snapshot in cluster state

 *
When cluster state is updated the {@link #beginSnapshot(ClusterState, SnapshotMetaData.Entry, boolean, CreateSnapshotListener)} method
 * kicks in and initializes the snapshot in the repository and then populates list of shards that needs to be snapshotted in cluster state

 *
Each data node is watching for these shards and when new shards scheduled for snapshotting appear in the cluster state, data nodes
 * start processing them through {@link #processIndexShardSnapshots(SnapshotMetaData)} method

 *
Once shard snapshot is created data node updates state of the shard in the cluster state using the {@link #updateIndexShardSnapshotStatus(UpdateIndexShardSnapshotStatusRequest)} method

 *
When last shard is completed master node in {@link #innerUpdateSnapshotState} method marks the snapshot as completed

 *
After cluster state is updated, the {@link #endSnapshot(SnapshotMetaData.Entry)} finalizes snapshot in the repository,
 * notifies all {@link #snapshotCompletionListeners} that snapshot is completed, and finally calls {@link #removeSnapshotFromClusterState(SnapshotId, SnapshotInfo, Throwable)} to remove snapshot from cluster state

 *

 */
public class SnapshotsService extends AbstractComponent implements ClusterStateListener {

    private final ClusterService clusterService;

    private final RepositoriesService repositoriesService;

    private final ThreadPool threadPool;

    private final IndicesService indicesService;

    private final TransportService transportService;

    private volatile ImmutableMap<SnapshotId, SnapshotShards> shardSnapshots = ImmutableMap.of();

    private final CopyOnWriteArrayList<SnapshotCompletionListener> snapshotCompletionListeners = new CopyOnWriteArrayList<SnapshotCompletionListener>();

    @Inject
    public SnapshotsService(Settings settings, ClusterService clusterService, RepositoriesService repositoriesService, ThreadPool threadPool,
                            IndicesService indicesService, TransportService transportService) {
        super(settings);
        this.clusterService = clusterService;
        this.repositoriesService = repositoriesService;
        this.threadPool = threadPool;
        this.indicesService = indicesService;
        this.transportService = transportService;

        transportService.registerHandler(UpdateSnapshotStateRequestHandler.ACTION, new UpdateSnapshotStateRequestHandler());

        // addLast to make sure that Repository will be created before snapshot
        clusterService.addLast(this);
    }

    /**
     * Retrieves snapshot from repository
     *
     * @param snapshotId snapshot id
     * @return snapshot
     * @throws SnapshotMissingException if snapshot is not found
     */
    public Snapshot snapshot(SnapshotId snapshotId) {
        return repositoriesService.repository(snapshotId.getRepository()).readSnapshot(snapshotId);
    }

    /**
     * Returns a list of snapshots from repository sorted by snapshot creation date
     *
     * @param repositoryName repository name
     * @return list of snapshots
     */
    public ImmutableList<Snapshot> snapshots(String repositoryName) {
        ArrayList<Snapshot> snapshotList = newArrayList();
        Repository repository = repositoriesService.repository(repositoryName);
        ImmutableList<SnapshotId> snapshotIds = repository.snapshots();
        for (SnapshotId snapshotId : snapshotIds) {
            snapshotList.add(repository.readSnapshot(snapshotId));
        }
        CollectionUtil.timSort(snapshotList);
        return ImmutableList.copyOf(snapshotList);
    }

    /**
     * Initializes the snapshotting process.
     *

     * This method is used by clients to start snapshot. It makes sure that there is no snapshots are currently running and
     * creates a snapshot record in cluster state metadata.
     *
     * @param request  snapshot request
     * @param listener snapshot creation listener
     */
    public void createSnapshot(final SnapshotRequest request, final CreateSnapshotListener listener) {
        final SnapshotId snapshotId = new SnapshotId(request.repository(), request.name());
        clusterService.submitStateUpdateTask(request.cause(), new TimeoutClusterStateUpdateTask() {

            private SnapshotMetaData.Entry newSnapshot = null;

            @Override
            public ClusterState execute(ClusterState currentState) {
                validate(request, currentState);

                MetaData metaData = currentState.metaData();
                MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                SnapshotMetaData snapshots = metaData.custom(SnapshotMetaData.TYPE);
                if (snapshots == null || snapshots.entries().isEmpty()) {
                    // Store newSnapshot here to be processed in clusterStateProcessed
                    ImmutableList<String> indices = ImmutableList.copyOf(metaData.concreteIndices(request.indices(), request.indicesOptions()));
                    logger.trace("[{}][{}] creating snapshot for indices [{}]", request.repository(), request.name(), indices);
                    newSnapshot = new SnapshotMetaData.Entry(snapshotId, request.includeGlobalState(), State.INIT, indices, null);
                    snapshots = new SnapshotMetaData(newSnapshot);
                } else {
                    // TODO: What should we do if a snapshot is already running?
                    throw new ConcurrentSnapshotExecutionException(snapshotId, "a snapshot is already running");
                }
                mdBuilder.putCustom(SnapshotMetaData.TYPE, snapshots);
                return ClusterState.builder(currentState).metaData(mdBuilder).build();
            }

            @Override
            public void onFailure(String source, Throwable t) {
                logger.warn("[{}][{}] failed to create snapshot", t, request.repository(), request.name());
                newSnapshot = null;
                listener.onFailure(t);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
                if (newSnapshot != null) {
                    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
                        @Override
                        public void run() {
                            beginSnapshot(newState, newSnapshot, request.partial, listener);
                        }
                    });
                }
            }

            @Override
            public TimeValue timeout() {
                return request.masterNodeTimeout();
            }

        });
    }

    /**
     * Validates snapshot request
     *
     * @param request snapshot request
     * @param state   current cluster state
     * @throws org.elasticsearch.ElasticsearchException
     */
    private void validate(SnapshotRequest request, ClusterState state) throws ElasticsearchException {
        RepositoriesMetaData repositoriesMetaData = state.getMetaData().custom(RepositoriesMetaData.TYPE);
        if (repositoriesMetaData == null || repositoriesMetaData.repository(request.repository()) == null) {
            throw new RepositoryMissingException(request.repository());
        }
        if (!Strings.hasLength(request.name())) {
            throw new InvalidSnapshotNameException(new SnapshotId(request.repository(), request.name()), "cannot be empty");
        }
        if (request.name().contains(" ")) {
            throw new InvalidSnapshotNameException(new SnapshotId(request.repository(), request.name()), "must not contain whitespace");
        }
        if (request.name().contains(",")) {
            throw new InvalidSnapshotNameException(new SnapshotId(request.repository(), request.name()), "must not contain ','");
        }
        if (request.name().contains("#")) {
            throw new InvalidSnapshotNameException(new SnapshotId(request.repository(), request.name()), "must not contain '#'");
        }
        if (request.name().charAt(0) == '_') {
            throw new InvalidSnapshotNameException(new SnapshotId(request.repository(), request.name()), "must not start with '_'");
        }
        if (!request.name().toLowerCase(Locale.ROOT).equals(request.name())) {
            throw new InvalidSnapshotNameException(new SnapshotId(request.repository(), request.name()), "must be lowercase");
        }
        if (!Strings.validFileName(request.name())) {
            throw new InvalidSnapshotNameException(new SnapshotId(request.repository(), request.name()), "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
        }
    }

    /**
     * Starts snapshot.
     *

     * Creates snapshot in repository and updates snapshot metadata record with list of shards that needs to be processed.
     *
     * @param clusterState               cluster state
     * @param snapshot                   snapshot meta data
     * @param partial                    allow partial snapshots
     * @param userCreateSnapshotListener listener
     */
    private void beginSnapshot(ClusterState clusterState, final SnapshotMetaData.Entry snapshot, final boolean partial, final CreateSnapshotListener userCreateSnapshotListener) {
        boolean snapshotCreated = false;
        try {
            Repository repository = repositoriesService.repository(snapshot.snapshotId().getRepository());

            MetaData metaData = clusterState.metaData();
            if (!snapshot.includeGlobalState()) {
                // Remove global state from the cluster state
                MetaData.Builder builder = MetaData.builder();
                for (String index : snapshot.indices()) {
                    builder.put(metaData.index(index), false);
                }
                metaData = builder.build();
            }

            repository.initializeSnapshot(snapshot.snapshotId(), snapshot.indices(), metaData);
            snapshotCreated = true;
            if (snapshot.indices().isEmpty()) {
                // No indices in this snapshot - we are done
                userCreateSnapshotListener.onResponse();
                endSnapshot(snapshot);
                return;
            }
            clusterService.submitStateUpdateTask("update_snapshot [" + snapshot + "]", new ProcessedClusterStateUpdateTask() {
                boolean accepted = false;
                SnapshotMetaData.Entry updatedSnapshot;
                String failure = null;

                @Override
                public ClusterState execute(ClusterState currentState) {
                    MetaData metaData = currentState.metaData();
                    MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                    SnapshotMetaData snapshots = metaData.custom(SnapshotMetaData.TYPE);
                    ImmutableList.Builder<SnapshotMetaData.Entry> entries = ImmutableList.builder();
                    for (SnapshotMetaData.Entry entry : snapshots.entries()) {
                        if (entry.snapshotId().equals(snapshot.snapshotId())) {
                            // Replace the snapshot that was just created
                            ImmutableMap<ShardId, SnapshotMetaData.ShardSnapshotStatus> shards = shards(snapshot.snapshotId(), currentState, snapshot.indices());
                            if (!partial) {
                                Set<String> indicesWithMissingShards = indicesWithMissingShards(shards);
                                if (indicesWithMissingShards != null) {
                                    updatedSnapshot = new SnapshotMetaData.Entry(snapshot.snapshotId(), snapshot.includeGlobalState(), State.FAILED, snapshot.indices(), shards);
                                    entries.add(updatedSnapshot);
                                    failure = "Indices don't have primary shards +[" + indicesWithMissingShards + "]";
                                    continue;
                                }
                            }
                            updatedSnapshot = new SnapshotMetaData.Entry(snapshot.snapshotId(), snapshot.includeGlobalState(), State.STARTED, snapshot.indices(), shards);
                            entries.add(updatedSnapshot);
                            if (!completed(shards.values())) {
                                accepted = true;
                            }
                        } else {
                            entries.add(entry);
                        }
                    }
                    mdBuilder.putCustom(SnapshotMetaData.TYPE, new SnapshotMetaData(entries.build()));
                    return ClusterState.builder(currentState).metaData(mdBuilder).build();
                }

                @Override
                public void onFailure(String source, Throwable t) {
                    logger.warn("[{}] failed to create snapshot", t, snapshot.snapshotId());
                    userCreateSnapshotListener.onFailure(t);
                }

                @Override
                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                    // The userCreateSnapshotListener.onResponse() notifies caller that the snapshot was accepted
                    // for processing. If client wants to wait for the snapshot completion, it can register snapshot
                    // completion listener in this method. For the snapshot completion to work properly, the snapshot
                    // should still exist when listener is registered.
                    userCreateSnapshotListener.onResponse();

                    // Now that snapshot completion listener is registered we can end the snapshot if needed
                    // We should end snapshot only if 1) we didn't accept it for processing (which happens when there
                    // is nothing to do) and 2) there was a snapshot in metadata that we should end. Otherwise we should
                    // go ahead and continue working on this snapshot rather then end here.
                    if (!accepted && updatedSnapshot != null) {
                        endSnapshot(updatedSnapshot, failure);
                    }
                }
            });
        } catch (Throwable t) {
            logger.warn("failed to create snapshot [{}]", t, snapshot.snapshotId());
            clusterService.submitStateUpdateTask("fail_snapshot [" + snapshot.snapshotId() + "]", new ClusterStateUpdateTask() {

                @Override
                public ClusterState execute(ClusterState currentState) {
                    MetaData metaData = currentState.metaData();
                    MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                    SnapshotMetaData snapshots = metaData.custom(SnapshotMetaData.TYPE);
                    ImmutableList.Builder<SnapshotMetaData.Entry> entries = ImmutableList.builder();
                    for (SnapshotMetaData.Entry entry : snapshots.entries()) {
                        if (!entry.snapshotId().equals(snapshot.snapshotId())) {
                            entries.add(entry);
                        }
                    }
                    mdBuilder.putCustom(SnapshotMetaData.TYPE, new SnapshotMetaData(entries.build()));
                    return ClusterState.builder(currentState).metaData(mdBuilder).build();
                }

                @Override
                public void onFailure(String source, Throwable t) {
                    logger.warn("[{}] failed to delete snapshot", t, snapshot.snapshotId());
                }
            });
            if (snapshotCreated) {
                try {
                    repositoriesService.repository(snapshot.snapshotId().getRepository()).finalizeSnapshot(snapshot.snapshotId(), ExceptionsHelper.detailedMessage(t), 0, ImmutableList.<SnapshotShardFailure>of());
                } catch (Throwable t2) {
                    logger.warn("[{}] failed to close snapshot in repository", snapshot.snapshotId());
                }
            }
            userCreateSnapshotListener.onFailure(t);
        }
    }

    @Override
    public void clusterChanged(ClusterChangedEvent event) {
        try {
            if (event.localNodeMaster()) {
                if (event.nodesRemoved()) {
                    processSnapshotsOnRemovedNodes(event);
                }
            }
            SnapshotMetaData prev = event.previousState().metaData().custom(SnapshotMetaData.TYPE);
            SnapshotMetaData curr = event.state().metaData().custom(SnapshotMetaData.TYPE);

            if (prev == null) {
                if (curr != null) {
                    processIndexShardSnapshots(curr);
                }
            } else {
                if (!prev.equals(curr)) {
                    processIndexShardSnapshots(curr);
                }
            }
        } catch (Throwable t) {
            logger.warn("Failed to update snapshot state ", t);
        }
    }

    /**
     * Cleans up shard snapshots that were running on removed nodes
     *
     * @param event cluster changed event
     */
    private void processSnapshotsOnRemovedNodes(ClusterChangedEvent event) {
        if (removedNodesCleanupNeeded(event)) {
            // Check if we just became the master
            final boolean newMaster = !event.previousState().nodes().localNodeMaster();
            clusterService.submitStateUpdateTask("update snapshot state after node removal", new ClusterStateUpdateTask() {
                @Override
                public ClusterState execute(ClusterState currentState) throws Exception {
                    DiscoveryNodes nodes = currentState.nodes();
                    MetaData metaData = currentState.metaData();
                    MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                    SnapshotMetaData snapshots = metaData.custom(SnapshotMetaData.TYPE);
                    if (snapshots == null) {
                        return currentState;
                    }
                    boolean changed = false;
                    ArrayList<SnapshotMetaData.Entry> entries = newArrayList();
                    for (final SnapshotMetaData.Entry snapshot : snapshots.entries()) {
                        SnapshotMetaData.Entry updatedSnapshot = snapshot;
                        boolean snapshotChanged = false;
                        if (snapshot.state() == State.STARTED) {
                            ImmutableMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableMap.builder();
                            for (ImmutableMap.Entry<ShardId, ShardSnapshotStatus> shardEntry : snapshot.shards().entrySet()) {
                                ShardSnapshotStatus shardStatus = shardEntry.getValue();
                                if (!shardStatus.state().completed() && shardStatus.nodeId() != null) {
                                    if (nodes.nodeExists(shardStatus.nodeId())) {
                                        shards.put(shardEntry);
                                    } else {
                                        // TODO: Restart snapshot on another node?
                                        snapshotChanged = true;
                                        logger.warn("failing snapshot of shard [{}] on closed node [{}]", shardEntry.getKey(), shardStatus.nodeId());
                                        shards.put(shardEntry.getKey(), new ShardSnapshotStatus(shardStatus.nodeId(), State.FAILED, "node shutdown"));
                                    }
                                }
                            }
                            if (snapshotChanged) {
                                changed = true;
                                ImmutableMap<ShardId, ShardSnapshotStatus> shardsMap = shards.build();
                                if (!snapshot.state().completed() && completed(shardsMap.values())) {
                                    updatedSnapshot = new SnapshotMetaData.Entry(snapshot.snapshotId(), snapshot.includeGlobalState(), State.SUCCESS, snapshot.indices(), shardsMap);
                                    endSnapshot(updatedSnapshot);
                                } else {
                                    updatedSnapshot = new SnapshotMetaData.Entry(snapshot.snapshotId(), snapshot.includeGlobalState(), snapshot.state(), snapshot.indices(), shardsMap);
                                }
                            }
                            entries.add(updatedSnapshot);
                        } else if (snapshot.state() == State.INIT && newMaster) {
                            // Clean up the snapshot that failed to start from the old master
                            deleteSnapshot(snapshot.snapshotId(), new DeleteSnapshotListener() {
                                @Override
                                public void onResponse() {
                                    logger.debug("cleaned up abandoned snapshot {} in INIT state", snapshot.snapshotId());
                                }

                                @Override
                                public void onFailure(Throwable t) {
                                    logger.warn("failed to clean up abandoned snapshot {} in INIT state", snapshot.snapshotId());
                                }
                            });
                        } else if (snapshot.state() == State.SUCCESS && newMaster) {
                            // Finalize the snapshot
                            endSnapshot(snapshot);
                        }
                    }
                    if (changed) {
                        snapshots = new SnapshotMetaData(entries.toArray(new SnapshotMetaData.Entry[entries.size()]));
                        mdBuilder.putCustom(SnapshotMetaData.TYPE, snapshots);
                        return ClusterState.builder(currentState).metaData(mdBuilder).build();
                    }
                    return currentState;
                }

                @Override
                public void onFailure(String source, Throwable t) {
                    logger.warn("failed to update snapshot state after node removal");
                }
            });
        }
    }

    private boolean removedNodesCleanupNeeded(ClusterChangedEvent event) {
        // Check if we just became the master
        boolean newMaster = !event.previousState().nodes().localNodeMaster();
        SnapshotMetaData snapshotMetaData = event.state().getMetaData().custom(SnapshotMetaData.TYPE);
        if (snapshotMetaData == null) {
            return false;
        }
        for (SnapshotMetaData.Entry snapshot : snapshotMetaData.entries()) {
            if (newMaster && (snapshot.state() == State.SUCCESS || snapshot.state() == State.INIT)) {
                // We just replaced old master and snapshots in intermediate states needs to be cleaned
                return true;
            }
            for (DiscoveryNode node : event.nodesDelta().removedNodes()) {
                for (ImmutableMap.Entry<ShardId, ShardSnapshotStatus> shardEntry : snapshot.shards().entrySet()) {
                    ShardSnapshotStatus shardStatus = shardEntry.getValue();
                    if (!shardStatus.state().completed() && node.getId().equals(shardStatus.nodeId())) {
                        // At least one shard was running on the removed node - we need to fail it
                        return true;
                    }
                }
            }
        }
        return false;
    }

    /**
     * Checks if any new shards should be snapshotted on this node
     *
     * @param snapshotMetaData snapshot metadata to be processed
     */
    private void processIndexShardSnapshots(SnapshotMetaData snapshotMetaData) {
        Map<SnapshotId, SnapshotShards> survivors = newHashMap();
        // First, remove snapshots that are no longer there
        for (Map.Entry<SnapshotId, SnapshotShards> entry : shardSnapshots.entrySet()) {
            if (snapshotMetaData != null && snapshotMetaData.snapshot(entry.getKey()) != null) {
                survivors.put(entry.getKey(), entry.getValue());
            }
        }

        // For now we will be mostly dealing with a single snapshot at a time but might have multiple simultaneously running
        // snapshots in the future
        HashMap<SnapshotId, SnapshotShards> newSnapshots = null;
        // Now go through all snapshots and update existing or create missing
        final String localNodeId = clusterService.localNode().id();
        for (SnapshotMetaData.Entry entry : snapshotMetaData.entries()) {
            HashMap<ShardId, IndexShardSnapshotStatus> startedShards = null;
            for (Map.Entry<ShardId, SnapshotMetaData.ShardSnapshotStatus> shard : entry.shards().entrySet()) {
                // Check if we have new shards to start processing on
                if (localNodeId.equals(shard.getValue().nodeId())) {
                    if (entry.state() == State.STARTED) {
                        if (startedShards == null) {
                            startedShards = newHashMap();
                        }
                        startedShards.put(shard.getKey(), new IndexShardSnapshotStatus());
                    } else if (entry.state() == State.ABORTED) {
                        SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshotId());
                        if (snapshotShards != null) {
                            IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.getKey());
                            if (snapshotStatus != null) {
                                snapshotStatus.abort();
                            }
                        }
                    }
                }
            }
            if (startedShards != null) {
                if (!survivors.containsKey(entry.snapshotId())) {
                    if (newSnapshots == null) {
                        newSnapshots = newHashMapWithExpectedSize(2);
                    }
                    newSnapshots.put(entry.snapshotId(), new SnapshotShards(ImmutableMap.copyOf(startedShards)));
                }
            }
        }

        if (newSnapshots != null) {
            survivors.putAll(newSnapshots);
        }

        // Update the list of snapshots that we saw and tried to started
        // If startup of these shards fails later, we don't want to try starting these shards again
        shardSnapshots = ImmutableMap.copyOf(survivors);

        // We have new snapshots to process -
        if (newSnapshots != null) {
            for (final Map.Entry<SnapshotId, SnapshotShards> entry : newSnapshots.entrySet()) {
                for (final Map.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : entry.getValue().shards.entrySet()) {
                    try {
                        final IndexShardSnapshotAndRestoreService shardSnapshotService = indicesService.indexServiceSafe(shardEntry.getKey().getIndex()).shardInjectorSafe(shardEntry.getKey().id())
                                .getInstance(IndexShardSnapshotAndRestoreService.class);
                        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
                            @Override
                            public void run() {
                                try {
                                    shardSnapshotService.snapshot(entry.getKey(), shardEntry.getValue());
                                    updateIndexShardSnapshotStatus(new UpdateIndexShardSnapshotStatusRequest(entry.getKey(), shardEntry.getKey(), new ShardSnapshotStatus(localNodeId, SnapshotMetaData.State.SUCCESS)));
                                } catch (Throwable t) {
                                    logger.warn("[{}] [{}] failed to create snapshot", t, shardEntry.getKey(), entry.getKey());
                                    updateIndexShardSnapshotStatus(new UpdateIndexShardSnapshotStatusRequest(entry.getKey(), shardEntry.getKey(), new ShardSnapshotStatus(localNodeId, SnapshotMetaData.State.FAILED, ExceptionsHelper.detailedMessage(t))));
                                }
                            }
                        });
                    } catch (Throwable t) {
                        updateIndexShardSnapshotStatus(new UpdateIndexShardSnapshotStatusRequest(entry.getKey(), shardEntry.getKey(), new ShardSnapshotStatus(localNodeId, SnapshotMetaData.State.FAILED, ExceptionsHelper.detailedMessage(t))));
                    }
                }
            }
        }
    }

    /**
     * Updates the shard status
     *
     * @param request update shard status request
     */
    private void updateIndexShardSnapshotStatus(UpdateIndexShardSnapshotStatusRequest request) {
        try {
            if (clusterService.state().nodes().localNodeMaster()) {
                innerUpdateSnapshotState(request);
            } else {
                transportService.sendRequest(clusterService.state().nodes().masterNode(),
                        UpdateSnapshotStateRequestHandler.ACTION, request, EmptyTransportResponseHandler.INSTANCE_SAME);
            }
        } catch (Throwable t) {
            logger.warn("[{}] [{}] failed to update snapshot state", t, request.snapshotId(), request.status());
        }
    }

    /**
     * Checks if all shards in the list have completed
     *
     * @param shards list of shard statuses
     * @return true if all shards have completed (either successfully or failed), false otherwise
     */
    private boolean completed(Collection<SnapshotMetaData.ShardSnapshotStatus> shards) {
        for (ShardSnapshotStatus status : shards) {
            if (!status.state().completed()) {
                return false;
            }
        }
        return true;
    }

    /**
     * Returns list of indices with missing shards
     *
     * @param shards list of shard statuses
     * @return list of failed indices
     */
    private Set<String> indicesWithMissingShards(ImmutableMap<ShardId, SnapshotMetaData.ShardSnapshotStatus> shards) {
        Set<String> indices = null;
        for (ImmutableMap.Entry<ShardId, SnapshotMetaData.ShardSnapshotStatus> entry : shards.entrySet()) {
            if (entry.getValue().state() == State.MISSING) {
                if (indices == null) {
                    indices = newHashSet();
                }
                indices.add(entry.getKey().getIndex());
            }
        }
        return indices;
    }

    /**
     * Updates the shard status on master node
     *
     * @param request update shard status request
     */
    private void innerUpdateSnapshotState(final UpdateIndexShardSnapshotStatusRequest request) {
        clusterService.submitStateUpdateTask("update snapshot state", new ClusterStateUpdateTask() {
            @Override
            public ClusterState execute(ClusterState currentState) {
                MetaData metaData = currentState.metaData();
                MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                SnapshotMetaData snapshots = metaData.custom(SnapshotMetaData.TYPE);
                if (snapshots != null) {
                    boolean changed = false;
                    ArrayList<SnapshotMetaData.Entry> entries = newArrayList();
                    for (SnapshotMetaData.Entry entry : snapshots.entries()) {
                        if (entry.snapshotId().equals(request.snapshotId())) {
                            HashMap<ShardId, ShardSnapshotStatus> shards = newHashMap(entry.shards());
                            logger.trace("[{}] Updating shard [{}] with status [{}]", request.snapshotId(), request.shardId(), request.status().state());
                            shards.put(request.shardId(), request.status());
                            if (!completed(shards.values())) {
                                entries.add(new SnapshotMetaData.Entry(entry.snapshotId(), entry.includeGlobalState(), entry.state(), entry.indices(), ImmutableMap.copyOf(shards)));
                            } else {
                                // Snapshot is finished - mark it as done
                                // TODO: Add PARTIAL_SUCCESS status?
                                SnapshotMetaData.Entry updatedEntry = new SnapshotMetaData.Entry(entry.snapshotId(), entry.includeGlobalState(), State.SUCCESS, entry.indices(), ImmutableMap.copyOf(shards));
                                entries.add(updatedEntry);
                                // Finalize snapshot in the repository
                                endSnapshot(updatedEntry);
                                logger.info("snapshot [{}] is done", updatedEntry.snapshotId());
                            }
                            changed = true;
                        } else {
                            entries.add(entry);
                        }
                    }
                    if (changed) {
                        snapshots = new SnapshotMetaData(entries.toArray(new SnapshotMetaData.Entry[entries.size()]));
                        mdBuilder.putCustom(SnapshotMetaData.TYPE, snapshots);
                        return ClusterState.builder(currentState).metaData(mdBuilder).build();
                    }
                }
                return currentState;
            }

            @Override
            public void onFailure(String source, Throwable t) {
                logger.warn("[{}][{}] failed to update snapshot status to [{}]", t, request.snapshotId(), request.shardId(), request.status());
            }
        });
    }

    /**
     * Finalizes the shard in repository and then removes it from cluster state
     *

     * This is non-blocking method that runs on a thread from SNAPSHOT thread pool
     *
     * @param entry snapshot
     */
    private void endSnapshot(SnapshotMetaData.Entry entry) {
        endSnapshot(entry, null);
    }

    /**
     * Finalizes the shard in repository and then removes it from cluster state
     *

     * This is non-blocking method that runs on a thread from SNAPSHOT thread pool
     *
     * @param entry   snapshot
     * @param failure failure reason or null if snapshot was successful
     */
    private void endSnapshot(final SnapshotMetaData.Entry entry, final String failure) {
        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
            @Override
            public void run() {
                SnapshotId snapshotId = entry.snapshotId();
                try {
                    final Repository repository = repositoriesService.repository(snapshotId.getRepository());
                    logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshotId, entry.state(), failure);
                    ArrayList<ShardSearchFailure> failures = newArrayList();
                    ArrayList<SnapshotShardFailure> shardFailures = newArrayList();
                    for (Map.Entry<ShardId, ShardSnapshotStatus> shardStatus : entry.shards().entrySet()) {
                        ShardId shardId = shardStatus.getKey();
                        ShardSnapshotStatus status = shardStatus.getValue();
                        if (status.state().failed()) {
                            failures.add(new ShardSearchFailure(status.reason(), new SearchShardTarget(status.nodeId(), shardId.getIndex(), shardId.id())));
                            shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId.getIndex(), shardId.id(), status.reason()));
                        }
                    }
                    Snapshot snapshot = repository.finalizeSnapshot(snapshotId, failure, entry.shards().size(), ImmutableList.copyOf(shardFailures));
                    removeSnapshotFromClusterState(snapshotId, new SnapshotInfo(snapshot), null);
                } catch (Throwable t) {
                    logger.warn("[{}] failed to finalize snapshot", t, snapshotId);
                    removeSnapshotFromClusterState(snapshotId, null, t);
                }
            }
        });
    }

    /**
     * Removes record of running snapshot from cluster state
     *
     * @param snapshotId snapshot id
     * @param snapshot   snapshot info if snapshot was successful
     * @param t          exception if snapshot failed
     */
    private void removeSnapshotFromClusterState(final SnapshotId snapshotId, final SnapshotInfo snapshot, final Throwable t) {
        clusterService.submitStateUpdateTask("remove snapshot metadata", new ProcessedClusterStateUpdateTask() {
            @Override
            public ClusterState execute(ClusterState currentState) {
                MetaData metaData = currentState.metaData();
                MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                SnapshotMetaData snapshots = metaData.custom(SnapshotMetaData.TYPE);
                if (snapshots != null) {
                    boolean changed = false;
                    ArrayList<SnapshotMetaData.Entry> entries = newArrayList();
                    for (SnapshotMetaData.Entry entry : snapshots.entries()) {
                        if (entry.snapshotId().equals(snapshotId)) {
                            changed = true;
                        } else {
                            entries.add(entry);
                        }
                    }
                    if (changed) {
                        snapshots = new SnapshotMetaData(entries.toArray(new SnapshotMetaData.Entry[entries.size()]));
                        mdBuilder.putCustom(SnapshotMetaData.TYPE, snapshots);
                        return ClusterState.builder(currentState).metaData(mdBuilder).build();
                    }
                }
                return currentState;
            }

            @Override
            public void onFailure(String source, Throwable t) {
                logger.warn("[{}][{}] failed to remove snapshot metadata", t, snapshotId);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                for (SnapshotCompletionListener listener : snapshotCompletionListeners) {
                    try {
                        if (snapshot != null) {
                            listener.onSnapshotCompletion(snapshotId, snapshot);
                        } else {
                            listener.onSnapshotFailure(snapshotId, t);
                        }
                    } catch (Throwable t) {
                        logger.warn("failed to refresh settings for [{}]", t, listener);
                    }
                }

            }
        });
    }

    /**
     * Deletes snapshot from repository.
     *

     * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
     *
     * @param snapshotId snapshot id
     * @param listener   listener
     */
    public void deleteSnapshot(final SnapshotId snapshotId, final DeleteSnapshotListener listener) {
        clusterService.submitStateUpdateTask("delete snapshot", new ProcessedClusterStateUpdateTask() {

            boolean waitForSnapshot = false;

            @Override
            public ClusterState execute(ClusterState currentState) throws Exception {
                MetaData metaData = currentState.metaData();
                MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
                SnapshotMetaData snapshots = metaData.custom(SnapshotMetaData.TYPE);
                if (snapshots == null) {
                    // No snapshots running - we can continue
                    return currentState;
                }
                SnapshotMetaData.Entry snapshot = snapshots.snapshot(snapshotId);
                if (snapshot == null) {
                    // This snapshot is not running - continue
                    if (!snapshots.entries().isEmpty()) {
                        // However other snapshots are running - cannot continue
                        throw new ConcurrentSnapshotExecutionException(snapshotId, "another snapshot is currently running cannot delete");
                    }
                    return currentState;
                } else {
                    // This snapshot is currently running - stopping shards first
                    waitForSnapshot = true;
                    ImmutableMap<ShardId, ShardSnapshotStatus> shards;
                    if (snapshot.state() == State.STARTED && snapshot.shards() != null) {
                        // snapshot is currently running - stop started shards
                        ImmutableMap.Builder<ShardId, ShardSnapshotStatus> shardsBuilder = ImmutableMap.builder();
                        for (ImmutableMap.Entry<ShardId, ShardSnapshotStatus> shardEntry : snapshot.shards().entrySet()) {
                            ShardSnapshotStatus status = shardEntry.getValue();
                            if (!status.state().completed()) {
                                shardsBuilder.put(shardEntry.getKey(), new ShardSnapshotStatus(status.nodeId(), State.ABORTED));
                            } else {
                                shardsBuilder.put(shardEntry.getKey(), status);
                            }
                        }
                        shards = shardsBuilder.build();
                    } else if (snapshot.state() == State.INIT) {
                        // snapshot hasn't started yet - end it
                        shards = snapshot.shards();
                        endSnapshot(snapshot);
                    } else {
                        // snapshot is being finalized - wait for it
                        logger.trace("trying to delete completed snapshot - save to delete");
                        return currentState;
                    }
                    SnapshotMetaData.Entry newSnapshot = new SnapshotMetaData.Entry(snapshotId, snapshot.includeGlobalState(), State.ABORTED, snapshot.indices(), shards);
                    snapshots = new SnapshotMetaData(newSnapshot);
                    mdBuilder.putCustom(SnapshotMetaData.TYPE, snapshots);
                    return ClusterState.builder(currentState).metaData(mdBuilder).build();
                }
            }

            @Override
            public void onFailure(String source, Throwable t) {
                listener.onFailure(t);
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                if (waitForSnapshot) {
                    logger.trace("adding snapshot completion listener to wait for deleted snapshot to finish");
                    addListener(new SnapshotCompletionListener() {
                        @Override
                        public void onSnapshotCompletion(SnapshotId snapshotId, SnapshotInfo snapshot) {
                            logger.trace("deleted snapshot completed - deleting files");
                            removeListener(this);
                            deleteSnapshotFromRepository(snapshotId, listener);
                        }

                        @Override
                        public void onSnapshotFailure(SnapshotId snapshotId, Throwable t) {
                            logger.trace("deleted snapshot failed - deleting files", t);
                            removeListener(this);
                            deleteSnapshotFromRepository(snapshotId, listener);
                        }
                    });
                } else {
                    logger.trace("deleted snapshot is not running - deleting files");
                    deleteSnapshotFromRepository(snapshotId, listener);
                }
            }
        });
    }

    /**
     * Checks if a repository is currently in use by one of the snapshots
     *
     * @param clusterState cluster state
     * @param repository   repository id
     * @return true if repository is currently in use by one of the running snapshots
     */
    public static boolean isRepositoryInUse(ClusterState clusterState, String repository) {
        MetaData metaData = clusterState.metaData();
        SnapshotMetaData snapshots = metaData.custom(SnapshotMetaData.TYPE);
        if (snapshots != null) {
            for (SnapshotMetaData.Entry snapshot : snapshots.entries()) {
                if (repository.equals(snapshot.snapshotId().getRepository())) {
                    return true;
                }
            }
        }
        return false;
    }

    /**
     * Deletes snapshot from repository
     *
     * @param snapshotId snapshot id
     * @param listener   listener
     */
    private void deleteSnapshotFromRepository(final SnapshotId snapshotId, final DeleteSnapshotListener listener) {
        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
            @Override
            public void run() {
                try {
                    Repository repository = repositoriesService.repository(snapshotId.getRepository());
                    repository.deleteSnapshot(snapshotId);
                    listener.onResponse();
                } catch (Throwable t) {
                    listener.onFailure(t);
                }
            }
        });
    }

    /**
     * Calculates the list of shards that should be included into the current snapshot
     *
     * @param snapshotId   snapshot id
     * @param clusterState cluster state
     * @param indices      list of indices to be snapshotted
     * @return list of shard to be included into current snapshot
     */
    private ImmutableMap<ShardId, SnapshotMetaData.ShardSnapshotStatus> shards(SnapshotId snapshotId, ClusterState clusterState, ImmutableList<String> indices) {
        ImmutableMap.Builder<ShardId, SnapshotMetaData.ShardSnapshotStatus> builder = ImmutableMap.builder();
        MetaData metaData = clusterState.metaData();
        for (String index : indices) {
            IndexMetaData indexMetaData = metaData.index(index);
            IndexRoutingTable indexRoutingTable = clusterState.getRoutingTable().index(index);
            if (indexRoutingTable == null) {
                throw new SnapshotCreationException(snapshotId, "Missing routing table for index [" + index + "]");
            }
            for (int i = 0; i < indexMetaData.numberOfShards(); i++) {
                ShardId shardId = new ShardId(index, i);
                ShardRouting primary = indexRoutingTable.shard(i).primaryShard();
                if (primary == null || !primary.assignedToNode()) {
                    builder.put(shardId, new SnapshotMetaData.ShardSnapshotStatus(null, State.MISSING, "primary shard is not allocated"));
                } else if (!primary.started()) {
                    builder.put(shardId, new SnapshotMetaData.ShardSnapshotStatus(primary.currentNodeId(), State.MISSING, "primary shard hasn't been started yet"));
                } else {
                    builder.put(shardId, new SnapshotMetaData.ShardSnapshotStatus(primary.currentNodeId()));
                }
            }
        }

        return builder.build();
    }

    /**
     * Adds snapshot completion listener
     *
     * @param listener listener
     */
    public void addListener(SnapshotCompletionListener listener) {
        this.snapshotCompletionListeners.add(listener);
    }

    /**
     * Removes snapshot completion listener
     *
     * @param listener listener
     */
    public void removeListener(SnapshotCompletionListener listener) {
        this.snapshotCompletionListeners.remove(listener);
    }

    /**
     * Listener for create snapshot operation
     */
    public static interface CreateSnapshotListener {

        /**
         * Called when snapshot has successfully started
         */
        void onResponse();

        /**
         * Called if a snapshot operation couldn't start
         */
        void onFailure(Throwable t);
    }

    /**
     * Listener for delete snapshot operation
     */
    public static interface DeleteSnapshotListener {

        /**
         * Called if delete operation was successful
         */
        void onResponse();

        /**
         * Called if delete operation failed
         */
        void onFailure(Throwable t);
    }

    public static interface SnapshotCompletionListener {

        void onSnapshotCompletion(SnapshotId snapshotId, SnapshotInfo snapshot);

        void onSnapshotFailure(SnapshotId snapshotId, Throwable t);
    }

    /**
     * Snapshot creation request
     */
    public static class SnapshotRequest {

        private String cause;

        private String name;

        private String repository;

        private String[] indices;

        private IndicesOptions indicesOptions = IndicesOptions.strict();

        private boolean partial;

        private Settings settings;

        private boolean includeGlobalState;

        private TimeValue masterNodeTimeout;

        /**
         * Constructs new snapshot creation request
         *
         * @param cause      cause for snapshot operation
         * @param name       name of the snapshot
         * @param repository name of the repository
         */
        public SnapshotRequest(String cause, String name, String repository) {
            this.cause = cause;
            this.name = name;
            this.repository = repository;
        }

        /**
         * Sets the list of indices to be snapshotted
         *
         * @param indices list of indices
         * @return this request
         */
        public SnapshotRequest indices(String[] indices) {
            this.indices = indices;
            return this;
        }

        /**
         * Sets repository-specific snapshot settings
         *
         * @param settings snapshot settings
         * @return this request
         */
        public SnapshotRequest settings(Settings settings) {
            this.settings = settings;
            return this;
        }

        /**
         * Set to true if global state should be stored as part of the snapshot
         *
         * @param includeGlobalState true if global state should be stored as part of the snapshot
         * @return this request
         */
        public SnapshotRequest includeGlobalState(boolean includeGlobalState) {
            this.includeGlobalState = includeGlobalState;
            return this;
        }

        /**
         * Sets master node timeout
         *
         * @param masterNodeTimeout master node timeout
         * @return this request
         */
        public SnapshotRequest masterNodeTimeout(TimeValue masterNodeTimeout) {
            this.masterNodeTimeout = masterNodeTimeout;
            return this;
        }

        /**
         * Sets the indices options
         *
         * @param indicesOptions indices options
         * @return this request
         */
        public SnapshotRequest indicesOptions(IndicesOptions indicesOptions) {
            this.indicesOptions = indicesOptions;
            return this;
        }

        /**
         * Set to true if partial snapshot should be allowed
         *
         * @param partial true if partial snapshots should be allowed
         * @return this request
         */
        public SnapshotRequest partial(boolean partial) {
            this.partial = partial;
            return this;
        }

        /**
         * Returns cause for snapshot operation
         *
         * @return cause for snapshot operation
         */
        public String cause() {
            return cause;
        }

        /**
         * Returns snapshot name
         *
         * @return snapshot name
         */
        public String name() {
            return name;
        }

        /**
         * Returns snapshot repository
         *
         * @return snapshot repository
         */
        public String repository() {
            return repository;
        }

        /**
         * Returns the list of indices to be snapshotted
         *
         * @return the list of indices
         */
        public String[] indices() {
            return indices;
        }

        /**
         * Returns indices options
         *
         * @return indices options
         */
        public IndicesOptions indicesOptions() {
            return indicesOptions;
        }

        /**
         * Returns repository-specific settings for the snapshot operation
         *
         * @return repository-specific settings
         */
        public Settings settings() {
            return settings;
        }

        /**
         * Returns true if global state should be stored as part of the snapshot
         *
         * @return true if global state should be stored as part of the snapshot
         */
        public boolean includeGlobalState() {
            return includeGlobalState;
        }

        /**
         * Returns master node timeout
         *
         * @return master node timeout
         */
        public TimeValue masterNodeTimeout() {
            return masterNodeTimeout;
        }

    }

    /**
     * Stores the list of shards that has to be snapshotted on this node
     */
    private static class SnapshotShards {
        private final ImmutableMap<ShardId, IndexShardSnapshotStatus> shards;

        private SnapshotShards(ImmutableMap<ShardId, IndexShardSnapshotStatus> shards) {
            this.shards = shards;
        }
    }

    /**
     * Internal request that is used to send changes in snapshot status to master
     */
    private static class UpdateIndexShardSnapshotStatusRequest extends TransportRequest {
        private SnapshotId snapshotId;
        private ShardId shardId;
        private SnapshotMetaData.ShardSnapshotStatus status;

        private UpdateIndexShardSnapshotStatusRequest() {

        }

        private UpdateIndexShardSnapshotStatusRequest(SnapshotId snapshotId, ShardId shardId, SnapshotMetaData.ShardSnapshotStatus status) {
            this.snapshotId = snapshotId;
            this.shardId = shardId;
            this.status = status;
        }

        @Override
        public void readFrom(StreamInput in) throws IOException {
            super.readFrom(in);
            snapshotId = SnapshotId.readSnapshotId(in);
            shardId = ShardId.readShardId(in);
            status = SnapshotMetaData.ShardSnapshotStatus.readShardSnapshotStatus(in);
        }

        @Override
        public void writeTo(StreamOutput out) throws IOException {
            super.writeTo(out);
            snapshotId.writeTo(out);
            shardId.writeTo(out);
            status.writeTo(out);
        }

        public SnapshotId snapshotId() {
            return snapshotId;
        }

        public ShardId shardId() {
            return shardId;
        }

        public SnapshotMetaData.ShardSnapshotStatus status() {
            return status;
        }
    }

    /**
     * Transport request handler that is used to send changes in snapshot status to master
     */
    private class UpdateSnapshotStateRequestHandler extends BaseTransportRequestHandler<UpdateIndexShardSnapshotStatusRequest> {

        static final String ACTION = "cluster/snapshot/update_snapshot";

        @Override
        public UpdateIndexShardSnapshotStatusRequest newInstance() {
            return new UpdateIndexShardSnapshotStatusRequest();
        }

        @Override
        public void messageReceived(UpdateIndexShardSnapshotStatusRequest request, final TransportChannel channel) throws Exception {
            innerUpdateSnapshotState(request);
            channel.sendResponse(TransportResponse.Empty.INSTANCE);
        }

        @Override
        public String executor() {
            return ThreadPool.Names.SAME;
        }
    }

}

redpajama_set_name: RedPajamaGithub


text:
layout:     post
title:      "博客模板"
subtitle:   "博客模板"
date:       2016-06-04 12:00:00
author:     "Lushun"
header-img: "img/post-bg-2015.jpg"
catalog: true
tags:
    - 技术
    - 博客
---

#Hux Blog

###[View Live Hux Blog →](http://huxpro.github.io)

![](http://huangxuan.me/img/blog-desktop.jpg)

### "Page Build Warning" email

These days, some of you must receive a "Page Build Warning" email from github after you commit happily. **Don't Worried!** It just that github changes its build environment.

In this mail, github told us:

> You are attempting to use the 'pygments' highlighter, which is currently unsupported on GitHub Pages. Your site will use 'rouge' for highlighting instead. To suppress this warning, change the 'highlighter' value to 'rouge' in your '_config.yml'.

So, just edit `_config.yml`, find `highlighter: pygments`, change it to `highlighter: rouge` and the warning will be gone.

## Boilerplate (beta)

Want to clone a boilerplate instead of my buzz blog? Here comes this!

```
$ git clone git@github.com:Huxpro/huxblog-boilerplate.git
```

**[View Boilerplate Here →](http://huangxuan.me/huxblog-boilerplate/)**

## Translation

 - ![cn](https://raw.githubusercontent.com/gosquared/flags/master/flags/flags/shiny/24/China.png) **Chinese (Simplified)**: 感谢 [@BrucZhaoR](https://github.com/BruceZhaoR) 的 [中文翻译 →](https://github.com/Huxpro/huxpro.github.io/blob/master/README.zh.md)

## Features

##### New Feature (V1.5.2)

- Annoyed to delete my blog post after clone or pull? **Boilerplate** comes to help you get started quickly and easily merge update.
- `-apple-system` is added in font rule, which display beautiful new font **San Francisco** in iOS 9 by default.
- Fixed [issue#15](https://github.com/Huxpro/huxpro.github.io/issues/15) about code wrap.

##### New Feature (V1.5.1)

- **[Comment](#comment)** support [**Disqus**](http://disqus.com) officially, thanks to @rpsh.

##### New Feature (V1.5)

- **[Comment](#comment)** and **[Analytics](#analytics)** is configurable now! We also add **Google Analytics support** and drop tencents. Both documents is updated.

##### New Feature (V1.4)

- **[Featured Tags](#featured-tags)** is now independent of [SideBar](#sidebar). Both documents is updated.
- New **[SEO Title](#seo-title)** for SEO usage which is differ from the site title

##### New Feature (V1.3.1)

- Support **PingFang (苹方)**, the new Chinese font presented by [OS X El Capitan](http://www.apple.com/cn/osx/whats-new/)

##### New Feature (V1.3)

- Big Improvement to the **Navigation Menu** *(especially in Android)*:  Dropping the old, stuttering, low-performance [Bootstrap collapse.js](http://getbootstrap.com/javascript/#collapse),  replaced with an own wrote, [jank free](http://jankfree.org/) navbar menu in a pretty high-performance implementation of [Google Material Design](https://www.google.com/design/spec/material-design/introduction.html).

<img src="http://huangxuan.me/img/blog-md-navbar.gif" width="320" />

##### New Feature (V1.2)

- Brand new **[Keynote Layout](#keynote-layout)** is provided for easily posting beautiful HTML presentations you have created with this blog

##### New Feature (V1.1)

- We now support a clean and gorgeous **[SideBar](#sidebar)** for displaying more info
- **[Friends](#friends)** is also added as a common feature of blog help you do SEO

##### V1.0

- Full-feature **Tag** support
- **Mobile first** user experience optimization
- **Typographic optimization** for Chinese Fonts
- **Network optimizaition** for China, dropping Google webfont, using local CDN
- Using [Github Flavored Markdown](https://help.github.com/articles/github-flavored-markdown/)
- Using Baidu, Tencent/QQ analytics
- Using [DuoShuo](http://duoshuo.com/) as the Disqus-like third party discussion system

## Support

- **Feel free to fork. I'll Appreciate it if you keep the Author & Github link at footer**
- Give it a **Star** if you like, fork or just clone to use ;)
- If any problem or requirement, just open an issue here and I will help you.

## Document

* Get Started
	* [Environment](#environment)
	* [Get Started](#get-started)
	* [Write Posts](#write-posts)
* Components
	* [SideBar](#sidebar)
	* [Mini About Me](#mini-about-me)
	* [Featured Tags](#featured-tags)
	* [Friends](#friends)
	* [Keynote Layout](#keynote-layout)
* Comment & Analysis
	* [Comment](#comment)
	* [Analytics](#analytics)
* Advanced
	* [Customization](#customization)
	* [Header Image](#header-image)
	* [SEO Title](#seo-title)

#### Environment

If you have jekyll installed, simply run `jekyll serve` in Command Line
and preview the themes in your browser. You can use `jekyll serve --watch` to watch for changes in the source files as well.

#### Get Started

You can easily get started by modifying `_config.yml`:

```
# Site settings
title: Hux Blog             # title of your website
SEOTitle: Hux Blog			# check out docs for more detail
description: "Cool Blog"    # ...

# SNS settings
github_username: huxpro     # modify this account to yours
weibo_username: huxpro      # the footer woule be auto-updated.

# Build settings
# paginate: 10              # nums of posts in one page
```

There are more options you can check out in the [Jekyll - Official Site](http://jekyllrb.com/), or you can directly dive into code to find more.

#### Write Posts

Feel free to checkout Markdown files in the `_posts/`, you will quickly realized how to post your articles with magical markdown plus this nice theme.

The **front-matter** of a post looks like that:

```
---
layout:     post
title:      "Hello 2015"
subtitle:   "Hello World, Hello Blog"
date:       2015-01-29 12:00:00
author:     "Hux"
header-img: "img/post-bg-2015.jpg"
tags:
    - Life
---

```

#### SideBar

![](http://huangxuan.me/img/blog-sidebar.jpg)

Seeing more information may be necessary for you to display, from V1.1, a clean, gorgeous **SideBar** is added for you, which provide more area for displaying possible modules. You can enable *(it is default enable)* this feature by simply config:

```
# Sidebar settings
sidebar: true
sidebar-about-description: "your description here"
sidebar-avatar: /img/avatar-hux.jpg     # use absolute URL.
```

We default support *[Featured Tags](#featured-tags)*, *[Mini About Me](#mini-about-me)* and *[Friends](#friends)* these three modules and you can add your own. The sidebar is naturally responsive and would be push to bottom in a small screen size (`<= 992px`, according to [Bootstarp Grid System](http://getbootstrap.com/css/#grid))
More details of these three separate modules are talking below.

#### Mini About Me

Mini-About-Me module display all your SNS buttons also your avatar and the description if you set `sidebar-avatar` and `sidebar-about-description` which is very useful and common for a sidebar so it is default with your sidebar.

It is really nice-looking and well-designed. It would be hidden in a small screen seeing the sidebar would be push to bottom and there is already a footer including SNS feature which is similar.

#### Featured Tags

Considering the Featured-Tags feature in [Medium](http://medium.com) is pretty cool, so I add it in my blog theme also.
This module is independent of sidebar from V1.4, so it can definitely live without enable sidebar, which would be displayed in the bottom when `sidebar` set to false, and it is not only displayed in home page but also every post page bottom.

```
# Featured Tags
featured-tags: true
featured-condition-size: 1     # A tag will be featured if the size of it is more than this condition value
```

The only one thing need to be paid attention to is the `featured-condition-size`: A tag will be featured if the size of it is more than this condition value.
Internally, a condition template  is used to do the filter.

#### Friends

Friends is a very common feature of a blog seeing the SEO, so I add it in V1.1 release to help that.
Friends can also live without enable sidebar, also be displayed in the bottom when sidebar unable, and be displayed in every post page bottom.

You can just add your friends information in `_config.yml` with a familiar JSON syntax and everything is done, very easy:

```
# Friends
friends: [
    {
        title: "Foo Blog",
        href: "http://foo.github.io/"
    },
    {
        title: "Bar Blog",
        href: "http://bar.github.io"
    }
]
```

#### Keynote Layout

![](http://huangxuan.me/img/blog-keynote.jpg)

There is a increasing tendency to use Open Web technology to create keynotes, presentations, like Reveal.js, Impress.js, Slides, Prezi etc. I consider a modern blog should have abilities to post these HTML based presentation easily also abilities to play it directly.

Under the hood, a `iframe` is used to include webpage from outer source, so the only things left is to give a url in the **front-matter**:

```
---
layout:     keynote
iframe:     "http://huangxuan.me/js-module-7day/"
---
```

The iframe will be automatically resized to adapt different form factors also the device orientation. A padding is left to imply user that there has more content below, also to ensure that there is a area for user to scroll down in mobile device seeing most of the keynote framework prevent the browser default scroll behavior.

#### Comment

This theme support both [Disqus](http://disqus.com) and [Duoshuo](http://duoshuo.com) as the third party discussion system.

First, you need to sign up and get your own account. **Repeat, DO NOT use mine!** (I have set Trusted Domains) It is deathly simple to sign up and you will get the full power of management system. Please give it a try!

Second, from V1.5, you can easily complete your comment configuration by just adding your **short name** into `_config.yml`:

```
duoshuo_username: _your_duoshuo_short_name_
# OR
disqus_username: _your_disqus_short_name_
```

**To the old version user**, it's better that you pull the new version, otherwise you have to replace code in `post.html`, `keynote.html` and `about.html` by yourselves.

Furthermore, Duoshuo support Sharing. if you only wanna use Duoshuo comment without sharing, you can set `duoshuo_share: false`. You can use Duoshuo Sharing and Disqus Comments together also.

#### Analytics

From V1.5, we support Google Analytics and Baidu Tongji officially with a deathly simple config:

```
# Baidu Analytics
ba_track_id: 4cc1f2d8f3067386cc5cdb626a202900

# Google Analytics
ga_track_id: 'UA-49627206-1'            # Format: UA-xxxxxx-xx
ga_domain: huangxuan.me
```

Just checkout the code offered by Google/Baidu, and copy paste here, all the rest is already done for you.

#### Customization

If you wanna do more customization and change code yourself, a [Grunt](gruntjs.com) environment is also included. (Thanks to Clean Blog.)

There are a number of tasks it performs like minification of the JavaScript, compiling of the LESS files, adding banners to keep the Apache 2.0 license intact, and watching for changes. Run the grunt default task by entering `grunt ` into your command line which will build the files. You can use `grunt watch` if you are working on the JavaScript or the LESS.

**Try to understand code in `_include/` and `_layouts/`, then you can modify Jekyll [Liquid](https://github.com/Shopify/liquid/wiki) template directly to do more creative customization.**

#### Header Image

Change header images of any pages or any posts is pretty easy as mentioned above. But, thanks to [issue #6 (in Chinese)](https://github.com/Huxpro/huxpro.github.io/issues/6) asked, **how to make it looks great?**

**Well...it is actually a design issue**, not a coding stuff. It is better that you have basic design knowledge, but not is ok, let me told you how to make it well-designed:

Seeing the title text above image is **white**, the image should be **dark** to emphasize the contract. so we can easily add a **black overlay with fews of opacity**, which is depends on the brightness of the original images you used. you can process it in Photoshop, Sketch etc.

In technical views, it can be done with CSS. However, the opacity of the black overlay is really hard to assigned, **every image has different brightness so the  degree it should be adjusted is different so it is impossible to hard code it.**

#### SEO Title

Before V1.4, site setting `title` is not only used for displayed in Home Page and Navbar, but also used to generate the `<title>` in HTML.
It's possible that you want the two things different. For me, my site-title is **"Hux Blog"** but I want the title shows in search engine is **"黄玄的博客 | Hux Blog"** which is multi-language.

So, the SEO Title is introduced to solve this problem, you can set `SEOTitle` different from `title`, and it would be only used to generate HTML `<title>` and setting DuoShuo Sharing.

## Thanks

This theme is forked from [IronSummitMedia/startbootstrap-clean-blog-jekyll](https://github.com/IronSummitMedia/startbootstrap-clean-blog-jekyll)
Thanks Jekyll and Github Pages!


Thanks for taking a few moments to stop by our Queens, NY Renters Insurance Guide. We recognize that Queens renters insurance can vary significantly even between the different neighborhoods, and the risks you're protecting against might even vary between blocks. Effective Coverage has years of experience writing New York Renters Insurance and we know the importance of evaluating the risks to make sure you get the correct policy for your needs.
Effective Coverage takes the time to understand your individual risk profile because we want you to be protected. Queens renters insurance isn't just something we sell at Effective Coverage, it's our mission. Many years ago, Eric Narcisco lost everything he owned to an apartment fire that his neighbor started accidentally. From the ashes of his apartment arose Effective Coverage, on a mission to educate Americans about the importance of renters insurance nationwide and to make the coverage accessible to everyone.
Accessibility has traditionally been a big concern with Queens, NY renters insurance. Part of the problem is that most of the agents who have storefront agencies want nothing to do with renters insurance in Queens. There's just no revenue in a $150 a year renters policy, from their perspective. In order to make renters insurance accessible to everyone, Effective Coverage has turned the model on its head. Iterative improvement is a crucial part of what we do here, and it's led to creation of new efficiencies in the process that allow us to offer Queens renters insurance while actually taking the time to ask the right questions.
Effective Coverage isn't just an assembly line, either. You'll work with a New York Renters Insurance expert who is salaried, not on commission, and whose job it is to get you the right coverage to meet your needs, regardless of the time it takes. There's always a friendly Queens, NY renters insurance expert you can call to get expert advice on personal risk management, and who will make sure you're satisfied before you're off the phone. If you know exactly what you need, and you want to take care of it online without a consultation, we've made that process more efficient, as well. But if you do find that you have a question while you're quoting renters insurance online, Effective Coverage even offers live chat with renters insurance experts to help you find the answers you need.
Relationships and education make what we do possible, and we value those relationships. Take just a few minutes and call or click above to get your Queens, NY renters insurance started today, and to give your family the protection they so richly deserve.
How Does Queens, NY Renters Insurance Help With Personal Risk Management?
This just means that you understand the types and amounts of risk that you retain, and accept that you're going to be responsible for the results should something untoward happen. Do you have a $1,000 deductible on your collision coverage on your auto policy? If so, that means you've retained $1,000 of risk – anything beyond that the insurance company will pay, subject to the policy limits. Queens, NY renters insurance also comes with a deductible, usually lower than the above example.
Risk mitigation is the part of personal risk management where you do the things reasonably in your power to reduce the amount of risk that you face. This could range from locking your door to having smoke detectors in the correct places, even just keeping your head up and being aware of your surroundings when you're walking down the street is a form of risk mitigation. Since complete risk avoidance is an impossible task, we have to settle for mitigation. But there's a better way, that complements risk retention and risk mitigation.
Risk transference is the process of transferring some or all of the risk you face to someone else. The tradeoff is that this costs money – it's called an insurance premium. Queens renters insurance is nothing but risk transference arranged in a very specific way to transfer specific types of risk. You pay a token premium to the company, in return for which they assume the risk that something will happen, such as a fire. If that insured event or events occur, the insurance company will pay to return you to the financial condition you were in before the loss.
Queens, NY renters insurance helps with personal risk management by making sure that an event that's not your fault doesn't cause you to lose everything. Most people can't afford to replace everything they own if it were lost in a fire, for instance. Those people need to insure against that risk, and that also comes with insurance against other risks, as well. Queens renters insurance protects you from common risks as well as uncommon risks.
Just as importantly, your policy protects against the risk of you causing bodily injury or property damage to someone else through your negligence. What if you spilled your coffee on someone walking down the street? What if you were frying chicken, and the phone rang, distracting you long enough for that hot oil to set your kitchen on fire? How would you pay for the damage to the building, or the damages suffered by other residents? Chances are you wouldn't be able to. That's what renters insurance in Queens is for. The insurance assumes the risk of those damages, as well as the risk of the cost of defending you in that situation.
How Much Does Renters Insurance Cost In Queens?
The average Queens, NY renters insurance policy costs just fifteen dollars a month. That works out to about $187.00 for the year, though it's often even less expensive than that. You can save money on renters insurance in Queens by paying for the year in full – often there are either discounts available for doing that or you save on the installment fees that nearly every underwriter will charge. You also often are able to split it into two payments about six months apart. By doing that, you maximize the time value of your money but still save on the installment fees.
The best way to find out how much Queens renters insurance will cost you is to call Effective Coverage. You'll get a personalized quote customized to your individual needs in a matter of minutes, with exact prices and coverage. It's easy, fast, and inexpensive.
Where Can I Buy Renters Insurance In Queens, NY?
While Queens does lend itself to walking into several storefront agencies to compare prices, most of those agencies aren't going to have much interest in a standalone renters insurance policy in Queens. They'll want to sell you other things, because a single renters insurance policy doesn't fit into their revenue model. For the best prices and the best service, call the Queens, NY renters insurance experts at Effective Coverage. The renters insurance expert you speak with will be someone who is focused on standalone renters insurance policies. As we noted about, our insurance experts are on salary, so they're willing to take the time to get the right policy for you and to answer all of your questions – they're not under sales pressure, their entire job is to help educate you about the coverage and how it meshes with your needs, and to get you the correct policy to meet those needs. It just takes a few minutes on the phone or a few clicks online and you can be covered the same day, with proof of coverage sent to your leasing office in seconds!
Many people believe that Queens is by far the best borough of New York City. As the most ethnically diverse urban area in the world, Queens offers a bit of something for anyone, no matter their background. Of course, there are just some things that everyone understands and loves, like sports, music, and the arts. Queens has all of it. The USTA National Tennis Center in Queens is a large, open venue that hosts so many tennis matches you'll wonder how to find time to enjoy all of them. With a variety of food and drinks, clean seats, plenty of space, and a friendly and helpful staff, tennis matches can be enjoyed equally by those with a profound love for the sport and by those who just watch sports from the couch once in a while. It's great fun for the whole family, couples, or hanging out with friends.
Next to the complex, you can catch a Mets game as well. City Field is another place to enjoy a good game of baseball with the Yankees or Mets. With fair ticket prices, amazingly comfortable seats, and top-notch food, this modern stadium is a great place to enjoy a game. Wherever you find yourself in the stadium, even in the bathrooms, there are televisions that will keep you apprised of the game being played. The entire stadium is clean and spotless, and there is a surprising amount of interaction with the crowd. Everyone goes out of their way to make it a good experience and pleasant atmosphere for attendees.
As of 2003, Queens had almost 40,000 business establishments scattered throughout the borough. Several large companies have their headquarters in Queens, including renowned watchmaker Bulova and famous international piano manufacturer Steinway & Sons, in Astoria. Glacéau, the makers of Vitamin Water, and JetBlue Airways also make their corporate homes here. Queens has the lowest unemployment rate in all of New York City, which is yet another reason to love making your home in Queens.
Choosing to make your home here might cost a bit more than in other boroughs of NYC, but it's worth it in the long run. The estimated median value of a home or condo here is $440,000, but with everything Queens has to offer and with non-stop expansion of culture and industry it's well worth the money to live here. Through the entire borough, the average median rent is about $2,042. This can vary by neighborhood, of course, but it's quite affordable for NYC. Sunnyside, Queens was named one of the most affordable neighborhoods in New York City. Sunnyside is easily accessible by the 7 train, just fifteen minutes from Midtown Manhattan. One report recently cited an average monthly rent of just $1,300 for a two bedroom apartment. That's an incredible deal for anyplace in NYC, especially for a neighborhood as safe as Sunnyside in a borough as diverse and welcoming as Queens. Bear in mind that people pay twice that much to live in Jersey City, of all places, where they have to struggle with the PATH Train and the other vagaries of life in New Jersey while working in NYC. It makes far more sense to just live in queens and be a quick subway ride from work. Not only is it far less expensive than living in Jersey City, but it also means you have a shorter commute and more time to spend with loved ones as a result.
A great cultural spot in Queens, NY is the Louis Armstrong House Museum, which was the home of the great musician and an even greater man. It's a very intimate and personal journey through this man's life, and you'll come away with changed views. Most of the original furnishings are still present, and passionate tour guides and audio clips from Louis' home-recorded tapes complete the experience so that you'll feel like he's standing right next to you as you wander through the tour.
Queens has come a long way in recent years. Leading Queens from its former status as an industrial wasteland to its current reputation as an arts district, Noguchi Museum has two levels and a rooftop terrace used as exhibition spaces for the astonishing Japanese art and gardens of Isamu Noguchi. The Museum is located opposite the Japanese American artist's former home and workplace for most of his career, where he created drawings for many installations, beautiful large sculptures, and projects for gardens, furniture, and other forms of art. The Museum is a different experience with each season, with different plantings for each season blending with various stone and gravel installations in different parts of the Museum. The Museum shows Noguchi at his best, with statues that look different from each angle, so that each viewing angle makes a different impact on the visitor.
A great place to run, skate, or ride a bike through the beautiful scenery is Gantry Plaza Stat Park. Nestled close to the water, this park is a great spot for a picnic. With a lot of open space for kids to play and many creatively designed built-in lounge chairs, this is the perfect spot to observe the city, enjoy the sunshine, or watch the ferry cross the river. With this park's location, there are numerous spots to take a picture of the sunset or a panorama of the city. It's a great place to enjoy the sunset with your date and to make some memories. The best part of all, of course, is that it's free!
Flushing Meadows Corona Park is definitely the place to escape the noise of the city and relax. With each season, this park brings a different feeling, from a beautiful snow covering in the winter to watching the cherry trees blossom in the spring. For those who plan to bring the family along, there are numerous children's' playgrounds and slides. The park is close to the Queens Museum, Queens Zoo, and Citi Field, as well as right next to Arthur Ashe stadium, so there are plenty of options to choose from if you feel like going elsewhere.
Queens, NY offers more cultural and family fun opportunities than any other borough of the city, and is by far the best place to make a home in NYC. It's also far more affordable than you'd expect, as you can see above. Once you find the perfect home in Queens, make sure to protect it with Queens, NY renters insurance. For just a few cents a day, you can protect the things and the people who are important to you. It only takes a few minutes, so call or click today to get protected with Effective Coverage, the Queens, NY Renters insurance experts.