text:
require 'spec_helper'

describe Sendgrid::Web::Parameters do

  describe 'is based on Hash' do
    subject { described_class }
    its(:superclass) { should eql(Hash) }
  end

  describe 'cleanses when setting values' do
    context DateTime do
      before { subject[:provider] = DateTime.new(2013, 01, 23) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context Time do
      before { subject[:provider] = Time.new(2013, 01, 23) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context true do
      before { subject[:provider] = true }
      it 'converts to "true"' do
        subject[:provider].should eql('true')
      end
    end
    context false do
      before { subject[:provider] = false }
      it 'converts to "false"' do
        subject[:provider].should eql('false')
      end
    end
  end

  describe 'cleanses when merging' do
    context DateTime do
      before { subject.merge!(provider: DateTime.new(2013, 01, 23)) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context Time do
      before { subject.merge!(provider: Time.new(2013, 01, 23)) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context true do
      before { subject.merge!(provider: true) }
      it 'converts to "true"' do
        subject[:provider].should eql('true')
      end
    end
    context false do
      before { subject.merge!(provider: false) }
      it 'converts to "false"' do
        subject[:provider].should eql('false')
      end
    end
    it 'into new Parameters' do
      params = subject.merge(provider: true)
      params.should be_instance_of(Sendgrid::Web::Parameters)
      params[:provider].should eql('true')
      params.object_id.should_not eql(subject.object_id)
    end
  end

  describe 'cleanses when replacing' do
    context DateTime do
      before { subject.replace(provider: DateTime.new(2013, 01, 23)) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context Time do
      before { subject.replace(provider: Time.new(2013, 01, 23)) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context true do
      before { subject.replace(provider: true) }
      it 'converts to "true"' do
        subject[:provider].should eql('true')
      end
    end
    context false do
      before { subject.replace(provider: false) }
      it 'converts to "false"' do
        subject[:provider].should eql('false')
      end
    end
  end

  describe 'cleanses when storing' do
    context DateTime do
      before { subject.store(:provider, DateTime.new(2013, 01, 23)) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context Time do
      before { subject.store(:provider, Time.new(2013, 01, 23)) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context true do
      before { subject.store(:provider, true) }
      it 'converts to "true"' do
        subject[:provider].should eql('true')
      end
    end
    context false do
      before { subject.store(:provider, false) }
      it 'converts to "false"' do
        subject[:provider].should eql('false')
      end
    end
  end

  describe 'cleanses when updating' do
    context DateTime do
      before { subject.update(provider: DateTime.new(2013, 01, 23)) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context Time do
      before { subject.update(provider: Time.new(2013, 01, 23)) }
      it 'converts to YYYY-MM-DD format' do
        subject[:provider].should eql('2013-01-23')
      end
    end
    context true do
      before { subject.update(provider: true) }
      it 'converts to "true"' do
        subject[:provider].should eql('true')
      end
    end
    context false do
      before { subject.update(provider: false) }
      it 'converts to "false"' do
        subject[:provider].should eql('false')
      end
    end
  end

end


text:
package org.apache.hadoop.hbase;

import org.apache.yetus.audience.InterfaceAudience;

@InterfaceAudience.Private
class KeyValueBuilder extends ExtendedCellBuilderImpl {

  @Override
  protected ExtendedCell innerBuild() {
    KeyValue kv = new KeyValue(row, rOffset, rLength,
            family, fOffset, fLength,
            qualifier, qOffset, qLength,
            timestamp, type,
            value, vOffset, vLength,
        tags, tagsOffset, tagsLength);
    kv.setSequenceId(seqId);
    return kv;
  }
}

redpajama_set_name: RedPajamaGithub


text:
Swift-Roster
============

Code Fellows iOS class roster app written in Swift


text:
"""The ants_gui module implements a GUI for Ants vs. SomeBees.

You should not feel that you need to read and understand this file, because all
of the game logic is instead contained within ants.py.  We have provided some
comments so that interested students can extend the graphics.

=== Optional reading beyond this point ===

The GUI for this game has a fixed layout specified by a series of global names
that are constant throughout execution.  From a design perspective, global
names are a fine solution for holding constants; most problems associated with
global variables arise in programs that assign to global names.

The GUI layout itself is divided into a control panel that lists all
implemented ants and a play area populated with places.

The Hive is handled as a special case so that the player can visually inspect
how many bees remain in the hive.
"""

import ants
import graphics
from graphics import shift_point
from ucb import *
from math import pi
import math
import os
import random

STRATEGY_SECONDS = 3
INSECT_FILES = {'Worker': 'img/ant_harvester.gif',
                'Thrower': 'img/ant_thrower.gif',
                'Long': 'img/ant_longthrower.gif',
                'Short': 'img/ant_shortthrower.gif',
                'Harvester': 'img/ant_harvester.gif',
                'Fire': 'img/ant_fire.gif',
                'Bush': 'img/ant_weeds.gif',
                'Hungry': 'img/ant_hungry.gif',
                'Slow': 'img/ant_freeze.gif',
                'Stun': 'img/ant_stun.gif',
                'Ninja': 'img/ant_ninja.gif',
                'Wall': 'img/ant_wall.gif',
                'Scuba': 'img/ant_scuba.gif',
                'Bee': 'img/bee.gif',
                'Remover': 'img/remover.gif'}
TUNNEL_FILE = 'img/tunnel.gif'
ANT_IMAGE_WIDTH = 66
ANT_IMAGE_HEIGHT = 71
BEE_IMAGE_WIDTH = 58
PANEL_PADDING = (2, 4)
PLACE_PADDING = (10, 10)
PLACE_POS = (40, 180)
PANEL_POS = (20, 40)
CRYPT = 650
MESSAGE_POS = (120, 20)
HIVE_HEIGHT = 300
PLACE_MARGIN = 10
LEAF_START_OFFSET = (30, 30)
LEAF_END_OFFSET = (50, 30)
LEAF_COLORS = {'Thrower': 'ForestGreen',
               'Short': 'Green',
               'Long': 'DarkGreen',
               'Slow': 'LightBlue',
               'Stun': 'Red',
               'Scuba': 'Blue'}

class AntsGUI(object):
    """GUI-based interactive strategy that logs all colony updates."""

    def __init__(self):
        self.initialized = False

    def initialize_colony_graphics(self, colony):
        """Create canvas, control panel, places, and labels."""
        self.initialized = True
        self.canvas = graphics.Canvas()
        self.food_text = self.canvas.draw_text('Food: 1  Time: 0', (20, 20))
        self.ant_text = self.canvas.draw_text('Ant selected: None', (20, 140))
        self._click_rectangles = list()
        self._init_control_panel(colony)
        self._init_places(colony)

        start_text = self.canvas.draw_text('CLICK TO START', MESSAGE_POS)
        self.canvas.wait_for_click()
        self.canvas.clear(start_text)

    def _init_control_panel(self, colony):
        """Construct the control panel of available ant types."""
        self.ant_type_selected = None
        self.ant_type_frames = []  # rectangle ids of frames.
        panel_pos = PANEL_POS
        for name, ant_type in colony.ant_types.items():
            width = ANT_IMAGE_WIDTH + 2 * PANEL_PADDING[0]
            height = ANT_IMAGE_HEIGHT + 6 + 2 * PANEL_PADDING[1]
            def on_click(colony, frame, name=name):
                self.ant_type_selected = name
                self._update_control_panel(colony)

            frame = self.add_click_rect(panel_pos, width, height, on_click)
            self.ant_type_frames.append((name, frame))
            img_pos = shift_point(panel_pos, PANEL_PADDING)
            self.canvas.draw_image(img_pos, INSECT_FILES[name])
            cost_pos = shift_point(panel_pos, (width / 2, ANT_IMAGE_HEIGHT + 4
                + PANEL_PADDING[1]))
            food_str = str(ant_type.food_cost)
            self.canvas.draw_text(food_str, cost_pos, anchor="center")
            panel_pos = shift_point(panel_pos, (width + 2, 0))

    def _init_places(self, colony):
        """Construct places in the play area."""
        self.place_points = dict()
        # self.images: place_name -> insect instance -> image id
        self.images = {'AntQueen': dict()}
        place_pos = PLACE_POS
        width = BEE_IMAGE_WIDTH + 2 * PLACE_PADDING[0]
        height = ANT_IMAGE_HEIGHT + 2 * PLACE_PADDING[1]
        rows = 0
        for name, place in colony.places.items():
            if place.name == 'Hive':
                continue  # Handled as a special case later
            if place.exit.name == 'AntQueen':
                row_offset = (0, rows * (height + PLACE_MARGIN))
                place_pos = shift_point(PLACE_POS, row_offset)
                rows += 1
            def on_click(colony, frame, name=name):
                ant_type = self.ant_type_selected
                existing_ant = colony.places[name].ant
                if ant_type is 'Remover':
                    if existing_ant is not None:
                        print("colony.remove_ant('{0}')".format(name))
                        colony.remove_ant(name)
                        self._update_places(colony)
                elif ant_type is not None:
                    try:
                        print("colony.deploy_ant('{0}', '{1}')".format(name,
                                                                       ant_type))
                        colony.deploy_ant(name, ant_type)
                        self._update_places(colony)
                    except Exception as e:
                        print(e)
            color = 'Blue' if place.name.startswith('water') else 'White'
            frame = self.add_click_rect(place_pos, width, height, on_click,
                                             color=color)
            self.canvas.draw_image(place_pos, TUNNEL_FILE)
            self.place_points[name] = place_pos
            self.images[name] = dict()
            place_pos = shift_point(place_pos, (width + PLACE_MARGIN, 0))

        # Hive
        self.images[colony.hive.name] = dict()
        self.place_points[colony.hive.name] = (place_pos[0] + width,
                                               HIVE_HEIGHT)
        for bee in colony.hive.bees:
            self._draw_insect(bee, colony.hive.name, True)

    def add_click_rect(self, pos, width, height, on_click, color='White'):
        """Construct a rectangle that can be clicked."""
        frame_points = graphics.rectangle_points(pos, width, height)
        frame = self.canvas.draw_polygon(frame_points, fill_color=color)
        self._click_rectangles.append((pos, width, height, frame, on_click))
        return frame

    def strategy(self, colony):
        """The strategy function is called by the ants.AntColony each turn."""
        if not self.initialized:
            self.initialize_colony_graphics(colony)
        elapsed = 0  # Physical time elapsed this turn
        while elapsed < STRATEGY_SECONDS:
            self._update_control_panel(colony)
            self._update_places(colony)
            msg = 'Food: {0}  Time: {1}'.format(colony.food, colony.time)
            self.canvas.edit_text(self.food_text, text=msg)
            pos, el = self.canvas.wait_for_click(STRATEGY_SECONDS - elapsed)
            elapsed += el
            if pos is not None:
                self._interpret_click(pos, colony)

        # Throw leaves at the end of the turn
        has_ant = lambda a: hasattr(a, 'ant') and a.ant
        for ant in colony.ants + [a.ant for a in colony.ants if has_ant(a)]:
            if ant.name in LEAF_COLORS:
                self._throw(ant, colony)

    def _interpret_click(self, pos, colony):
        """Interpret a click position by finding its click rectangle."""
        x, y = pos
        for corner, width, height, frame, on_click in self._click_rectangles:
            cx, cy = corner
            if x >= cx and x <= cx + width and y >= cy and y <= cy + height:
                on_click(colony, frame)

    def _update_control_panel(self, colony):
        """Reflect the game state in the control panel."""
        for name, frame in self.ant_type_frames:
            cost = colony.ant_types[name].food_cost
            color = 'White'
            if cost > colony.food:
                color = 'Gray'
            elif name == self.ant_type_selected:
                color = 'Blue'
                msg = 'Ant selected: {0}'.format(name)
                self.canvas.edit_text(self.ant_text, text=msg)
            self.canvas._canvas.itemconfigure(frame, fill=color)

    def _update_places(self, colony):
        """Reflect the game state in the play area.

        This function handles several aspects of the game:
          - Adding Ant images for newly placed ants
          - Moving Bee images for bees that have advanced
          - Moving insects out of play when they have expired
        """
        for name, place in colony.places.items():
            if place.name == 'Hive':
                continue
            current = self.images[name].keys()

            # Add/move missing insects
            if place.ant is not None:
                if hasattr(place.ant, 'container') and place.ant.container \
                    and place.ant.ant and place.ant.ant not in current:
                    container = self.images[name][place.ant]
                    self._draw_insect(place.ant.ant, name, behind=container)
                if place.ant not in current:
                    self._draw_insect(place.ant, name)
            for bee in place.bees:
                if bee not in current:
                    for other_place in colony.places.values():
                        if other_place.exit is place:
                            break
                    else:
                        other_place = colony.hive
                    image = self.images[other_place.name].pop(bee)
                    pos = shift_point(self.place_points[name], PLACE_PADDING)
                    self.canvas.slide_shape(image, pos, STRATEGY_SECONDS)
                    self.images[name][bee] = image

            # Remove expired insects
            valid_insects = set(place.bees + [place.ant])
            if place.ant is not None and hasattr(place.ant, 'container') and \
                place.ant.container:
                valid_insects.add(place.ant.ant)
            for insect in current - valid_insects:
                if not place.exit or insect not in self.images[place.exit.name]:
                    image = self.images[name].pop(insect)
                    pos = (self.place_points[name][0], CRYPT)
                    self.canvas.slide_shape(image, pos, STRATEGY_SECONDS)

    def _draw_insect(self, insect, place_name, random_offset=False, behind=0):
        """Draw an insect and store the ID of its image."""
        image_file = INSECT_FILES[insect.name]
        pos = shift_point(self.place_points[place_name], PLACE_PADDING)
        if random_offset:
            pos = shift_point(pos, (random.randint(-10, 10), random.randint(-50, 50)))
        image = self.canvas.draw_image(pos, image_file, behind=behind)
        self.images[place_name][insect] = image

    def _throw(self, ant, colony):
        """Animate a leaf thrown at a Bee."""
        bee = ant.nearest_bee(colony.hive)  # nearest_bee logic from ants.py
        if bee:
            start = shift_point(self.place_points[ant.place.name], LEAF_START_OFFSET)
            end = shift_point(self.place_points[bee.place.name], LEAF_END_OFFSET)
            animate_leaf(self.canvas, start, end, color=LEAF_COLORS[ant.name])

def leaf_coords(pos, angle, length):
    """Return the coordinates of a leaf polygon."""
    angles = [angle - pi, angle - pi/2, angle, angle + pi/2]
    distances = [length/3, length/2, length, length/2]
    return [graphics.translate_point(pos, a, d) for a, d in zip(angles, distances)]

def animate_leaf(canvas, start, end, duration=0.3, color='ForestGreen'):
    """Define the animation frames for a thrown leaf."""
    length = 40
    leaf = canvas.draw_polygon(leaf_coords(start, 0, length),
            color='DarkGreen', fill_color=color, smooth=1)
    num_frames = duration / graphics.FRAME_TIME
    increment = tuple([(e-s) / num_frames for s, e in zip(start, end)])
    def points_fn(frame_count):
        nonlocal start
        angle = pi / 8 * frame_count
        cs = leaf_coords(start, angle, length)
        start = shift_point(start, increment)
        return cs
    canvas.animate_shape(leaf, duration, points_fn)
    canvas._canvas.after(int(1000*duration) + 1, lambda: canvas.clear(leaf))

@main
def run(*args):
    ants.start_with_strategy(args, AntsGUI().strategy)


text:
package linebot

import (
	"context"
	"io"
	"net/http"
)

// NewRawCall method
func (client *Client) NewRawCall(method string, endpoint string) (*RawCall, error) {
	req, err := http.NewRequest(method, client.url(client.endpointBase, endpoint), nil)
	if err != nil {
		return nil, err
	}
	return &RawCall{
		c: client,
		req: req,
	}, nil
}

// NewRawCallWithBody method
func (client *Client) NewRawCallWithBody(method string, endpoint string, body io.Reader) (*RawCall, error) {
	req, err := http.NewRequest(method, client.url(client.endpointBase, endpoint), body)
	if err != nil {
		return nil, err
	}
	return &RawCall{
		c: client,
		req: req,
	}, nil
}

// RawCall type
type RawCall struct {
	c   *Client
	ctx context.Context

	req         *http.Request
}

// AddHeader method
func (call *RawCall) AddHeader(key string, value string) {
	call.req.Header.Add(key, value)
}

// WithContext method
func (call *RawCall) WithContext(ctx context.Context) *RawCall {
	call.ctx = ctx
	return call
}

// Do method. Callee must close response object.
func (call *RawCall) Do() (*http.Response, error) {
	return call.c.do(call.ctx, call.req)
}


text:
package com.intellij.facet;

import com.intellij.openapi.vfs.VirtualFile;
import org.jetbrains.annotations.NotNull;

import java.util.Collection;

/**
 * @author nik
 */
public interface FacetRootsProvider {

  @NotNull
  Collection<VirtualFile> getFacetRoots();

}


text:
package org.osgi.impl.service.dal.functions;

import java.math.BigDecimal;
import java.util.Dictionary;
import java.util.HashMap;
import java.util.Map;

import org.osgi.framework.BundleContext;
import org.osgi.impl.service.dal.PropertyMetadataImpl;
import org.osgi.impl.service.dal.SimulatedFunction;
import org.osgi.service.dal.Function;
import org.osgi.service.dal.FunctionData;
import org.osgi.service.dal.PropertyMetadata;
import org.osgi.service.dal.functions.MultiLevelSensor;
import org.osgi.service.dal.functions.data.LevelData;
import org.osgi.service.event.EventAdmin;
import org.osgi.util.tracker.ServiceTracker;

/**
 * Simulated {@code MultiLevelSensor}.
 */
public final class SimulatedMultiLevelSensor extends SimulatedFunction implements MultiLevelSensor { // NO_UCD

	private static final Map<String,Object>	PROPERTY_METADATA;
	private static final Map<String,Object>	OPERATION_METADATA	= null;
	private static final BigDecimal	VALUE				= new BigDecimal(1);
	private static final LevelData	LEVEL_DATA			= new LevelData(Long.MIN_VALUE, null, VALUE, null);

	static {
		Map<String,Object> metadata = new HashMap<>();
		metadata.put(
				PropertyMetadata.ACCESS,
				Integer.valueOf(
						PropertyMetadata.ACCESS_READABLE |
								PropertyMetadata.ACCESS_EVENTABLE));
		PropertyMetadata propMetadata = new PropertyMetadataImpl(
				metadata, // metadata
				null, // step
				new FunctionData[] {LEVEL_DATA}, // enumValues
				LEVEL_DATA, // minValue
				LEVEL_DATA); // maxValue
		PROPERTY_METADATA = new HashMap<>();
		PROPERTY_METADATA.put(PROPERTY_DATA, propMetadata);
	}

	/**
	 * Constructs a new instance with the specified arguments.
	 *
	 * @param functionProps The service properties.
	 * @param bc The bundle context used to register the service.
	 * @param eventAdminTracker The event admin service tracker to post events.
	 */
	public SimulatedMultiLevelSensor(Dictionary<String,Object> functionProps,
			BundleContext bc,
			ServiceTracker<EventAdmin,EventAdmin> eventAdminTracker) {
		super(PROPERTY_METADATA, OPERATION_METADATA, eventAdminTracker);
		super.register(
				new String[] {Function.class.getName(), MultiLevelSensor.class.getName()},
				addPropertyAndOperationNames(functionProps), bc);
	}

	private static Dictionary<String,Object> addPropertyAndOperationNames(
			Dictionary<String,Object> functionProps) {
		functionProps.put(
				SERVICE_PROPERTY_NAMES,
				new String[] {PROPERTY_DATA});
		return functionProps;
	}

	@Override
	public LevelData getData() {
		return new LevelData(System.currentTimeMillis(), null, VALUE, null);
	}

	@Override
	public void publishEvent(String propName) {
		if (!PROPERTY_DATA.equals(propName)) {
			throw new IllegalArgumentException("The property is not supported: " + propName);
		}
		super.postEvent(propName, getData());
	}
}


text:
#ifdef __cplusplus
extern "C" {
#endif

    // Returns true if iPod is playing.
    bool IsIPodMusicPlaying ();

    // Case sensitive Unity game object and method registering for callbacks.
    void RegisterUnityIPodCallbackListener (const char* gameObject, const char* method);

#ifdef __cplusplus
}
#endif


text:
<?php

declare(strict_types=1);

namespace Sylius\Bundle\GridBundle\Twig;

use Sylius\Bundle\GridBundle\Templating\Helper\GridHelper;

final class GridExtension extends \Twig_Extension
{
    /** @var GridHelper */
    private $gridHelper;

    public function __construct(GridHelper $gridHelper)
    {
        $this->gridHelper = $gridHelper;
    }

    /**
     * {@inheritdoc}
     */
    public function getFunctions(): array
    {
        return [
            new \Twig_Function('sylius_grid_render', [$this->gridHelper, 'renderGrid'], ['is_safe' => ['html']]),
            new \Twig_Function('sylius_grid_render_field', [$this->gridHelper, 'renderField'], ['is_safe' => ['html']]),
            new \Twig_Function('sylius_grid_render_action', [$this->gridHelper, 'renderAction'], ['is_safe' => ['html']]),
            new \Twig_Function('sylius_grid_render_filter', [$this->gridHelper, 'renderFilter'], ['is_safe' => ['html']]),
        ];
    }
}


text:
@interface UITabBarController (Statistic)

@end


text:
@interface QMChatIncomingLinkPreviewCell : QMChatBaseLinkPreviewCell

@end


text:
using System;
using System.Collections.Generic;
using System.Text;

namespace GenericSwapMethod
{
    public class Box<T>
    {
        public Box(List<T> items)
        {
            this.Items = items;
        }

        public List<T> Items { get; set; }

        public void Swap(int index1, int index2)
        {
            T temp = this.Items[index1];
            this.Items[index1] = this.Items[index2];
            this.Items[index2] = temp;
        }

        public override string ToString()
        {
            StringBuilder sb = new StringBuilder();

            foreach (var item in this.Items)
            {
                sb.AppendLine($"{item.GetType().FullName}: {item}");
            }
            return sb.ToString().TrimEnd();
        }
    }
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<chapter
  version="5.0"
  xml:id="performance"
  xmlns="http://docbook.org/ns/docbook"
  xmlns:xlink="http://www.w3.org/1999/xlink"
  xmlns:xi="http://www.w3.org/2001/XInclude"
  xmlns:svg="http://www.w3.org/2000/svg"
  xmlns:m="http://www.w3.org/1998/Math/MathML"
  xmlns:html="http://www.w3.org/1999/xhtml"
  xmlns:db="http://docbook.org/ns/docbook">

  <title>Apache HBase Performance Tuning</title>

    <title>Operating System</title>

      <title>Memory</title>
      <para>RAM, RAM, RAM. Don't starve HBase.</para>

      <title>64-bit</title>
      <para>Use a 64-bit platform (and 64-bit JVM).</para>

      <title>Swapping</title>
      <para>Watch out for swapping. Set swappiness to 0.</para>

    <title>Network</title>
    <para> Perhaps the most important factor in avoiding network issues degrading Hadoop and HBase
      performance is the switching hardware that is used, decisions made early in the scope of the
      project can cause major problems when you double or triple the size of your cluster (or more). </para>
    <para> Important items to consider: <itemizedlist>
        <listitem>
          <para>Switching capacity of the device</para>
        </listitem>
        <listitem>
          <para>Number of systems connected</para>
        </listitem>
        <listitem>
          <para>Uplink capacity</para>
        </listitem>
      </itemizedlist>
    </para>

      <title>Single Switch</title>
      <para>The single most important factor in this configuration is that the switching capacity of
        the hardware is capable of handling the traffic which can be generated by all systems
        connected to the switch. Some lower priced commodity hardware can have a slower switching
        capacity than could be utilized by a full switch. </para>

      <title>Multiple Switches</title>
      <para>Multiple switches are a potential pitfall in the architecture. The most common
        configuration of lower priced hardware is a simple 1Gbps uplink from one switch to another.
        This often overlooked pinch point can easily become a bottleneck for cluster communication.
        Especially with MapReduce jobs that are both reading and writing a lot of data the
        communication across this uplink could be saturated. </para>
      <para>Mitigation of this issue is fairly simple and can be accomplished in multiple ways: </para>
      <itemizedlist>
        <listitem>
          <para>Use appropriate hardware for the scale of the cluster which you're attempting to
            build.</para>
        </listitem>
        <listitem>
          <para>Use larger single switch configurations i.e. single 48 port as opposed to 2x 24
            port</para>
        </listitem>
        <listitem>
          <para>Configure port trunking for uplinks to utilize multiple interfaces to increase cross
            switch bandwidth.</para>
        </listitem>
      </itemizedlist>

      <title>Multiple Racks</title>
      <para>Multiple rack configurations carry the same potential issues as multiple switches, and
        can suffer performance degradation from two main areas: </para>
      <itemizedlist>
        <listitem>
          <para>Poor switch capacity performance</para>
        </listitem>
        <listitem>
          <para>Insufficient uplink to another rack</para>
        </listitem>
      </itemizedlist>
      <para>If the the switches in your rack have appropriate switching capacity to handle all the
        hosts at full speed, the next most likely issue will be caused by homing more of your
        cluster across racks. The easiest way to avoid issues when spanning multiple racks is to use
        port trunking to create a bonded uplink to other racks. The downside of this method however,
        is in the overhead of ports that could potentially be used. An example of this is, creating
        an 8Gbps port channel from rack A to rack B, using 8 of your 24 ports to communicate between
        racks gives you a poor ROI, using too few however can mean you're not getting the most out
        of your cluster. </para>
      <para>Using 10Gbe links between racks will greatly increase performance, and assuming your
        switches support a 10Gbe uplink or allow for an expansion card will allow you to save your
        ports for machines as opposed to uplinks. </para>

      <title>Network Interfaces</title>
      <para>Are all the network interfaces functioning correctly? Are you sure? See the
        Troubleshooting Case Study in <xref
          linkend="casestudies.slownode" />. </para>

    <title>Java</title>

      <title>The Garbage Collector and Apache HBase</title>

        <title>Long GC pauses</title>

        <para xml:id="mslab">In his presentation, <link
            xlink:href="http://www.slideshare.net/cloudera/hbase-hug-presentation">Avoiding Full GCs
            with MemStore-Local Allocation Buffers</link>, Todd Lipcon describes two cases of
          stop-the-world garbage collections common in HBase, especially during loading; CMS failure
          modes and old generation heap fragmentation brought. To address the first, start the CMS
          earlier than default by adding -XX:CMSInitiatingOccupancyFraction and setting
          it down from defaults. Start at 60 or 70 percent (The lower you bring down the threshold,
          the more GCing is done, the more CPU used). To address the second fragmentation issue,
          Todd added an experimental facility, <indexterm><primary>MSLAB</primary></indexterm>, that
          must be explicitly enabled in Apache HBase 0.90.x (Its defaulted to be on in Apache 0.92.x
          HBase). See hbase.hregion.memstore.mslab.enabled to true in your
            <classname>Configuration</classname>. See the cited slides for background and detail.
          The latest jvms do better regards fragmentation so make sure you are running a recent
          release. Read down in the message, <link
            xlink:href="http://osdir.com/ml/hotspot-gc-use/2011-11/msg00002.html">Identifying
            concurrent mode failures caused by fragmentation</link>. Be aware that when enabled,
          each MemStore instance will occupy at least an MSLAB instance of memory. If you have
          thousands of regions or lots of regions each with many column families, this allocation of
          MSLAB may be responsible for a good portion of your heap allocation and in an extreme case
          cause you to OOME. Disable MSLAB in this case, or lower the amount of memory it uses or
          float less regions per server. </para>
        <para>If you have a write-heavy workload, check out <link
            xlink:href="https://issues.apache.org/jira/browse/HBASE-8163">HBASE-8163
            MemStoreChunkPool: An improvement for JAVA GC when using MSLAB</link>. It describes
          configurations to lower the amount of young GC during write-heavy loadings. If you do not
          have HBASE-8163 installed, and you are trying to improve your young GC times, one trick to
          consider -- courtesy of our Liang Xie -- is to set the GC config
            <varname>-XX:PretenureSizeThreshold</varname> in <filename>hbase-env.sh</filename> to be
          just smaller than the size of <varname>hbase.hregion.memstore.mslab.chunksize</varname> so
          MSLAB allocations happen in the tenured space directly rather than first in the young gen.
          You'd do this because these MSLAB allocations are going to likely make it to the old gen
          anyways and rather than pay the price of a copies between s0 and s1 in eden space followed
          by the copy up from young to old gen after the MSLABs have achieved sufficient tenure,
          save a bit of YGC churn and allocate in the old gen directly. </para>
        <para>For more information about GC logs, see <xref
            linkend="trouble.log.gc" />. </para>
    <para>Consider also enabling the offheap Block Cache.  This has been shown to mitigate
        GC pause times.  See <xref linkend="block.cache" /></para>

    <title>HBase Configurations</title>

    <para>See <xref
        linkend="recommended_configurations" />.</para>

      <title>Managing Compactions</title>

      <para>For larger systems, managing <link
      linkend="disable.splitting">compactions and splits</link> may be
      something you want to consider.</para>

        <title><varname>hbase.regionserver.handler.count</varname></title>
        <para>See <xref linkend="hbase.regionserver.handler.count"/>.
	    </para>

        <title><varname>hfile.block.cache.size</varname></title>
        <para>See <xref linkend="hfile.block.cache.size"/>.
        A memory setting for the RegionServer process.
        </para>

      <title>Prefetch Option for Blockcache</title>
      <para><link xlink:href="https://issues.apache.org/jira/browse/HBASE-9857">HBASE-9857</link>
        adds a new option to prefetch HFile contents when opening the blockcache, if a columnfamily
        or regionserver property is set. This option is available for HBase 0.98.3 and later. The
        purpose is to warm the blockcache as rapidly as possible after the cache is opened, using
        in-memory table data, and not counting the prefetching as cache misses. This is great for
        fast reads, but is not a good idea if the data to be preloaded will not fit into the
        blockcache. It is useful for tuning the IO impact of prefetching versus the time before all
        data blocks are in cache. </para>
      <para>To enable prefetching on a given column family, you can use HBase Shell or use the
        API.</para>
      <example>
        <title>Enable Prefetch Using HBase Shell</title>
        <screen>hbase> create 'MyTable', { NAME => 'myCF', PREFETCH_BLOCKS_ON_OPEN => 'true' }</screen>
      </example>
      <example>
        <title>Enable Prefetch Using the API</title>
        <programlisting language="java">
// ...
HTableDescriptor tableDesc = new HTableDescriptor("myTable");
HColumnDescriptor cfDesc = new HColumnDescriptor("myCF");
cfDesc.setPrefetchBlocksOnOpen(true);
tableDesc.addFamily(cfDesc);
// ...
        </programlisting>
      </example>
      <para>See the API documentation for <link
          xlink:href="https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/io/hfile/CacheConfig.html"
          >CacheConfig</link>.</para>

        <title><varname>hbase.regionserver.global.memstore.size</varname></title>
        <para>See <xref linkend="hbase.regionserver.global.memstore.size"/>.
        This memory setting is often adjusted for the RegionServer process depending on needs.
        </para>

        <title><varname>hbase.regionserver.global.memstore.size.lower.limit</varname></title>
        <para>See <xref linkend="hbase.regionserver.global.memstore.size.lower.limit"/>.
        This memory setting is often adjusted for the RegionServer process depending on needs.
        </para>

        <title><varname>hbase.hstore.blockingStoreFiles</varname></title>
        <para>See <xref linkend="hbase.hstore.blockingStoreFiles"/>.
        If there is blocking in the RegionServer logs, increasing this can help.
        </para>

        <title><varname>hbase.hregion.memstore.block.multiplier</varname></title>
        <para>See <xref linkend="hbase.hregion.memstore.block.multiplier"/>.
        If there is enough RAM, increasing this can help.
        </para>

        <title><varname>hbase.regionserver.checksum.verify</varname></title>
        <para>Have HBase write the checksum into the datablock and save
        having to do the checksum seek whenever you read.</para>

        <para>See <xref linkend="hbase.regionserver.checksum.verify"/>,
        <xref linkend="hbase.hstore.bytes.per.checksum"/> and <xref linkend="hbase.hstore.checksum.algorithm"/>
        For more information see the
        release note on <link xlink:href="https://issues.apache.org/jira/browse/HBASE-5074">HBASE-5074 support checksums in HBase block cache</link>.
        </para>

      <title>Tuning callQueue Options</title>
      <para><link xlink:href="https://issues.apache.org/jira/browse/HBASE-11355">HBASE-11355</link>
        introduces several callQueue tuning mechanisms which can increase performance. See the JIRA
        for some benchmarking information.</para>
      <itemizedlist>
        <listitem>
          <para>To increase the number of callqueues, set
              <option>hbase.ipc.server.num.callqueue</option> to a value greater than
              <literal>1</literal>.</para>
        </listitem>
        <listitem>
          <para>To split the callqueue into separate read and write queues, set
              hbase.ipc.server.callqueue.read.ratio to a value between
              <literal>0</literal> and <literal>1</literal>. This factor weights the queues toward
            writes (if below .5) or reads (if above .5). Another way to say this is that the factor
            determines what percentage of the split queues are used for reads. The following
            examples illustrate some of the possibilities. Note that you always have at least one
            write queue, no matter what setting you use.</para>
          <itemizedlist>
            <listitem>
              <para>The default value of <literal>0</literal> does not split the queue.</para>
            </listitem>
            <listitem>
              <para>A value of <literal>.3</literal> uses 30% of the queues for reading and 60% for
                writing. Given a value of <literal>10</literal> for
                  <option>hbase.ipc.server.num.callqueue</option>, 3 queues would be used for reads
                and 7 for writes.</para>
            </listitem>
            <listitem>
              <para>A value of <literal>.5</literal> uses the same number of read queues and write
                queues. Given a value of <literal>10</literal> for
                <option>hbase.ipc.server.num.callqueue</option>, 5 queues would be used for reads
                and 5 for writes.</para>
            </listitem>
            <listitem>
              <para>A value of <literal>.6</literal> uses 60% of the queues for reading and 30% for
                reading. Given a value of <literal>10</literal> for
                <option>hbase.ipc.server.num.callqueue</option>, 7 queues would be used for reads
                and 3 for writes.</para>
            </listitem>
            <listitem>
              <para>A value of <literal>1.0</literal> uses one queue to process write requests, and
                all other queues process read requests. A value higher than <literal>1.0</literal>
                has the same effect as a value of <literal>1.0</literal>. Given a value of
                  <literal>10</literal> for <option>hbase.ipc.server.num.callqueue</option>, 9
                queues would be used for reads and 1 for writes.</para>
            </listitem>
          </itemizedlist>
        </listitem>
        <listitem>
          <para>You can also split the read queues so that separate queues are used for short reads
            (from Get operations) and long reads (from Scan operations), by setting the
              <option>hbase.ipc.server.callqueue.scan.ratio</option> option. This option is a factor
            between 0 and 1, which determine the ratio of read queues used for Gets and Scans. More
            queues are used for Gets if the value is below <literal>.5</literal> and more are used
            for scans if the value is above <literal>.5</literal>. No matter what setting you use,
            at least one read queue is used for Get operations.</para>
          <itemizedlist>
            <listitem>
              <para>A value of <literal>0</literal> does not split the read queue.</para>
            </listitem>
            <listitem>
              <para>A value of <literal>.3</literal> uses 60% of the read queues for Gets and 30%
                for Scans. Given a value of <literal>20</literal> for
                  <option>hbase.ipc.server.num.callqueue</option> and a value of <literal>.5
                </literal> for <option>hbase.ipc.server.callqueue.read.ratio</option>, 10 queues
                would be used for reads, out of those 10, 7 would be used for Gets and 3 for
                Scans.</para>
            </listitem>
            <listitem>
              <para>A value of <literal>.5</literal> uses half the read queues for Gets and half for
                Scans. Given a value of <literal>20</literal> for
                  <option>hbase.ipc.server.num.callqueue</option> and a value of <literal>.5
                </literal> for <option>hbase.ipc.server.callqueue.read.ratio</option>, 10 queues
                would be used for reads, out of those 10, 5 would be used for Gets and 5 for
                Scans.</para>
            </listitem>
            <listitem>
              <para>A value of <literal>.6</literal> uses 30% of the read queues for Gets and 60%
                for Scans. Given a value of <literal>20</literal> for
                  <option>hbase.ipc.server.num.callqueue</option> and a value of <literal>.5
                </literal> for <option>hbase.ipc.server.callqueue.read.ratio</option>, 10 queues
                would be used for reads, out of those 10, 3 would be used for Gets and 7 for
                Scans.</para>
            </listitem>
            <listitem>
              <para>A value of <literal>1.0</literal> uses all but one of the read queues for Scans.
                Given a value of <literal>20</literal> for
                  <option>hbase.ipc.server.num.callqueue</option> and a value of <literal>.5
                </literal> for <option>hbase.ipc.server.callqueue.read.ratio</option>, 10 queues
                would be used for reads, out of those 10, 1 would be used for Gets and 9 for
                Scans.</para>
            </listitem>
          </itemizedlist>
        </listitem>
        <listitem>
          <para>You can use the new option
              <option>hbase.ipc.server.callqueue.handler.factor</option> to programmatically tune
            the number of queues:</para>
          <itemizedlist>
            <listitem>
              <para>A value of <literal>0</literal> uses a single shared queue between all the
                handlers.</para>
            </listitem>
            <listitem>
              <para>A value of <literal>1</literal> uses a separate queue for each handler.</para>
            </listitem>
            <listitem>
              <para>A value between <literal>0</literal> and <literal>1</literal> tunes the number
                of queues against the number of handlers. For instance, a value of
                  <literal>.5</literal> shares one queue between each two handlers.</para>
            </listitem>
          </itemizedlist>
          <para>Having more queues, such as in a situation where you have one queue per handler,
            reduces contention when adding a task to a queue or selecting it from a queue. The
            trade-off is that if you have some queues with long-running tasks, a handler may end up
            waiting to execute from that queue rather than processing another queue which has
            waiting tasks.</para>
        </listitem>
      </itemizedlist>
      <para>For these values to take effect on a given Region Server, the Region Server must be
        restarted. These parameters are intended for testing purposes and should be used
        carefully.</para>

    <title>ZooKeeper</title>
    <para>See <xref
        linkend="zookeeper" /> for information on configuring ZooKeeper, and see the part about
      having a dedicated disk. </para>

    <title>Schema Design</title>

      <title>Number of Column Families</title>
      <para>See <xref
          linkend="number.of.cfs" />.</para>

      <title>Key and Attribute Lengths</title>
      <para>See <xref
          linkend="keysize" />. See also <xref
          linkend="perf.compression.however" /> for compression caveats.</para>

      <title>Table RegionSize</title>
      <para>The regionsize can be set on a per-table basis via setFileSize on <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HTableDescriptor.html">HTableDescriptor</link>
        in the event where certain tables require different regionsizes than the configured default
        regionsize. </para>
      <para>See <xref
          linkend="ops.capacity.regions" /> for more information. </para>

      <title>Bloom Filters</title>
      <para>A Bloom filter, named for its creator, Burton Howard Bloom, is a data structure which is
        designed to predict whether a given element is a member of a set of data. A positive result
        from a Bloom filter is not always accurate, but a negative result is guaranteed to be
        accurate. Bloom filters are designed to be "accurate enough" for sets of data which are so
        large that conventional hashing mechanisms would be impractical. For more information about
        Bloom filters in general, refer to <link
          xlink:href="http://en.wikipedia.org/wiki/Bloom_filter" />.</para>
      <para>In terms of HBase, Bloom filters provide a lightweight in-memory structure to reduce the
        number of disk reads for a given Get operation (Bloom filters do not work with Scans) to only the StoreFiles likely to
        contain the desired Row. The potential performance gain increases with the number of
        parallel reads. </para>
      <para>The Bloom filters themselves are stored in the metadata of each HFile and never need to
        be updated. When an HFile is opened because a region is deployed to a RegionServer, the
        Bloom filter is loaded into memory. </para>
      <para>HBase includes some tuning mechanisms for folding the Bloom filter to reduce the size
        and keep the false positive rate within a desired range.</para>
      <para>Bloom filters were introduced in <link
          xlink:href="https://issues.apache.org/jira/browse/HBASE-1200">HBASE-1200</link>. Since
        HBase 0.96, row-based Bloom filters are enabled by default. (<link
          xlink:href="https://issues.apache.org/jira/browse/HBASE-8450">HBASE-</link>)</para>
      <para>For more information on Bloom filters in relation to HBase, see <xref
          linkend="blooms" /> for more information, or the following Quora discussion: <link
          xlink:href="http://www.quora.com/How-are-bloom-filters-used-in-HBase">How are bloom
          filters used in HBase?</link>. </para>

        <title>When To Use Bloom Filters</title>
        <para>Since HBase 0.96, row-based Bloom filters are enabled by default. You may choose to
          disable them or to change some tables to use row+column Bloom filters, depending on the
          characteristics of your data and how it is loaded into HBase.</para>

        <para>To determine whether Bloom filters could have a positive impact, check the value of
          blockCacheHitRatio in the RegionServer metrics. If Bloom filters are enabled, the value of
          blockCacheHitRatio should increase, because the Bloom filter is filtering out blocks that
          are definitely not needed. </para>
        <para>You can choose to enable Bloom filters for a row or for a row+column combination. If
          you generally scan entire rows, the row+column combination will not provide any benefit. A
          row-based Bloom filter can operate on a row+column Get, but not the other way around.
          However, if you have a large number of column-level Puts, such that a row may be present
          in every StoreFile, a row-based filter will always return a positive result and provide no
          benefit. Unless you have one column per row, row+column Bloom filters require more space,
          in order to store more keys. Bloom filters work best when the size of each data entry is
          at least a few kilobytes in size. </para>
        <para>Overhead will be reduced when your data is stored in a few larger StoreFiles, to avoid
          extra disk IO during low-level scans to find a specific row. </para>
        <para>Bloom filters need to be rebuilt upon deletion, so may not be appropriate in
          environments with a large number of deletions.</para>

        <title>Enabling Bloom Filters</title>
        <para>Bloom filters are enabled on a Column Family. You can do this by using the
          setBloomFilterType method of HColumnDescriptor or using the HBase API. Valid values are
            <literal>NONE</literal> (the default), <literal>ROW</literal>, or
            <literal>ROWCOL</literal>. See <xref
            linkend="bloom.filters.when" /> for more information on <literal>ROW</literal> versus
            <literal>ROWCOL</literal>. See also the API documentation for <link
            xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HColumnDescriptor.html">HColumnDescriptor</link>.</para>
          <para>The following example creates a table and enables a ROWCOL Bloom filter on the
            <literal>colfam1</literal> column family.</para>
        <screen>
hbase> <userinput>create 'mytable',{NAME => 'colfam1', BLOOMFILTER => 'ROWCOL'}</userinput>
        </screen>

        <title>Configuring Server-Wide Behavior of Bloom Filters</title>
        <para>You can configure the following settings in the <filename>hbase-site.xml</filename>.
        </para>
        <informaltable>
          <tgroup cols="3">

              <row>
                <entry>Parameter</entry>
                <entry>Default</entry>
                <entry>Description</entry>
              </row>

              <row>
                <entry><para>io.hfile.bloom.enabled</para></entry>
                <entry><para><literal>yes</literal></para></entry>
                <entry><para>Set to <literal>no</literal> to kill bloom filters server-wide if
                    something goes wrong</para></entry>
              </row>
              <row>
                <entry><para>io.hfile.bloom.error.rate</para></entry>
                <entry><para><literal>.01</literal></para></entry>
                <entry><para>The average false positive rate for bloom filters. Folding is used to
                  maintain the false positive rate. Expressed as a decimal representation of a
                  percentage.</para></entry>
              </row>
              <row>
                <entry><para>io.hfile.bloom.max.fold</para></entry>
                <entry><para><literal>7</literal></para></entry>
                <entry><para>The guaranteed maximum fold rate. Changing this setting should not be
                  necessary and is not recommended.</para></entry>
              </row>
              <row>
                <entry><para>io.storefile.bloom.max.keys</para></entry>
                <entry><para><literal>128000000</literal></para></entry>
                <entry><para>For default (single-block) Bloom filters, this specifies the maximum
                    number of keys.</para></entry>
              </row>
              <row>
                <entry><para>io.storefile.delete.family.bloom.enabled</para></entry>
                <entry><para><literal>true</literal></para></entry>
                <entry><para>Master switch to enable Delete Family Bloom filters and store them in
                  the StoreFile.</para></entry>
              </row>
              <row>
                <entry><para>io.storefile.bloom.block.size</para></entry>
                <entry><para><literal>65536</literal></para></entry>
                <entry><para>Target Bloom block size. Bloom filter blocks of approximately this size
                    are interleaved with data blocks.</para></entry>
              </row>
              <row>
                <entry><para>hfile.block.bloom.cacheonwrite</para></entry>
                <entry><para><literal>false</literal></para></entry>
                <entry><para>Enables cache-on-write for inline blocks of a compound Bloom filter.</para></entry>
              </row>

          </tgroup>
        </informaltable>

      <title>ColumnFamily BlockSize</title>
      <para>The blocksize can be configured for each ColumnFamily in a table, and this defaults to
        64k. Larger cell values require larger blocksizes. There is an inverse relationship between
        blocksize and the resulting StoreFile indexes (i.e., if the blocksize is doubled then the
        resulting indexes should be roughly halved). </para>
      <para>See <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HColumnDescriptor.html">HColumnDescriptor</link>
        and <xref
          linkend="store" />for more information. </para>

      <title>In-Memory ColumnFamilies</title>
      <para>ColumnFamilies can optionally be defined as in-memory. Data is still persisted to disk,
        just like any other ColumnFamily. In-memory blocks have the highest priority in the <xref
          linkend="block.cache" />, but it is not a guarantee that the entire table will be in
        memory. </para>
      <para>See <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HColumnDescriptor.html">HColumnDescriptor</link>
        for more information. </para>

      <title>Compression</title>
      <para>Production systems should use compression with their ColumnFamily definitions. See <xref
          linkend="compression" /> for more information. </para>

        <title>However...</title>
        <para>Compression deflates data <emphasis>on disk</emphasis>. When it's in-memory (e.g., in
          the MemStore) or on the wire (e.g., transferring between RegionServer and Client) it's
          inflated. So while using ColumnFamily compression is a best practice, but it's not going
          to completely eliminate the impact of over-sized Keys, over-sized ColumnFamily names, or
          over-sized Column names. </para>
        <para>See <xref
            linkend="keysize" /> on for schema design tips, and <xref
            linkend="keyvalue" /> for more information on HBase stores data internally. </para>

    <title>HBase General Patterns</title>

      <title>Constants</title>
      <para>When people get started with HBase they have a tendency to write code that looks like
        this:</para>
      <programlisting language="java">
Get get = new Get(rowkey);
Result r = htable.get(get);
byte[] b = r.getValue(Bytes.toBytes("cf"), Bytes.toBytes("attr"));  // returns current version of value
      </programlisting>
      <para>But especially when inside loops (and MapReduce jobs), converting the columnFamily and
        column-names to byte-arrays repeatedly is surprisingly expensive. It's better to use
        constants for the byte-arrays, like this:</para>
      <programlisting language="java">
public static final byte[] CF = "cf".getBytes();
public static final byte[] ATTR = "attr".getBytes();
...
Get get = new Get(rowkey);
Result r = htable.get(get);
byte[] b = r.getValue(CF, ATTR);  // returns current version of value
      </programlisting>

    <title>Writing to HBase</title>

      <title>Batch Loading</title>
      <para>Use the bulk load tool if you can. See <xref
          linkend="arch.bulk.load" />. Otherwise, pay attention to the below. </para>

      <title> Table Creation: Pre-Creating Regions </title>
      <para> Tables in HBase are initially created with one region by default. For bulk imports,
        this means that all clients will write to the same region until it is large enough to split
        and become distributed across the cluster. A useful pattern to speed up the bulk import
        process is to pre-create empty regions. Be somewhat conservative in this, because too-many
        regions can actually degrade performance. </para>
      <para>There are two different approaches to pre-creating splits. The first approach is to rely
        on the default HBaseAdmin strategy (which is implemented in
          Bytes.split)... </para>
      <programlisting language="java">
byte[] startKey = ...;   	// your lowest key
byte[] endKey = ...;   		// your highest key
int numberOfRegions = ...;	// # of regions to create
admin.createTable(table, startKey, endKey, numberOfRegions);
      </programlisting>
      <para>And the other approach is to define the splits yourself... </para>
      <programlisting language="java">
byte[][] splits = ...;   // create your own splits
admin.createTable(table, splits);
</programlisting>
      <para> See <xref linkend="rowkey.regionsplits"/> for issues related to understanding your
        keyspace and pre-creating regions. See <xref linkend="manual_region_splitting_decisions"/>
        for discussion on manually pre-splitting regions.</para>

      <title> Table Creation: Deferred Log Flush </title>
      <para> The default behavior for Puts using the Write Ahead Log (WAL) is that
          <classname>WAL</classname> edits will be written immediately. If deferred log flush is
        used, WAL edits are kept in memory until the flush period. The benefit is aggregated and
        asynchronous <classname>WAL</classname>- writes, but the potential downside is that if the
        RegionServer goes down the yet-to-be-flushed edits are lost. This is safer, however, than
        not using WAL at all with Puts. </para>
      <para> Deferred log flush can be configured on tables via <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/HTableDescriptor.html">HTableDescriptor</link>.
        The default value of <varname>hbase.regionserver.optionallogflushinterval</varname> is
        1000ms. </para>

      <title>HBase Client: AutoFlush</title>

      <para>When performing a lot of Puts, make sure that setAutoFlush is set to false on your <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html">HTable</link>
        instance. Otherwise, the Puts will be sent one at a time to the RegionServer. Puts added via
           htable.add(Put) and  htable.add( <List> Put) wind up in
        the same write buffer. If autoFlush = false, these messages are not sent until
        the write-buffer is filled. To explicitly flush the messages, call
          <methodname>flushCommits</methodname>. Calling <methodname>close</methodname> on the
          <classname>HTable</classname> instance will invoke
        <methodname>flushCommits</methodname>.</para>

      <title>HBase Client: Turn off WAL on Puts</title>
      <para>A frequent request is to disable the WAL to increase performance of Puts. This is only
        appropriate for bulk loads, as it puts your data at risk by removing the protection of the
        WAL in the event of a region server crash. Bulk loads can be re-run in the event of a crash,
        with little risk of data loss.</para>
      <warning>
        <para>If you disable the WAL for anything other than bulk loads, your data is at
          risk.</para></warning>
      <para>In general, it is best to use WAL for Puts, and where loading throughput is a concern to
        use <link linkend="perf.batch.loading">bulk loading</link> techniques instead. For normal
        Puts, you are not likely to see a performance improvement which would outweigh the risk. To
        disable the WAL, see <xref linkend="wal.disable"/>.</para>

      <title>HBase Client: Group Puts by RegionServer</title>
      <para>In addition to using the writeBuffer, grouping <classname>Put</classname>s by
        RegionServer can reduce the number of client RPC calls per writeBuffer flush. There is a
        utility <classname>HTableUtil</classname> currently on TRUNK that does this, but you can
        either copy that or implement your own version for those still on 0.90.x or earlier. </para>

      <title>MapReduce: Skip The Reducer</title>
      <para>When writing a lot of data to an HBase table from a MR job (e.g., with <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.html">TableOutputFormat</link>),
        and specifically where Puts are being emitted from the Mapper, skip the Reducer step. When a
        Reducer step is used, all of the output (Puts) from the Mapper will get spooled to disk,
        then sorted/shuffled to other Reducers that will most likely be off-node. It's far more
        efficient to just write directly to HBase. </para>
      <para>For summary jobs where HBase is used as a source and a sink, then writes will be coming
        from the Reducer step (e.g., summarize values then write out result). This is a different
        processing problem than from the the above case. </para>

      <title>Anti-Pattern: One Hot Region</title>
      <para>If all your data is being written to one region at a time, then re-read the section on
        processing <link
          linkend="timeseries">timeseries</link> data.</para>
      <para>Also, if you are pre-splitting regions and all your data is <emphasis>still</emphasis>
        winding up in a single region even though your keys aren't monotonically increasing, confirm
        that your keyspace actually works with the split strategy. There are a variety of reasons
        that regions may appear "well split" but won't work with your data. As the HBase client
        communicates directly with the RegionServers, this can be obtained via <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html#getRegionLocation%28byte[]%29">HTable.getRegionLocation</link>. </para>
      <para>See <xref
          linkend="precreate.regions" />, as well as <xref
          linkend="perf.configurations" />
      </para>

    <title>Reading from HBase</title>
    <para>The mailing list can help if you are having performance issues. For example, here is a
      good general thread on what to look at addressing read-time issues: <link
        xlink:href="http://search-hadoop.com/m/qOo2yyHtCC1">HBase Random Read latency >
      100ms</link></para>

      <title>Scan Caching</title>

      <para>If HBase is used as an input source for a MapReduce job, for example, make sure that the
        input <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Scan.html">Scan</link>
        instance to the MapReduce job has <methodname>setCaching</methodname> set to something
        greater than the default (which is 1). Using the default value means that the map-task will
        make call back to the region-server for every record processed. Setting this value to 500,
        for example, will transfer 500 rows at a time to the client to be processed. There is a
        cost/benefit to have the cache value be large because it costs more in memory for both
        client and RegionServer, so bigger isn't always better.</para>

        <title>Scan Caching in MapReduce Jobs</title>
        <para>Scan settings in MapReduce jobs deserve special attention. Timeouts can result (e.g.,
          UnknownScannerException) in Map tasks if it takes longer to process a batch of records
          before the client goes back to the RegionServer for the next set of data. This problem can
          occur because there is non-trivial processing occuring per row. If you process rows
          quickly, set caching higher. If you process rows more slowly (e.g., lots of
          transformations per row, writes), then set caching lower. </para>
        <para>Timeouts can also happen in a non-MapReduce use case (i.e., single threaded HBase
          client doing a Scan), but the processing that is often performed in MapReduce jobs tends
          to exacerbate this issue. </para>

      <title>Scan Attribute Selection</title>

      <para>Whenever a Scan is used to process large numbers of rows (and especially when used as a
        MapReduce source), be aware of which attributes are selected. If scan.addFamily
        is called then <emphasis>all</emphasis> of the attributes in the specified ColumnFamily will
        be returned to the client. If only a small number of the available attributes are to be
        processed, then only those attributes should be specified in the input scan because
        attribute over-selection is a non-trivial performance penalty over large datasets. </para>

      <title>Avoid scan seeks</title>
      <para>When columns are selected explicitly with scan.addColumn, HBase will
        schedule seek operations to seek between the selected columns. When rows have few columns
        and each column has only a few versions this can be inefficient. A seek operation is
        generally slower if does not seek at least past 5-10 columns/versions or 512-1024
        bytes.</para>
      <para>In order to opportunistically look ahead a few columns/versions to see if the next
        column/version can be found that way before a seek operation is scheduled, a new attribute
          Scan.HINT_LOOKAHEAD can be set the on Scan object. The following code
        instructs the RegionServer to attempt two iterations of next before a seek is
        scheduled:</para>
      <programlisting language="java">
Scan scan = new Scan();
scan.addColumn(...);
scan.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
table.getScanner(scan);
      </programlisting>

      <title>MapReduce - Input Splits</title>
      <para>For MapReduce jobs that use HBase tables as a source, if there a pattern where the
        "slow" map tasks seem to have the same Input Split (i.e., the RegionServer serving the
        data), see the Troubleshooting Case Study in <xref
          linkend="casestudies.slownode" />. </para>

      <title>Close ResultScanners</title>

      <para>This isn't so much about improving performance but rather <emphasis>avoiding</emphasis>
        performance problems. If you forget to close <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/ResultScanner.html">ResultScanners</link>
        you can cause problems on the RegionServers. Always have ResultScanner processing enclosed
        in try/catch blocks...</para>
      <programlisting language="java">
Scan scan = new Scan();
// set attrs...
ResultScanner rs = htable.getScanner(scan);
try {
  for (Result r = rs.next(); r != null; r = rs.next()) {
  // process result...
} finally {
  rs.close();  // always close the ResultScanner!
}
htable.close();
      </programlisting>

      <title>Block Cache</title>

      <para><link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Scan.html">Scan</link>
        instances can be set to use the block cache in the RegionServer via the
          <methodname>setCacheBlocks</methodname> method. For input Scans to MapReduce jobs, this
        should be <varname>false</varname>. For frequently accessed rows, it is advisable to use the
        block cache.</para>

    <para>Cache more data by moving your Block Cache offheap.  See <xref linkend="offheap.blockcache" /></para>

      <title>Optimal Loading of Row Keys</title>
      <para>When performing a table <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Scan.html">scan</link>
        where only the row keys are needed (no families, qualifiers, values or timestamps), add a
        FilterList with a <varname>MUST_PASS_ALL</varname> operator to the scanner using
          <methodname>setFilter</methodname>. The filter list should include both a <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/FirstKeyOnlyFilter.html">FirstKeyOnlyFilter</link>
        and a <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/filter/KeyOnlyFilter.html">KeyOnlyFilter</link>.
        Using this filter combination will result in a worst case scenario of a RegionServer reading
        a single value from disk and minimal network traffic to the client for a single row. </para>

      <title>Concurrency: Monitor Data Spread</title>
      <para>When performing a high number of concurrent reads, monitor the data spread of the target
        tables. If the target table(s) have too few regions then the reads could likely be served
        from too few nodes. </para>
      <para>See <xref
          linkend="precreate.regions" />, as well as <xref
          linkend="perf.configurations" />
      </para>

      <title>Bloom Filters</title>
      <para>Enabling Bloom Filters can save your having to go to disk and can help improve read
        latencies.</para>
      <para><link
          xlink:href="http://en.wikipedia.org/wiki/Bloom_filter">Bloom filters</link> were developed
        over in <link xlink:href="https://issues.apache.org/jira/browse/HBASE-1200">HBase-1200 Add
          bloomfilters</link>. For description of the development process -- why static blooms rather than dynamic
            -- and for an overview of the unique properties that pertain to blooms in HBase, as well
            as possible future directions, see the <emphasis>Development Process</emphasis> section
            of the document <link
              xlink:href="https://issues.apache.org/jira/secure/attachment/12444007/Bloom_Filters_in_HBase.pdf">BloomFilters
              in HBase</link> attached to <link
              xlink:href="https://issues.apache.org/jira/browse/HBASE-1200">HBase-1200</link>.  The bloom filters described here are actually version two of blooms in HBase. In
            versions up to 0.19.x, HBase had a dynamic bloom option based on work done by the <link
              xlink:href="http://www.one-lab.org">European Commission One-Lab Project 034819</link>.
            The core of the HBase bloom work was later pulled up into Hadoop to implement
            org.apache.hadoop.io.BloomMapFile. Version 1 of HBase blooms never worked that well.
            Version 2 is a rewrite from scratch though again it starts with the one-lab work.</para>

      <para>See also <xref
          linkend="schema.bloom" />. </para>

        <title>Bloom StoreFile footprint</title>

        <para>Bloom filters add an entry to the <classname>StoreFile</classname> general
            <classname>FileInfo</classname> data structure and then two extra entries to the
            <classname>StoreFile</classname> metadata section.</para>

          <title>BloomFilter in the <classname>StoreFile</classname>
            <classname>FileInfo</classname> data structure</title>

          <para><classname>FileInfo</classname> has a <varname>BLOOM_FILTER_TYPE</varname> entry
            which is set to <varname>NONE</varname>, <varname>ROW</varname> or
              <varname>ROWCOL.</varname></para>

          <title>BloomFilter entries in <classname>StoreFile</classname> metadata</title>

          <para><varname>BLOOM_FILTER_META</varname> holds Bloom Size, Hash Function used, etc. Its
            small in size and is cached on <classname>StoreFile.Reader</classname> load</para>
          <para><varname>BLOOM_FILTER_DATA</varname> is the actual bloomfilter data. Obtained
            on-demand. Stored in the LRU cache, if it is enabled (Its enabled by default).</para>

        <title>Bloom Filter Configuration</title>

          <title><varname>io.hfile.bloom.enabled</varname> global kill switch</title>

          <para>io.hfile.bloom.enabled in <classname>Configuration</classname> serves
            as the kill switch in case something goes wrong. Default =
            <varname>true</varname>.</para>

          <title><varname>io.hfile.bloom.error.rate</varname></title>

          <para><varname>io.hfile.bloom.error.rate</varname> = average false positive rate. Default
            = 1%. Decrease rate by ½ (e.g. to .5%) == +1 bit per bloom entry.</para>

          <title><varname>io.hfile.bloom.max.fold</varname></title>

          <para><varname>io.hfile.bloom.max.fold</varname> = guaranteed minimum fold rate. Most
            people should leave this alone. Default = 7, or can collapse to at least 1/128th of
            original size. See the <emphasis>Development Process</emphasis> section of the document <link
              xlink:href="https://issues.apache.org/jira/secure/attachment/12444007/Bloom_Filters_in_HBase.pdf">BloomFilters
              in HBase</link> for more on what this option means.</para>

      <title>Hedged Reads</title>
      <para>Hedged reads are a feature of HDFS, introduced in <link
          xlink:href="https://issues.apache.org/jira/browse/HDFS-5776">HDFS-5776</link>. Normally, a
        single thread is spawned for each read request. However, if hedged reads are enabled, the
        client waits some configurable amount of time, and if the read does not return, the client
        spawns a second read request, against a different block replica of the same data. Whichever
        read returns first is used, and the other read request is discarded. Hedged reads can be
        helpful for times where a rare slow read is caused by a transient error such as a failing
        disk or flaky network connection.</para>
      <para> Because a HBase RegionServer is a HDFS client, you can enable hedged reads in HBase, by
        adding the following properties to the RegionServer's hbase-site.xml and tuning the values
        to suit your environment.</para>
      <itemizedlist>
        <title>Configuration for Hedged Reads</title>
        <listitem>
          <para>dfs.client.hedged.read.threadpool.size - the number of threads
            dedicated to servicing hedged reads. If this is set to 0 (the default), hedged reads are
            disabled.</para>
        </listitem>
        <listitem>
          <para>dfs.client.hedged.read.threshold.millis - the number of milliseconds to
            wait before spawning a second read thread.</para>
        </listitem>
      </itemizedlist>
      <example>
        <title>Hedged Reads Configuration Example</title>
        <screen><![CDATA[<property>
  <name>dfs.client.hedged.read.threadpool.size</name>
  <value>20</value>
</property>
<property>
  <name>dfs.client.hedged.read.threshold.millis</name>
  <value>10</value>
</property>]]></screen>
      </example>
      <para>Use the following metrics to tune the settings for hedged reads on
        your cluster. See <xref linkend="hbase_metrics"/>  for more information.</para>
      <itemizedlist>
        <title>Metrics for Hedged Reads</title>
        <listitem>
          <para>hedgedReadOps - the number of times hedged read threads have been triggered. This
            could indicate that read requests are often slow, or that hedged reads are triggered too
            quickly.</para>
        </listitem>
        <listitem>
          <para>hedgeReadOpsWin - the number of times the hedged read thread was faster than the
            original thread. This could indicate that a given RegionServer is having trouble
            servicing requests.</para>
        </listitem>
      </itemizedlist>

    <title>Deleting from HBase</title>

      <title>Using HBase Tables as Queues</title>
      <para>HBase tables are sometimes used as queues. In this case, special care must be taken to
        regularly perform major compactions on tables used in this manner. As is documented in <xref
          linkend="datamodel" />, marking rows as deleted creates additional StoreFiles which then
        need to be processed on reads. Tombstones only get cleaned up with major compactions. </para>
      <para>See also <xref
          linkend="compaction" /> and <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HBaseAdmin.html#majorCompact%28java.lang.String%29">HBaseAdmin.majorCompact</link>.
      </para>

      <title>Delete RPC Behavior</title>
      <para>Be aware that htable.delete(Delete) doesn't use the writeBuffer. It will
        execute an RegionServer RPC with each invocation. For a large number of deletes, consider
          htable.delete(List). </para>
      <para>See <link
          xlink:href="http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/HTable.html#delete%28org.apache.hadoop.hbase.client.Delete%29" />
      </para>

    <title>HDFS</title>
    <para>Because HBase runs on <xref
        linkend="arch.hdfs" /> it is important to understand how it works and how it affects HBase. </para>

      <title>Current Issues With Low-Latency Reads</title>
      <para>The original use-case for HDFS was batch processing. As such, there low-latency reads
        were historically not a priority. With the increased adoption of Apache HBase this is
        changing, and several improvements are already in development. See the <link
          xlink:href="https://issues.apache.org/jira/browse/HDFS-1599">Umbrella Jira Ticket for HDFS
          Improvements for HBase</link>. </para>

      <title>Leveraging local data</title>
      <para>Since Hadoop 1.0.0 (also 0.22.1, 0.23.1, CDH3u3 and HDP 1.0) via <link
          xlink:href="https://issues.apache.org/jira/browse/HDFS-2246">HDFS-2246</link>, it is
        possible for the DFSClient to take a "short circuit" and read directly from the disk instead
        of going through the DataNode when the data is local. What this means for HBase is that the
        RegionServers can read directly off their machine's disks instead of having to open a socket
        to talk to the DataNode, the former being generally much faster. See JD's <link
              xlink:href="http://files.meetup.com/1350427/hug_ebay_jdcryans.pdf">Performance
              Talk</link>. Also see <link
          xlink:href="http://search-hadoop.com/m/zV6dKrLCVh1">HBase, mail # dev - read short
          circuit</link> thread for more discussion around short circuit reads. </para>
      <para>To enable "short circuit" reads, it will depend on your version of Hadoop. The original
        shortcircuit read patch was much improved upon in Hadoop 2 in <link
          xlink:href="https://issues.apache.org/jira/browse/HDFS-347">HDFS-347</link>. See <link
          xlink:href="http://blog.cloudera.com/blog/2013/08/how-improved-short-circuit-local-reads-bring-better-performance-and-security-to-hadoop/" />
        for details on the difference between the old and new implementations. See <link
          xlink:href="http://archive.cloudera.com/cdh4/cdh/4/hadoop/hadoop-project-dist/hadoop-hdfs/ShortCircuitLocalReads.html">Hadoop
          shortcircuit reads configuration page</link> for how to enable the latter, better version
        of shortcircuit. For example, here is a minimal config. enabling short-circuit reads added
        to <filename>hbase-site.xml</filename>: </para>
      <programlisting language="xml"><![CDATA[<property>
  <name>dfs.client.read.shortcircuit</name>
  <value>true</value>
  <description>
    This configuration parameter turns on short-circuit local reads.
  </description>
</property>
<property>
  <name>dfs.domain.socket.path</name>
  <value>/home/stack/sockets/short_circuit_read_socket_PORT</value>
  <description>
    Optional.  This is a path to a UNIX domain socket that will be used for
    communication between the DataNode and local HDFS clients.
    If the string "_PORT" is present in this path, it will be replaced by the
    TCP port of the DataNode.
  </description>
</property>]]></programlisting>
      <para>Be careful about permissions for the directory that hosts the shared domain socket;
        dfsclient will complain if open to other than the hbase user. </para>
      <para>If you are running on an old Hadoop, one that is without <link
          xlink:href="https://issues.apache.org/jira/browse/HDFS-347">HDFS-347</link> but that has
          <link xlink:href="https://issues.apache.org/jira/browse/HDFS-2246">HDFS-2246</link>, you
        must set two configurations. First, the hdfs-site.xml needs to be amended. Set the property
          <varname>dfs.block.local-path-access.user</varname> to be the <emphasis>only</emphasis>
        user that can use the shortcut. This has to be the user that started HBase. Then in
        hbase-site.xml, set <varname>dfs.client.read.shortcircuit</varname> to be
          <varname>true</varname>
      </para>

      <para> Services -- at least the HBase RegionServers -- will need to be restarted in order to
        pick up the new configurations. </para>
      <note
        xml:id="dfs.client.read.shortcircuit.buffer.size">
        <title>dfs.client.read.shortcircuit.buffer.size</title>
        <para>The default for this value is too high when running on a highly trafficed HBase. In
          HBase, if this value has not been set, we set it down from the default of 1M to 128k
          (Since HBase 0.98.0 and 0.96.1). See <link
            xlink:href="https://issues.apache.org/jira/browse/HBASE-8143">HBASE-8143 HBase on Hadoop
            2 with local short circuit reads (ssr) causes OOM</link>). The Hadoop DFSClient in HBase
          will allocate a direct byte buffer of this size for <emphasis>each</emphasis> block it has
          open; given HBase keeps its HDFS files open all the time, this can add up quickly.</para>
      </note>

      <title>Performance Comparisons of HBase vs. HDFS</title>
      <para>A fairly common question on the dist-list is why HBase isn't as performant as HDFS files
        in a batch context (e.g., as a MapReduce source or sink). The short answer is that HBase is
        doing a lot more than HDFS (e.g., reading the KeyValues, returning the most current row or
        specified timestamps, etc.), and as such HBase is 4-5 times slower than HDFS in this
        processing context. There is room for improvement and this gap will, over time, be reduced,
        but HDFS will always be faster in this use-case. </para>

    <title>Amazon EC2</title>
    <para>Performance questions are common on Amazon EC2 environments because it is a shared
      environment. You will not see the same throughput as a dedicated server. In terms of running
      tests on EC2, run them several times for the same reason (i.e., it's a shared environment and
      you don't know what else is happening on the server). </para>
    <para>If you are running on EC2 and post performance questions on the dist-list, please state
      this fact up-front that because EC2 issues are practically a separate class of performance
      issues. </para>

    <title>Collocating HBase and MapReduce</title>
    <para>It is often recommended to have different clusters for HBase and MapReduce. A better
      qualification of this is: don't collocate a HBase that serves live requests with a heavy MR
      workload. OLTP and OLAP-optimized systems have conflicting requirements and one will lose to
      the other, usually the former. For example, short latency-sensitive disk reads will have to
      wait in line behind longer reads that are trying to squeeze out as much throughput as
      possible. MR jobs that write to HBase will also generate flushes and compactions, which will
      in turn invalidate blocks in the <xref
        linkend="block.cache" />. </para>
    <para>If you need to process the data from your live HBase cluster in MR, you can ship the
      deltas with <xref
        linkend="copy.table" /> or use replication to get the new data in real time on the OLAP
      cluster. In the worst case, if you really need to collocate both, set MR to use less Map and
      Reduce slots than you'd normally configure, possibly just one. </para>
    <para>When HBase is used for OLAP operations, it's preferable to set it up in a hardened way
      like configuring the ZooKeeper session timeout higher and giving more memory to the MemStores
      (the argument being that the Block Cache won't be used much since the workloads are usually
      long scans). </para>

    <title>Case Studies</title>
    <para>For Performance and Troubleshooting Case Studies, see <xref
        linkend="casestudies" />. </para>

</chapter>


text:
//-----------------------------------------------
//
//	This file is part of the Siv3D Engine.
//
//	Copyright (c) 2008-2019 Ryo Suzuki
//	Copyright (c) 2016-2019 OpenSiv3D Project
//
//	Licensed under the MIT License.
//
//-----------------------------------------------

# include <Siv3D/Windows.hpp>
# include <Siv3DEngine.hpp>
# include <Window/IWindow.hpp>
# include <Siv3D/FileSystem.hpp>
# include <Siv3D/ReaderView.hpp>
# include <Siv3D/EngineLog.hpp>
# include "CClipboard.hpp"

namespace s3d
{
	namespace detail
	{
		static bool HasInvalidPremultipliedColors(const Color* image, const size_t num_pixels)
		{
			const Color* pSrc = image;
			const Color* const pSrcEnd = pSrc + num_pixels;

			while (pSrc != pSrcEnd)
			{
				if (pSrc->r > pSrc->a || pSrc->g > pSrc->a || pSrc->b > pSrc->a)
				{
					return true;
				}

				++pSrc;
			}

			return false;
		}

		static void LoadImageFromClipboard(Image& image)
		{
			const HANDLE hDIB = ::GetClipboardData(CF_DIB);

			if (!hDIB)
			{
				return;
			}

			const size_t memorySize = ::GlobalSize(hDIB);

			if (const void* memory = reinterpret_cast<uint8*>(::GlobalLock(hDIB)))
			{
				const BITMAPINFO* header = static_cast<const BITMAPINFO*>(memory);
				const int32 depth = header->bmiHeader.biBitCount;

				size_t colorTableSize = 0;

				if (depth == 8)
				{
					colorTableSize = header->bmiHeader.biClrUsed ? header->bmiHeader.biClrUsed
						: (1 << header->bmiHeader.biBitCount);
				}
				else if (depth == 32 && header->bmiHeader.biCompression == BI_BITFIELDS)
				{
					colorTableSize = 3;
				}

				const size_t offsetSize = header->bmiHeader.biSize + colorTableSize * sizeof(RGBQUAD);
				const void* bitmapData = static_cast<const Byte*>(memory) + offsetSize;

				const bool reversed	= header->bmiHeader.biHeight > 0;
				const int32 width	= header->bmiHeader.biWidth;
				const int32 height	= reversed ? header->bmiHeader.biHeight : -header->bmiHeader.biHeight;
				const bool hasAlpha = (depth == 32)
					&& !HasInvalidPremultipliedColors(static_cast<const Color*>(bitmapData), width * height);
				image.resize(width, height);

				ReaderView reader(bitmapData, memorySize - offsetSize);

				switch (depth)
				{
				case 8:
					{
						uint8 palette[1024];
						reader.read(palette);

						const uint32 rowSize = width + (width % 4 ? 4 - width % 4 : 0);
						const int32 lineStep = reversed ? -width : width;
						Color* pDstLine = image[reversed ? height - 1 : 0];
						uint8* const buffer = static_cast<uint8*>(std::malloc(rowSize));

						for (int32 y = 0; y < height; ++y)
						{
							if (y == height - 1)
							{
								reader.read(buffer, width);
							}
							else
							{
								reader.read(buffer, rowSize);
							}

							const uint8* pSrc = buffer;
							const Color* const pDstEnd = pDstLine + width;

							for (Color* pDst = pDstLine; pDst != pDstEnd; ++pDst)
							{
								const uint8 *src = palette + ((*pSrc++) << 2);

								pDst->set(src[2], src[1], src[0]);
							}

							pDstLine += lineStep;
						}

						std::free(buffer);

						break;
					}
				case 24:
				case 32:
					{
						const size_t rowSize = depth == 24 ? width * 3 + width % 4 : width * 4;
						const int32 depthBytes = depth / 8;
						const int32 lineStep = reversed ? -width : width;
						Color* pDstLine = image[reversed ? height - 1 : 0];
						uint8* const buffer = static_cast<uint8*>(std::malloc(rowSize));

						for (int32 y = 0; y < height; ++y)
						{
							if (y == height - 1)
							{
								reader.read(buffer, depthBytes * width);
							}
							else
							{
								reader.read(buffer, rowSize);
							}

							const Color* const pDstEnd = pDstLine + width;
							const uint8* pSrc = buffer;

							for (Color* pDst = pDstLine; pDst != pDstEnd; ++pDst)
							{
								pDst->set(pSrc[2], pSrc[1], pSrc[0], hasAlpha ? pSrc[3] : 255);

								pSrc += depthBytes;
							}

							pDstLine += lineStep;
						}

						std::free(buffer);

						break;
					}
				}

			}
			::GlobalUnlock(hDIB);
		}

		static void WriteBitmapToClipboard(HBITMAP hSourceBitmap, const Size& size, HWND hWnd)
		{
			HDC hDC				= ::GetDC(nullptr);
			HDC hCompatibleDC	= ::CreateCompatibleDC(nullptr);
			HDC hSourceDC		= ::CreateCompatibleDC(nullptr);
			HBITMAP hBitmap		= ::CreateCompatibleBitmap(hDC, size.x, size.y);

			if (!hBitmap)
			{
				::DeleteDC(hCompatibleDC);
				::DeleteDC(hSourceDC);
				::ReleaseDC(nullptr, hDC);
				return;
			}

			HBITMAP hOldBitmap = (HBITMAP)::SelectObject(hCompatibleDC, hBitmap);
			HBITMAP hOldSource = (HBITMAP)::SelectObject(hSourceDC, hSourceBitmap);
			{
				const BLENDFUNCTION blend = { AC_SRC_OVER, 0, 255, AC_SRC_ALPHA };
				::GdiAlphaBlend(hCompatibleDC, 0, 0, size.x, size.y, hSourceDC, 0, 0, size.x, size.y, blend);
			}
			::SelectObject(hCompatibleDC, hOldBitmap);
			::SelectObject(hSourceDC, hOldSource);

			::DeleteObject(hOldBitmap);
			::DeleteObject(hOldSource);
			::DeleteDC(hCompatibleDC);
			::DeleteDC(hSourceDC);
			::ReleaseDC(nullptr, hDC);

			if (::OpenClipboard(hWnd))
			{
				::EmptyClipboard();
				::SetClipboardData(CF_BITMAP, hBitmap);
				::CloseClipboard();
			}

			::DeleteObject(hBitmap);
		}
	}

	CClipboard::CClipboard()
	{

	}

	CClipboard::~CClipboard()
	{
		LOG_TRACE(U"CClipboard::~CClipboard()");
	}

	void CClipboard::init()
	{
		LOG_TRACE(U"CClipboard::init()");

		m_hWnd = static_cast<HWND>(Siv3DEngine::Get<ISiv3DWindow>()->getHandle());

		m_sequenceNumber = ::GetClipboardSequenceNumber();

		LOG_INFO(U"ℹ CClipboard initialized");
	}

	bool CClipboard::hasChanged()
	{
		const uint32 currentSequenceNumber = ::GetClipboardSequenceNumber();

		const bool hasChanged = (currentSequenceNumber != m_sequenceNumber);

		m_sequenceNumber = currentSequenceNumber;

		return hasChanged;
	}

	bool CClipboard::getText(String& text)
	{
		text.clear();

		if (::IsClipboardFormatAvailable(CF_UNICODETEXT))
		{
			if (!::OpenClipboard(m_hWnd))
			{
				return false;
			}

			if (const HGLOBAL hGlobal = static_cast<HGLOBAL>(::GetClipboardData(CF_UNICODETEXT)))
			{
				text = Unicode::FromWString(static_cast<const wchar_t*>(::GlobalLock(hGlobal)));

				::GlobalUnlock(hGlobal);

				text.remove(L'\r');
			}

			::CloseClipboard();
		}

		return !text.isEmpty();
	}

	bool CClipboard::getImage(Image& image)
	{
		image.clear();

		if (::IsClipboardFormatAvailable(CF_DIB))
		{
			if (!::OpenClipboard(m_hWnd))
			{
				return false;
			}

			detail::LoadImageFromClipboard(image);

			::CloseClipboard();
		}

		return !image.isEmpty();
	}

	bool CClipboard::getFilePaths(Array<FilePath>& paths)
	{
		paths.clear();

		if (::IsClipboardFormatAvailable(CF_HDROP))
		{
			if (!::OpenClipboard(m_hWnd))
			{
				return false;
			}

			if (const HDROP hDrop = (HDROP)::GetClipboardData(CF_HDROP))
			{
				const uint32 dropSize = ::DragQueryFileW(hDrop, ~0u, nullptr, 0);

				paths.reserve(dropSize);

				wchar_t tmpFilePath[MAX_PATH];

				for (uint32 i = 0; i < dropSize; ++i)
				{
					::DragQueryFileW(hDrop, i, tmpFilePath, MAX_PATH);

					paths.push_back(FileSystem::FullPath(Unicode::FromWString(tmpFilePath)));
				}
			}

			::CloseClipboard();
		}

		return !paths.isEmpty();
	}

	void CClipboard::setText(const String& _text)
	{
		if (!::OpenClipboard(m_hWnd))
		{
			return;
		}

		::EmptyClipboard();

		const std::wstring text = _text.toWstr();

		const size_t size = sizeof(char16_t) * (text.length() + 1);

		if (HANDLE hData = ::GlobalAlloc(GMEM_MOVEABLE, size))
		{
			void* pDst = ::GlobalLock(hData);

			std::memcpy(pDst, text.data(), size);

			::GlobalUnlock(hData);

			::SetClipboardData(CF_UNICODETEXT, hData);
		}

		::CloseClipboard();

		m_sequenceNumber = ::GetClipboardSequenceNumber();
	}

	void CClipboard::setImage(const Image& image)
	{
		HDC hDC = ::GetDC(nullptr);
		BITMAPINFO header = {};
		header.bmiHeader.biSize			= sizeof(BITMAPINFOHEADER);
		header.bmiHeader.biWidth		= image.width();
		header.bmiHeader.biHeight		= -image.height();
		header.bmiHeader.biPlanes		= 1;
		header.bmiHeader.biBitCount		= 32;
		header.bmiHeader.biCompression	= BI_RGB;

		void* bitmapData;
		HBITMAP hBitmap = ::CreateDIBSection(hDC, &header, DIB_RGB_COLORS, &bitmapData, nullptr, 0);

		if (bitmapData && hBitmap)
		{
			const Color* pSrc = image.data();
			const Color* pSrcEnd = pSrc + image.num_pixels();
			Color* pDst = static_cast<Color*>(bitmapData);

			while (pSrc != pSrcEnd)
			{
				if (pSrc->a == 255)
				{
					pDst->r = pSrc->b;
					pDst->g = pSrc->g;
					pDst->b = pSrc->r;
					pDst->a = 255;
				}
				else
				{
					const float fa = pSrc->a / 255.0f;
					pDst->r = static_cast<uint8>(pSrc->b * fa);
					pDst->g = static_cast<uint8>(pSrc->g * fa);
					pDst->b = static_cast<uint8>(pSrc->r * fa);
					pDst->a = pSrc->a;
				}

				++pDst;
				++pSrc;
			}

			detail::WriteBitmapToClipboard(hBitmap, image.size(), m_hWnd);

			m_sequenceNumber = ::GetClipboardSequenceNumber();
		}

		::DeleteObject(hBitmap);
		::ReleaseDC(nullptr, hDC);
	}

	void CClipboard::clear()
	{
		if (::OpenClipboard(m_hWnd))
		{
			::EmptyClipboard();
			::CloseClipboard();
		}

		m_sequenceNumber = ::GetClipboardSequenceNumber();
	}
}


text:
package noir.graphics;

import java.awt.BorderLayout;
import java.awt.Dimension;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.Rectangle;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.ComponentEvent;
import java.awt.event.ComponentListener;
import java.awt.event.KeyListener;
import java.awt.event.MouseListener;
import java.awt.event.WindowEvent;
import java.awt.event.WindowStateListener;

import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.Timer;

@SuppressWarnings("serial")
public class Panel extends JPanel
{
	static {
		System.setProperty("sun.java2d.opengl", "true");

//	    System.setProperty("sun.java2d.transaccel", "True");
//	    System.setProperty("sun.java2d.d3d", "True");
//	    System.setProperty("sun.java2d.ddforcevram", "True");
	}

    private JFrame frame;
    private Drawable drawable;

    public Panel(String title, int width, int height, Drawable provider)
    {
    	this(title, width, height, provider, 0.0);
    }

    public Panel(String title, int width, int height, Drawable provider, double fps)
    {
    	super();

    	frame = new JFrame(title);
        frame.setLayout(new BorderLayout());

        setPreferredSize(new Dimension(width, height));

        frame.add(this);
        frame.pack();
        frame.setLocationRelativeTo(null);
        frame.setVisible(true);
        frame.setIgnoreRepaint(fps > 0.0);

        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);

        provider.setPanel(this);
    	this.drawable = provider;

    	KeyListener keyListener = provider.getKeyListener();
    	if(keyListener != null)
    	{
    		frame.addKeyListener(keyListener);
    	}

    	MouseListener mouseListener = provider.getMouseListener();
    	if(mouseListener != null)
    	{
    		this.addMouseListener(mouseListener);
    	}

    	if(fps > 0.0)
    	{
    		setTimer(fps);
    	}

    	frame.addComponentListener(new ComponentListener() {

			@Override
			public void componentShown(ComponentEvent e) {}

			@Override
			public void componentResized(ComponentEvent e) {
				refresh();
			}

			@Override
			public void componentMoved(ComponentEvent e) { }

			@Override
			public void componentHidden(ComponentEvent e) {}
		});
    }

    public void setFrameTitle(String title)
    {
    	frame.setTitle(title);
    }

    private void refresh()
    {
		try(GraphicsBatch graphicsBatch = makeBatch(getGraphics()))
    	{
    		drawable.draw(graphicsBatch);
    	}
    	catch(Exception exn)
    	{
    		throw new RuntimeException(exn);
//			e.printStackTrace();
		}
    }

    private void setTimer(double fps)
    {
    	int delay = (int)(1000.0 / fps);
        Timer timer = new Timer(delay, new ActionListener() {

        	long lastTime = System.currentTimeMillis();

			@Override
			public void actionPerformed(ActionEvent evt)
			{
				refresh();

				long currentTime = System.currentTimeMillis();
				long elapsedTimeMillis = currentTime - lastTime;
	        	lastTime = currentTime;

	        	drawable.update(elapsedTimeMillis);
			}
		});
		timer.start();
    }

    private GraphicsBatch makeBatch(Graphics graphics)
    {
    	Rectangle bounds = getBounds();
    	setBackground(drawable.getBackgroundColor());
    	graphics.setColor(getBackground());
    	graphics.fillRect(0, 0, bounds.width, bounds.height);
    	graphics.setColor(getForeground());
    	GraphicsBatch graphicsBatch = new GraphicsBatch((Graphics2D)graphics, bounds, drawable);
    	return graphicsBatch;
    }

    @Override
	public void paint(Graphics graphics)
    {
    	super.paint(graphics);
    	GraphicsBatch graphicsBatch = makeBatch(graphics);
    	drawable.draw(graphicsBatch);
    }
}


text:
<?php

use yii\helpers\Html;
use yii\widgets\ActiveForm;

/* @var $this yii\web\View */
/* @var $model app\modules\panel\models\search\AccountSearch */
/* @var $form yii\widgets\ActiveForm */
?>

    <?php $form = ActiveForm::begin([
        'action' => ['index'],
        'method' => 'get',
    ]); ?>

    <?= $form->field($model, 'id') ?>

    <?= $form->field($model, 'tb_account') ?>

    <?= $form->field($model, 'tb_pwd') ?>

    <?= $form->field($model, 'tb_user_id') ?>

    <?= $form->field($model, 'tb_nick_name') ?>

    <?php // echo $form->field($model, 'access_token') ?>

    <?php // echo $form->field($model, 'refresh_token') ?>

    <?php // echo $form->field($model, 'host_id') ?>

    <?php // echo $form->field($model, 'remmenber_me') ?>

        <?= Html::submitButton('Search', ['class' => 'btn btn-primary']) ?>
        <?= Html::resetButton('Reset', ['class' => 'btn btn-default']) ?>

    <?php ActiveForm::end(); ?>


text:
[![Build Status](https://travis-ci.org/netarchivesuite/heritrix3-wrapper.svg)](https://travis-ci.org/netarchivesuite/heritrix3-wrapper)
[![codecov](https://codecov.io/gh/netarchivesuite/heritrix3-wrapper/branch/master/graph/badge.svg)](https://codecov.io/gh/netarchivesuite/heritrix3-wrapper)

heritrix3-wrapper
=================

Small wrapper to unzip, start/stop and communicate with Heritrix 3.

## Features ##

* Unzip and preserve unix permissions.
* Start H3 and capture exit value and output/error streams.
* Perform H3 REST API calls and convert result to Java objects using JAXB.
* XML validation of response data.
* anypath methods including support for header method request and/or byte range request.

## REST API supported and wrapped ##

### EngineResource ###

* Exit Java process.
* Garbage Collect.
* Re-scan job directory.
* Create new job.
* Add job directory.

All of these commands return an engine xml result which is converted to Java objects, with the exception of "Exit Java process" which kills the H3 JVM immediate if all the right conditions are met.

### JobResource ###

* Status.
* Copy.
* Build job configuration.
* Teardown.
* Launch.
* Pause.
* Unpause.
* Checkpoint.
* Terminate.

All of these commands return a job xml result which is converted to Java objects.

### ScriptResource ###

* Script

XML result convert to Java objects.

### Build and Deploy ###

Deploy snapshot
mvn -DskipTests -Dmaven.javadoc.skip=true clean deploy

Release
mvn -Darguments='-DskipTests' release:prepare
mvn -Darguments='-DskipTests -Dmaven.javadoc.skip=true' release:perform