text:
package showpaths

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"math/rand"
	"net"
	"strings"
	"time"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/daemon"
	"github.com/scionproto/scion/go/lib/serrors"
	"github.com/scionproto/scion/go/lib/snet"
	"github.com/scionproto/scion/go/pkg/app/path"
	"github.com/scionproto/scion/go/pkg/pathprobe"
)

// Result contains all the discovered paths.
type Result struct {
	LocalIA     addr.IA `json:"local_isd_as"`
	Destination addr.IA `json:"destination"`
	Paths       []Path  `json:"paths,omitempty"`
}

// Path holds information about the discovered path.
type Path struct {
	FullPath        snet.Path       `json:"-"`
	Fingerprint     string          `json:"fingerprint"`
	Hops            []Hop           `json:"hops"`
	NextHop         string          `json:"next_hop"`
	Expiry          time.Time       `json:"expiry"`
	MTU             uint16          `json:"mtu"`
	Latency         []time.Duration `json:"latency"`
	CarbonIntensity []int64         `json:"carbon_intensity"`
	Status          string          `json:"status,omitempty"`
	StatusInfo      string          `json:"status_info,omitempty"`
	Local           net.IP          `json:"local_ip,omitempty"`
}

// Hop represents an hop on the path.
type Hop struct {
	IfID common.IFIDType `json:"ifid"`
	IA   addr.IA         `json:"isd_as"`
}

// Human writes human readable output to the writer.
func (r Result) Human(w io.Writer, showExtendedMetadata, colored bool) {
	cs := path.DefaultColorScheme(!colored)

	idxWidth := len(fmt.Sprint(len(r.Paths) - 1))

	// max number of key-value entries before switching to multi-line mode.
	// Chosen to allow to include the expiration date in a single line, but switch to multi-line
	// once at least one of the beacon extension items contains a meaningful value.
	const maxEntriesSingleline = 6
	separatorSingleline := " "
	// in multi-line mode, the entries are newline separated and indented,
	// taking into account the length of the path index "header" before the
	// first entry (length of index + 2 brackets + 1 space)
	separatorMultiline := "\n" + strings.Repeat(" ", idxWidth+2+1)

	sectionHeader := func(intfs int) {
		cs.Header.Fprintf(w, "%d Hops:\n", (intfs/2)+1)
	}
	sectionHeader(len(r.Paths[0].Hops))
	for i, path := range r.Paths {
		if i != 0 && len(r.Paths[i-1].Hops) != len(path.Hops) {
			sectionHeader(len(path.Hops))
		}

		entries := cs.KeyValues(
			"Hops", cs.Path(path.FullPath),
			"MTU", fmt.Sprint(path.MTU),
			"NextHop", fmt.Sprint(path.NextHop),
		)
		if showExtendedMetadata {
			meta := path.FullPath.Metadata()
			ttl := time.Until(path.Expiry).Truncate(time.Second)
			entries = append(entries, cs.KeyValue(
				"Expires", fmt.Sprintf("%s (%s)", path.Expiry, ttl),
			))
			// Add entries for information from beacon extension, only if a non-empty
			// value can be shown.
			entries = append(entries, filteredKeyValues(cs,
				"Latency", humanLatency(meta),
				"Bandwidth", humanBandwidth(meta),
				"CarbonIntensity", humanCarbonIntensity(meta),
				"Geo", humanGeo(meta, cs),
				"LinkType", humanLinkType(meta),
				"InternalHops", humanInternalHops(meta),
				"Notes", humanNotes(meta),
			)...)
		}
		if path.Status != "" {
			statusColor := cs.Bad
			if strings.EqualFold(path.Status, string(pathprobe.StatusAlive)) {
				statusColor = cs.Good
			}
			entries = append(entries, cs.KeyValues(
				"Status", statusColor.Sprint(path.Status),
				"LocalIP", fmt.Sprint(path.Local),
			)...)
		}
		separator := separatorSingleline
		if len(entries) > maxEntriesSingleline {
			separator = separatorMultiline
		}
		fmt.Fprintf(w, "[%*d] %s\n", idxWidth, i, strings.Join(entries, separator))
	}
}

// filteredKeyValues is analogous to app.ColorScheme.KeyValues, but ignores
// entries with an empty value.
func filteredKeyValues(cs path.ColorScheme, kv ...string) []string {
	if len(kv)%2 != 0 {
		panic("KeyValues expects even number of parameters")
	}
	entries := make([]string, 0)
	for i := 0; i < len(kv); i += 2 {
		if kv[i+1] == "" {
			continue
		}
		entries = append(entries, cs.KeyValue(kv[i], kv[i+1]))
	}
	return entries
}

// humanLatency summarizes the latency information in the meta data in a human
// readable string. Returns empty string if no information is available.
func humanLatency(p *snet.PathMetadata) string {
	complete := true
	var tot time.Duration
	for _, v := range p.Latency {
		complete = complete && v >= 0
		if v >= 0 {
			tot += v
		}
	}
	if complete {
		return fmt.Sprint(tot)
	}
	if tot > 0 {
		return fmt.Sprintf(">%s (information incomplete)", tot)
	}
	return ""
}

// humanBandwidth summarizes the bandwidth information in the meta data in a
// human readable string. Returns empty string if no information is available.
func humanBandwidth(p *snet.PathMetadata) string {
	complete := true
	var bottleneck uint64 = math.MaxUint64
	for _, v := range p.Bandwidth {
		complete = complete && v > 0
		if v > 0 && v < bottleneck {
			bottleneck = v
		}
	}
	if complete {
		return fmt.Sprintf("%dKbit/s", bottleneck) // TODO(matzf) use appropriate metric prefixes?
	}
	if bottleneck < math.MaxUint64 {
		return fmt.Sprintf("%dKbit/s (information incomplete)", bottleneck)
	}
	return ""
}

// humanCarbonIntensity summarizes the carbon intensity information in the meta
// data in a human readable string. Returns empty string if no information is
// available.
func humanCarbonIntensity(p *snet.PathMetadata) string {
	complete := true
	var tot int64 = 0
	for _, v := range p.CarbonIntensity {
		complete = complete && v >= 0
		if v >= 0 {
			tot += v
		}
	}
	if complete {
		return fmt.Sprintf("%dgCO2/TB", tot)
	}
	if tot > 0 {
		return fmt.Sprintf(">%dgCO2/TB (information incomplete)", tot)
	}
	return ""
}

// humanGeo summarizes the geographical information in the meta data in a human
// readable string. Returns empty string if no information is available.
func humanGeo(p *snet.PathMetadata, cs path.ColorScheme) string {
	geos := make([]string, len(p.Geo))
	hasAny := false
	for i, geo := range p.Geo {
		hasLatLong := (geo.Latitude != 0.0 || geo.Longitude != 0.0)
		latLong := fmt.Sprintf("%g,%g", geo.Latitude, geo.Longitude)
		sanitizedAddr := sanitizeString(geo.Address)
		quotedAddr := fmt.Sprintf("\"%s\"", sanitizedAddr)
		hasAddr := sanitizedAddr != ""
		hasAny = hasAny || hasLatLong || hasAddr
		if hasLatLong && hasAddr {
			geos[i] = fmt.Sprintf("%s (%s)", latLong, quotedAddr)
		} else if hasLatLong {
			geos[i] = latLong
		} else if hasAddr {
			geos[i] = quotedAddr
		} else {
			geos[i] = "N/A"
		}
	}
	if !hasAny { // special case to hide the Geo entry when no information is available
		return ""
	}
	return fmt.Sprintf("[%s]", strings.Join(geos, cs.Link.Sprintf(" > ")))
}

// humanGeo summarizes the link type information in the meta data in a human
// readable string. Returns empty string if no information is available.
func humanLinkType(p *snet.PathMetadata) string {
	hasAny := false
	for _, lt := range p.LinkType {
		if lt != snet.LinkTypeUnset {
			hasAny = true
			break
		}
	}
	if !hasAny {
		return ""
	}

	linkTypes := make([]string, len(p.LinkType))
	for i, lt := range p.LinkType {
		linkTypes[i] = lt.String()
	}
	return fmt.Sprintf("[%s]", strings.Join(linkTypes, ", "))
}

// humanInternalHops summarizes the information on the number of AS internal
// hops along the path in the meta data in a human readable string. Returns
// empty string if no information is available.
func humanInternalHops(p *snet.PathMetadata) string {
	internalHops := []string{}
	for i, numHops := range p.InternalHops {
		if numHops == 0 {
			continue
		}
		interfaceIdx := 2*i + 1
		ia := p.Interfaces[interfaceIdx].IA
		internalHops = append(internalHops, fmt.Sprintf("%s: %d", ia, numHops))
	}
	if len(internalHops) == 0 {
		return ""
	}
	return fmt.Sprintf("[%s]", strings.Join(internalHops, ", "))
}

// humanNotes summarizes the notes in the meta data in a human readable string.
// Returns empty string if no information is available.
func humanNotes(p *snet.PathMetadata) string {
	notes := []string{}
	for i, note := range p.Notes {
		if note == "" {
			continue
		}
		interfaceIdx := 0
		if i > 0 {
			interfaceIdx = 2*i - 1
		}
		ia := p.Interfaces[interfaceIdx].IA
		notes = append(notes, fmt.Sprintf("%s: \"%s\"", ia, sanitizeString(note)))
	}
	if len(notes) == 0 {
		return ""
	}
	return fmt.Sprintf("[%s]", strings.Join(notes, ", "))
}

// sanitizeString returns a trimmed single line representation of the string,
// with any control characters or quotation marks removed.
func sanitizeString(str string) string {
	str = strings.ReplaceAll(str, "\n", ", ")
	str = strings.TrimSpace(str)
	return strings.Map(func(r rune) rune {
		if r >= 32 && r != 127 && r != '"' {
			return r
		}
		return -1
	}, str)
}

// JSON writes the showpaths result as a json object to the writer.
func (r Result) JSON(w io.Writer) error {
	enc := json.NewEncoder(w)
	enc.SetIndent("", "  ")
	enc.SetEscapeHTML(false)
	return enc.Encode(r)
}

// IsLocal returns true iff Source and Destination AS are identical
func (r Result) IsLocal() bool {
	return r.LocalIA == r.Destination
}

// Alive returns the number of alive paths.
func (r Result) Alive() int {
	var c int
	for _, path := range r.Paths {
		if strings.EqualFold(path.Status, string(pathprobe.StatusAlive)) {
			c++
		}
	}
	return c
}

// Run lists the paths to the specified ISD-AS to stdout.
func Run(ctx context.Context, dst addr.IA, cfg Config) (*Result, error) {
	sdConn, err := daemon.NewService(cfg.Daemon).Connect(ctx)
	if err != nil {
		return nil, serrors.WrapStr("connecting to the SCION Daemon", err, "addr", cfg.Daemon)
	}
	localIA, err := sdConn.LocalIA(ctx)
	if err != nil {
		return nil, serrors.WrapStr("determining local ISD-AS", err)
	}
	if dst == localIA {
		return &Result{
			LocalIA:     localIA,
			Destination: dst,
		}, nil
	}

	// TODO(lukedirtwalker): Replace this with snet.Router once we have the
	// possibility to have the same functionality, i.e. refresh, fetch all paths.
	// https://github.com/scionproto/scion/issues/3348
	allPaths, err := sdConn.Paths(ctx, dst, 0,
		daemon.PathReqFlags{Refresh: cfg.Refresh})
	if err != nil {
		return nil, serrors.WrapStr("retrieving paths from the SCION Daemon", err)
	}
	paths, err := path.Filter(cfg.Sequence, allPaths)
	if err != nil {
		return nil, err
	}
	if cfg.MaxPaths != 0 && len(paths) > cfg.MaxPaths {
		paths = paths[:cfg.MaxPaths]
	}

	var statuses map[string]pathprobe.Status
	if !cfg.NoProbe {
		p := pathprobe.FilterEmptyPaths(paths)
		statuses, err = pathprobe.Prober{
			DstIA:      dst,
			LocalIA:    localIA,
			LocalIP:    cfg.Local,
			ID:         uint16(rand.Uint32()),
			Dispatcher: cfg.Dispatcher,
		}.GetStatuses(ctx, p)
		if err != nil {
			return nil, serrors.WrapStr("getting statuses", err)
		}
	}
	path.Sort(paths)
	res := &Result{
		LocalIA:     localIA,
		Destination: dst,
		Paths:       []Path{},
	}
	for _, path := range paths {
		fingerprint := "local"
		if len(path.Metadata().Interfaces) > 0 {
			fp := snet.Fingerprint(path).String()
			fingerprint = fp[:16]
		}
		var nextHop string
		if nh := path.UnderlayNextHop(); nh != nil {
			nextHop = path.UnderlayNextHop().String()
		}
		pathMeta := path.Metadata()
		rpath := Path{
			FullPath:        path,
			Fingerprint:     fingerprint,
			NextHop:         nextHop,
			Expiry:          pathMeta.Expiry,
			MTU:             pathMeta.MTU,
			Latency:         pathMeta.Latency,
			CarbonIntensity: pathMeta.CarbonIntensity,
			Hops:            []Hop{},
		}
		for _, hop := range path.Metadata().Interfaces {
			rpath.Hops = append(rpath.Hops, Hop{IA: hop.IA, IfID: hop.ID})
		}
		if status, ok := statuses[pathprobe.PathKey(path)]; ok {
			rpath.Status = strings.ToLower(string(status.Status))
			rpath.StatusInfo = status.AdditionalInfo
			rpath.Local = status.LocalIP
		}
		res.Paths = append(res.Paths, rpath)
	}
	return res, nil
}


text:
#ifndef _MC_DB_INTERFACE_HEADER_
#define _MC_DB_INTERFACE_HEADER_
#include <stdlib.h>
#include <stdio.h>
#include <vector>
#include <string>
#include "utility.h"
#include "base_define.h"
#include "json.h"
#include "db_interface.h"

#include <libmemcached/memcached.h>

using namespace std;

/*
 * 从memcache获取数据
 * */

typedef struct _Mc_Struct{
	char ip_[IP_WORD_LEN];
	uint32_t port_;
	memcached_st *memc_;
} Mc_Struct;

class McDbInterface : public DbInterface{
	public:
		McDbInterface(const Db_Info_Struct& db_info_struct):
			DbInterface(db_info_struct){
				// load configure
			for(vector<string>::iterator port_it = vec_str_port_.begin();
				port_it != vec_str_port_.end(); port_it++){
				uint32_t n_port = strtoul((*port_it).c_str(), NULL, 10);

				for(vector<string>::iterator ip_it = vec_str_ip_.begin();
					ip_it != vec_str_ip_.end(); ip_it++){

					Mc_Struct mc_struct;
					strncpy(mc_struct.ip_, (*ip_it).c_str(), IP_WORD_LEN);
					mc_struct.port_ = n_port;
					mc_struct.memc_ = NULL;
					vec_mc_server_.push_back(mc_struct);
				}
			}
		}

		virtual ~McDbInterface(){
			for(vector<Mc_Struct>::iterator it = vec_mc_server_.begin();
			it != vec_mc_server_.end(); it ++){
				if((*it).memc_ != NULL)
				memcached_free((*it).memc_);
			}
			vec_mc_server_.clear();
		}

	protected:
		memcached_st* get_mc_raw(SvrIpPort& svr_ip_port){

			uint16_t total_index = svr_ip_port.ip_index_ + svr_ip_port.port_index_ * ip_num_;
			if( NULL == vec_mc_server_[total_index].memc_){
				memcached_st *memc = NULL;
				memcached_return rc;
				memcached_server_st *servers;
				memc = memcached_create(NULL);
				if(NULL == memc){
					LOG_ERROR("mc allocate is error!");
					return NULL;
				}
				servers = memcached_server_list_append(NULL, (char*)vec_mc_server_[total_index].ip_,
						vec_mc_server_[total_index].port_, &rc);
				rc = memcached_server_push(memc, servers);
				memcached_server_free(servers);
				memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, true);	// 非阻塞模式
				memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT, db_info_struct_.timeout_);
				memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_POLL_TIMEOUT, db_info_struct_.timeout_);
				memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 1);	// 单位是分钟

				vec_mc_server_[total_index].memc_ = memc;
				if(NULL == memc){
					LOG_ERROR("connect mc %s:%"PRIu32" error!", vec_mc_server_[total_index].ip_,
							vec_mc_server_[total_index].port_);
					return NULL;
				}
			}
			return vec_mc_server_[total_index].memc_;
		}

	public :
		int s_get(uint16_t type_id, const char* p_str_key, char* &p_result,
				char& split_char, char& second_split_char, uint64_t n_key){
			return get_i(type_id, n_key, 0, p_result, split_char, second_split_char, p_str_key, true);
		}

		// deal with item_key by char*
		int get_i(uint16_t type_id, const char* &p_str_key, uint32_t n_item_key,
				char* &p_result, char& split_char, char& second_split_char){

			initialize();

			split_char = db_info_struct_.value_split_first_char_;
			second_split_char = db_info_struct_.value_split_second_char_;

			return 1;

		}

		int get(uint16_t type_id, uint64_t n_key, char* &p_result,
				char& split_char, char& second_split_char){
			return get_i(type_id, n_key, 0, p_result, split_char, second_split_char);
		}

		int get_i(uint16_t type_id, uint64_t n_key, uint32_t n_item_key,
				char* &p_result, char& split_char, char& second_split_char,
				const char* other_key = NULL, bool other_flag = false) {

			initialize();

			struct timeval tv_begin, tv_end;
			gettimeofday(&tv_begin, NULL);

			SvrIpPort svr_ip_port;
			get_ip_port(n_key, svr_ip_port);
			uint16_t total_index = svr_ip_port.ip_index_ + svr_ip_port.port_index_ * ip_num_;
			memcached_st* memc = get_mc_raw(svr_ip_port);
			if(NULL == memc){
				LOG_ERROR("MC %s:%"PRIu32" CLIENT IS NULL", vec_mc_server_[total_index].ip_,
					vec_mc_server_[total_index].port_);
				return -1;
			}

			char str_key[WORD_LEN];
			size_t key_len = 0;
			if(!other_flag){
				key_len = snprintf(str_key, WORD_LEN, "%"PRIu64, n_key);
			}else{
				key_len = snprintf(str_key, WORD_LEN, "%s", other_key);
			}

			size_t result_str_length = 0;
			uint32_t flags = 0;
			memcached_return rc;

			char* result_str = memcached_get(memc, str_key, key_len, &result_str_length, &flags, &rc);

			gettimeofday(&tv_end, NULL);
			int time_get = (tv_end.tv_sec-tv_begin.tv_sec)*1000 + (tv_end.tv_usec-tv_begin.tv_usec)/1000;
			LOG_INFO("MC %s:%"PRIu32" GET %d ms", vec_mc_server_[total_index].ip_,
					vec_mc_server_[total_index].port_, time_get);

			if (rc == MEMCACHED_SUCCESS){
				if(db_info_struct_.key_value_type_ == 0 && db_info_struct_.value_string_type_ == 0){
					if(db_info_struct_.compress_flag_){
						uLongf un_compr_len = COMPRESS_LEN - 1;
						int err = uncompress((Bytef*)un_compress_[un_compress_index_], &un_compr_len,
							(Bytef*)result_str, result_str_length);
						if (err != Z_OK){
							un_compress_[un_compress_index_][0] = '\0';
							LOG_ERROR("uncompress error!:%d:%d", err, un_compress_index_);
							free(result_str);
							result_str = NULL;
							return -1;
						}
						un_compress_[un_compress_index_][un_compr_len] = '\0';
						result_str_length = un_compr_len;
					}else{
						if(result_str_length > COMPRESS_LEN - 1){
							result_str_length = COMPRESS_LEN - 1;
						}
						memcpy(un_compress_[un_compress_index_], result_str, result_str_length);
						un_compress_[un_compress_index_][result_str_length] = '\0';
					}
					p_result = un_compress_[un_compress_index_];
					un_compress_index_ ++;
				}
				free(result_str);
				result_str = NULL;
				split_char = db_info_struct_.value_split_first_char_;
				second_split_char = db_info_struct_.value_split_second_char_;
				return (int)result_str_length;
			}else{
				LOG_ERROR("MC %s:%"PRIu32" GET %s fail! %d", vec_mc_server_[total_index].ip_,
					vec_mc_server_[total_index].port_, str_key, rc);
				if(rc == MEMCACHED_SERVER_MARKED_DEAD){		// 服务进程死亡,则将长链接指针置空
					vec_mc_server_[total_index].memc_ = NULL;
				}
				return 0;
			}
		}

		int mget(uint16_t type_id, const char* str_keys[], uint32_t keys_num, char* results_array[],
				uint32_t& results_num, char& split_char, char& second_split_char){
			initialize();

			SvrIpPort svr_ip_port;
			get_ip_port(0, svr_ip_port);
			memcached_st* memc = get_mc_raw(svr_ip_port);
			if(NULL == memc){
				return 0;
			}

			size_t key_len_array[keys_num];

			for(uint32_t i = 0; i < keys_num; i++){
				key_len_array[i] = strlen(str_keys[i]);
			}

			struct timeval tv_begin, tv_end;
			gettimeofday(&tv_begin, NULL);

			memcached_return_t rc = memcached_mget(memc, str_keys, key_len_array, keys_num);

			gettimeofday(&tv_end, NULL);
			int time_mget = (tv_end.tv_sec-tv_begin.tv_sec)*1000 + (tv_end.tv_usec-tv_begin.tv_usec)/1000;
			uint16_t total_index = svr_ip_port.ip_index_ + svr_ip_port.port_index_ * ip_num_;
            LOG_INFO("MC %s:%"PRIu32" MGET %d ms", vec_mc_server_[total_index].ip_,
					vec_mc_server_[total_index].port_, time_mget);

			char return_key[MEMCACHED_MAX_KEY];
			char *return_value = NULL;
			size_t return_key_length = 0;
			size_t return_value_length = 0;
			uint32_t flags = 0;
			results_num = 0;
			while((return_value = memcached_fetch(memc, return_key,
							&return_key_length, &return_value_length, &flags, &rc))){
				if(rc == MEMCACHED_SUCCESS){
					//strncpy(un_compress_[un_compress_index_], return_value, return_value_length);
					memcpy(un_compress_[un_compress_index_], return_value, return_value_length);
					results_array[results_num] = un_compress_[un_compress_index_];
					if(return_value != NULL){
						free(return_value);
					}
					un_compress_index_ ++;
					results_num ++;
				}
			}
			return 0;
		}

		int mget(uint16_t type_id, uint64_t n_keys[], uint32_t key_num, MapMResult& map_result,
				char& split_char, char& second_split_char){

			uint32_t n_item_keys[1];
			return mget_i(type_id, n_keys, key_num, n_item_keys, 0, map_result,
					split_char, second_split_char);
		}

		int mget_i(uint16_t type_id, uint64_t n_keys[], uint32_t key_num,
				uint32_t n_item_keys[], uint32_t item_key_num,
				MapMResult& map_result, char& split_char, char& second_split_char){

			initialize();
			split_char = db_info_struct_.value_split_first_char_;
			second_split_char = db_info_struct_.value_split_second_char_;
			return 1;
		}

    protected:
		vector<Mc_Struct> vec_mc_server_;
};

#endif

redpajama_set_name: RedPajamaGithub


text:
#ifndef _COSSB_ENCRYPT_HPP_
#define _COSSB_ENCRYPT_HPP_

namespace cossb {
namespace secure {

class encrypt
{
public:
	encrypt();
	virtual ~encrypt();
};

} /* namespace secure */
} /* namespace cossb */

#endif /* _COSSB_ENCRYPT_HPP_ */


text:
Go libraries and utilities for working with Unicode character data.

_This package was forked from the [cooperhewitt/ucd](https://github.com/cooperhewitt/go-ucd) package but builds everything under its own `aaronland` namespace._

## Example

	package main

	import(
		"fmt"
		"flag"
		"github.com/aaronland/go-ucd/v13"
	)

	func main(){

	     flag.Parse()
	     char := flag.Arg(0)

	     name := ucd.Name(char)
	     fmt.Println(name)
	}

## Tools

The following tools are included in the `cmd` directory. Note however that you will need to [compile them](https://golang.org/cmd/go/#hdr-Compile_and_run_Go_program) yourself. You can do this (and all the steps in-between using the handy [Makefile](Makefile) and the `build` target included in this repository. Like this:

```
$> make tools
```

This will build the `ucd` and `ucd-server` applications and place them in the `bin` directory.

### ucd

	$> bin/ucd A
	LATIN CAPITAL LETTER A

	$> ucd THIS → WAY
	LATIN CAPITAL LETTER T
	LATIN CAPITAL LETTER H
	LATIN CAPITAL LETTER I
	LATIN CAPITAL LETTER S
	SPACE
	RIGHTWARDS ARROW
	SPACE
	LATIN CAPITAL LETTER W
	LATIN CAPITAL LETTER A
	LATIN CAPITAL LETTER Y

`ucd` supports the Unicode Han Data character set, or at least endeavours to. There may still be bugs.

	$> bin/ucd 䍕
	NET; WEB; NETWORK, NET FOR CATCHING RABBIT

### ucd-server

#### Usage

	$> bin/ucd-server --help
	Usage of ./ucd-server:
	  -host="localhost": host
	  -port=8080: port

#### Install as a service

To install as an init.d script, copy the example provided, replace the values of UCD_USER, UCD_DAEMON and UCD_PORT, and start the service.

        $> sudo cp init.d/ucd-server.sh.example /etc/init.d/ucd-server.sh
        $> sudo service ucd-server start

#### as JSON

	$> curl -X GET -s 'http://localhost:8080/?text=♕%20HAT' | python -mjson.tool
	{
	    "Chars": [
	        {
	            "Char": "\u2655",
	            "Hex": "2655",
	            "Name": "WHITE CHESS QUEEN"
	        },
	        {
	            "Char": " ",
	            "Hex": "0020",
	            "Name": "SPACE"
	        },
	        {
	            "Char": "H",
	            "Hex": "0048",
	            "Name": "LATIN CAPITAL LETTER H"
	        },
	        {
	            "Char": "A",
	            "Hex": "0041",
	            "Name": "LATIN CAPITAL LETTER A"
	        },
	        {
	            "Char": "T",
	            "Hex": "0054",
	            "Name": "LATIN CAPITAL LETTER T"
	        }
	    ]
	}

#### As plain text

	$> curl -H 'Accept: text/plain' -s 'http://localhost:8080/?text=♕%20HAT%20WITH%20😸'
	WHITE CHESS QUEEN
	SPACE
	LATIN CAPITAL LETTER H
	LATIN CAPITAL LETTER A
	LATIN CAPITAL LETTER T
	SPACE
	LATIN CAPITAL LETTER W
	LATIN CAPITAL LETTER I
	LATIN CAPITAL LETTER T
	LATIN CAPITAL LETTER H
	SPACE
	GRINNING CAT FACE WITH SMILING EYES

## Versions

`go-ucd` supports Unicode 13.0 as of February 16, 2021 and requires Go [1.16](https://golang.org/doc/go1.16) or higher to compile.

This package exports data defined in the `UnicodeData.txt` and the `Unihan.zip`. Both are available from
http://unicode.org/Public/UCD/latest/ucd/.

If the Unicode consortium releases newer data files and you want or need to
updated your version of `go-ucd` before we do you do so manually by using the
`ucd-build-unicodedata` and `ucd-build-unihan` tools included in the [bin
directory](https://github.com/aaronland/go-ucd/tree/master/bin). For example:

```
go run ./cmd/ucd-build-unicodedata.go > ./unicodedata/unicodedata.go
go run ./cmd/ucd-build-unihan.go > ./unihan/unihan.go
```

_Note: You will need to recompile your `ucd` and `ucd-server` binaries manually._

## Shout outs

Many thanks to friend and Go-friend [Richard Crowley](https://github.com/rcrowley) who is always kind and patient answering my Go-related questions. Go is lovely but Go is weird.

## See also

* http://unicode.org/Public/UCD/latest/ucd/
* http://www.washingtonpost.com/news/the-intersect/wp/2015/02/23/the-surprisingly-complex-reason-you-never-see-emoji-urls/
* https://modelviewculture.com/pieces/i-can-text-you-a-pile-of-poo-but-i-cant-write-my-name


text:
import collections
import dask
from dask import delayed
from dask.diagnostics import ProgressBar
import logging
import multiprocessing
import pandas as pd
import numpy as np
import re
import six
import string

import py_stringsimjoin as ssj
from py_stringsimjoin.filter.overlap_filter import OverlapFilter
from py_stringmatching.tokenizer.qgram_tokenizer import QgramTokenizer
from py_stringmatching.tokenizer.whitespace_tokenizer import WhitespaceTokenizer
from py_stringsimjoin.utils.missing_value_handler import get_pairs_with_missing_value

import py_entitymatching.catalog.catalog_manager as cm
from py_entitymatching.utils.catalog_helper import log_info, get_name_for_key, \
    add_key_column

from py_entitymatching.blocker.blocker import Blocker
import py_entitymatching.utils.generic_helper as gh
from py_entitymatching.utils.validation_helper import validate_object_type
from py_entitymatching.dask.utils import validate_chunks, get_num_partitions, \
    get_num_cores, wrap

logger = logging.getLogger(__name__)

class DaskOverlapBlocker(Blocker):

    def __init__(self):
        self.stop_words = ['a', 'an', 'and', 'are', 'as', 'at',
                           'be', 'by', 'for', 'from',
                           'has', 'he', 'in', 'is', 'it',
                           'its', 'on', 'that', 'the', 'to',
                           'was', 'were', 'will', 'with']
        logger.warning(
            "WARNING THIS BLOCKER IS EXPERIMENTAL AND NOT TESTED. USE AT YOUR OWN "
            "RISK.")

        self.regex_punctuation = re.compile('[%s]' % re.escape(string.punctuation))
        super(DaskOverlapBlocker, self).__init__()

    def block_tables(self, ltable, rtable, l_overlap_attr, r_overlap_attr,
                     rem_stop_words=False, q_val=None, word_level=True, overlap_size=1,
                     l_output_attrs=None, r_output_attrs=None,
                     l_output_prefix='ltable_', r_output_prefix='rtable_',
                     allow_missing=False, verbose=False, show_progress=True,
                     n_ltable_chunks=1, n_rtable_chunks=1):

        """
        WARNING THIS COMMAND IS EXPERIMENTAL AND NOT TESTED. USE AT YOUR OWN RISK.

        Blocks two tables based on the overlap of token sets of attribute
        values. Finds tuple pairs from left and right tables such that the overlap
        between (a) the set of tokens obtained by tokenizing the value of
        attribute l_overlap_attr of a tuple from the left table, and (b) the
        set of tokens obtained by tokenizing the value of attribute
        r_overlap_attr of a tuple from the right table, is above a certain
        threshold.

        Args:
            ltable (DataFrame): The left input table.

            rtable (DataFrame): The right input table.

            l_overlap_attr (string): The overlap attribute in left table.

            r_overlap_attr (string): The overlap attribute in right table.

            rem_stop_words (boolean): A flag to indicate whether stop words
             (e.g., a, an, the) should be removed from the token sets of the
             overlap attribute values (defaults to False).

            q_val (int): The value of q to use if the overlap attributes
             values are to be tokenized as qgrams (defaults to None).

            word_level (boolean): A flag to indicate whether the overlap
             attributes should be tokenized as words (i.e, using whitespace
             as delimiter) (defaults to True).

            overlap_size (int): The minimum number of tokens that must
             overlap (defaults to 1).
            l_output_attrs (list): A list of attribute names from the left
                table to be included in the output candidate set (defaults
                to None).
            r_output_attrs (list): A list of attribute names from the right
                table to be included in the output candidate set  (defaults
                to None).

            l_output_prefix (string): The prefix to be used for the attribute names
                                   coming from the left table in the output
                                   candidate set (defaults to 'ltable\_').
            r_output_prefix (string): The prefix to be used for the attribute names
                                   coming from the right table in the output
                                   candidate set (defaults to 'rtable\_').
            allow_missing (boolean): A flag to indicate whether tuple pairs
                                     with missing value in at least one of the
                                     blocking attributes should be included in
                                     the output candidate set (defaults to
                                     False). If this flag is set to True, a
                                     tuple in ltable with missing value in the
                                     blocking attribute will be matched with
                                     every tuple in rtable and vice versa.

            verbose (boolean): A flag to indicate whether the debug
                information should be logged (defaults to False).

            show_progress (boolean): A flag to indicate whether progress should
                be displayed to the user (defaults to True).

            n_ltable_chunks (int): The number of partitions to split the left table (
                                    defaults to 1). If it is set to -1, then the number of
                                    partitions is set to the number of cores in the
                                    machine.
            n_rtable_chunks (int): The number of partitions to split the right table (
                                    defaults to 1). If it is set to -1, then the number of
                                    partitions is set to the number of cores in the
                                    machine.

        Returns:
            A candidate set of tuple pairs that survived blocking (DataFrame).
        Raises:
            AssertionError: If `ltable` is not of type pandas
                DataFrame.

            AssertionError: If `rtable` is not of type pandas
                DataFrame.

            AssertionError: If `l_overlap_attr` is not of type string.

            AssertionError: If `r_overlap_attr` is not of type string.

            AssertionError: If `l_output_attrs` is not of type of
             list.

            AssertionError: If `r_output_attrs` is not of type of
             list.

            AssertionError: If the values in `l_output_attrs` is not of type
             string.

            AssertionError: If the values in `r_output_attrs` is not of type
             string.

            AssertionError: If `l_output_prefix` is not of type
             string.

            AssertionError: If `r_output_prefix` is not of type
             string.

            AssertionError: If `q_val` is not of type int.

            AssertionError: If `word_level` is not of type boolean.

            AssertionError: If `overlap_size` is not of type int.

            AssertionError: If `verbose` is not of type
             boolean.

            AssertionError: If `allow_missing` is not of type boolean.

            AssertionError: If `show_progress` is not of type
             boolean.

            AssertionError: If `n_ltable_chunks` is not of type
             int.

            AssertionError: If `n_rtable_chunks` is not of type
             int.

            AssertionError: If `l_overlap_attr` is not in the ltable
             columns.

            AssertionError: If `r_block_attr` is not in the rtable columns.

            AssertionError: If `l_output_attrs` are not in the ltable.

            AssertionError: If `r_output_attrs` are not in the rtable.

            SyntaxError: If `q_val` is set to a valid value and
                `word_level` is set to True.

            SyntaxError: If `q_val` is set to None and
                `word_level` is set to False.

        Examples:
            >>> from py_entitymatching.dask.dask_overlap_blocker import DaskOverlapBlocker
            >>> A = em.read_csv_metadata('path_to_csv_dir/table_A.csv', key='ID')
            >>> B = em.read_csv_metadata('path_to_csv_dir/table_B.csv', key='ID')
            >>> ob = DaskOverlapBlocker()
            # Use all cores
            # # Use word-level tokenizer
            >>> C1 = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'], word_level=True, overlap_size=1, n_ltable_chunks=-1, n_rtable_chunks=-1)
            # # Use q-gram tokenizer
            >>> C2 = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'], word_level=False, q_val=2, n_ltable_chunks=-1, n_rtable_chunks=-1)
            # # Include all possible missing values
            >>> C3 = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'], allow_missing=True, n_ltable_chunks=-1, n_rtable_chunks=-1)
        """
        logger.warning(
            "WARNING THIS COMMAND IS EXPERIMENTAL AND NOT TESTED. USE AT YOUR OWN "
            "RISK.")

        # Input validations
        self.validate_types_params_tables(ltable, rtable, l_output_attrs,
                                          r_output_attrs, l_output_prefix,
                                          r_output_prefix, verbose, n_ltable_chunks, n_rtable_chunks)
        self.validate_types_other_params(l_overlap_attr, r_overlap_attr,
                                         rem_stop_words, q_val, word_level, overlap_size)
        self.validate_allow_missing(allow_missing)
        self.validate_show_progress(show_progress)
        self.validate_overlap_attrs(ltable, rtable, l_overlap_attr, r_overlap_attr)
        self.validate_output_attrs(ltable, rtable, l_output_attrs, r_output_attrs)
        self.validate_word_level_qval(word_level, q_val)

        log_info(logger, 'Required metadata: ltable key, rtable key', verbose)

        l_key, r_key = cm.get_keys_for_ltable_rtable(ltable, rtable, logger, verbose)

        # validate metadata
        cm._validate_metadata_for_table(ltable, l_key, 'ltable', logger, verbose)
        cm._validate_metadata_for_table(rtable, r_key, 'rtable', logger, verbose)

        # validate input table chunks
        validate_object_type(n_ltable_chunks, int, 'Parameter n_ltable_chunks')
        validate_object_type(n_rtable_chunks, int,
                             'Parameter n_rtable_chunks')
        validate_chunks(n_ltable_chunks)
        validate_chunks(n_rtable_chunks)

        if n_ltable_chunks == -1:
            n_ltable_chunks = multiprocessing.cpu_count()

        ltable_chunks = np.array_split(ltable, n_ltable_chunks)

        # preprocess/tokenize ltable
        if word_level == True:
            tokenizer = WhitespaceTokenizer(return_set=True)
        else:
            tokenizer = QgramTokenizer(qval=q_val, return_set=True)

        preprocessed_tokenized_ltbl = []

        # Construct DAG for preprocessing/tokenizing ltable chunks
        start_row_id = 0
        for i in range(len(ltable_chunks)):
            result = delayed(self.process_tokenize_block_attr)(ltable_chunks[i][
                                                                  l_overlap_attr],
                                                              start_row_id,
                                                              rem_stop_words, tokenizer)
            preprocessed_tokenized_ltbl.append(result)
            start_row_id += len(ltable_chunks[i])
        preprocessed_tokenized_ltbl = delayed(wrap)(preprocessed_tokenized_ltbl)

        # Execute the DAG
        if show_progress:
            with ProgressBar():
                logger.info('Preprocessing/tokenizing ltable')
                preprocessed_tokenized_ltbl_vals = preprocessed_tokenized_ltbl.compute(
                scheduler="processes", num_workers=multiprocessing.cpu_count())
        else:
            preprocessed_tokenized_ltbl_vals = preprocessed_tokenized_ltbl.compute(
                scheduler="processes", num_workers=multiprocessing.cpu_count())

        ltable_processed_dict = {}
        for i in range(len(preprocessed_tokenized_ltbl_vals)):
            ltable_processed_dict.update(preprocessed_tokenized_ltbl_vals[i])

        # build inverted index
        inverted_index = self.build_inverted_index(ltable_processed_dict)

        if n_rtable_chunks == -1:
            n_rtable_chunks = multiprocessing.cpu_count()

        rtable_chunks = np.array_split(rtable, n_rtable_chunks)

        # Construct the DAG for probing
        probe_result = []
        start_row_id = 0
        for i in range(len(rtable_chunks)):
            result = delayed(self.probe)(rtable_chunks[i][r_overlap_attr],
                                         inverted_index, start_row_id, rem_stop_words,
                                         tokenizer, overlap_size)
            probe_result.append(result)
            start_row_id += len(rtable_chunks[i])
        probe_result = delayed(wrap)(probe_result)

        # Execute the DAG for probing
        if show_progress:
            with ProgressBar():
                logger.info('Probing using rtable')
                probe_result = probe_result.compute(scheduler="processes",
                                            num_workers=multiprocessing.cpu_count())
        else:
            probe_result = probe_result.compute(scheduler="processes",
                                                num_workers=multiprocessing.cpu_count())

        # construct a minimal dataframe that can be used to add more attributes
        flat_list = [item for sublist in probe_result for item in sublist]
        tmp = pd.DataFrame(flat_list, columns=['fk_ltable_rid', 'fk_rtable_rid'])
        fk_ltable = ltable.iloc[tmp.fk_ltable_rid][l_key].values
        fk_rtable = rtable.iloc[tmp.fk_rtable_rid][r_key].values
        id_vals = list(range(len(flat_list)))

        candset = pd.DataFrame.from_dict(
            {'_id': id_vals, l_output_prefix+l_key: fk_ltable, r_output_prefix+r_key: fk_rtable})

        # set the properties for the candidate set
        cm.set_key(candset, '_id')
        cm.set_fk_ltable(candset, 'ltable_'+l_key)
        cm.set_fk_rtable(candset, 'rtable_'+r_key)
        cm.set_ltable(candset, ltable)
        cm.set_rtable(candset, rtable)

        ret_candset = gh.add_output_attributes(candset, l_output_attrs=l_output_attrs,
                                               r_output_attrs=r_output_attrs,
                                               l_output_prefix=l_output_prefix,
                                               r_output_prefix=r_output_prefix,
                                               validate=False)

        # handle missing values
        if allow_missing:
            missing_value_pairs = get_pairs_with_missing_value(ltable, rtable, l_key,
                                                           r_key, l_overlap_attr,
                                                           r_overlap_attr,
                                                           l_output_attrs,
                                                           r_output_attrs,
                                                           l_output_prefix,
                                                           r_output_prefix, False, False)
            missing_value_pairs.insert(0, '_id', range(len(ret_candset),
                                                       len(ret_candset)+len(missing_value_pairs)))

            if len(missing_value_pairs) > 0:
                ret_candset = pd.concat([ret_candset, missing_value_pairs], ignore_index=True, sort=False)
                cm.set_key(ret_candset, '_id')
                cm.set_fk_ltable(ret_candset, 'ltable_' + l_key)
                cm.set_fk_rtable(ret_candset, 'rtable_' + r_key)
                cm.set_ltable(ret_candset, ltable)
                cm.set_rtable(ret_candset, rtable)

        # Return the final candidate set to user.
        return ret_candset

    def block_candset(self, candset, l_overlap_attr, r_overlap_attr,
                      rem_stop_words=False, q_val=None, word_level=True,
                      overlap_size=1, allow_missing=False,
                      verbose=False, show_progress=True, n_chunks=-1):

        """
        WARNING THIS COMMAND IS EXPERIMENTAL AND NOT TESTED. USE AT YOUR OWN RISK.

        Blocks an input candidate set of tuple pairs based on the overlap
        of token sets of attribute values. Finds tuple pairs from an input
        candidate set of tuple pairs such that
        the overlap between (a) the set of tokens obtained by tokenizing the
        value of attribute l_overlap_attr of the left tuple in a tuple pair,
        and (b) the set of tokens obtained by tokenizing the value of
        attribute r_overlap_attr of the right tuple in the tuple pair,
        is above a certain threshold.

        Args:
            candset (DataFrame): The input candidate set of tuple pairs.

            l_overlap_attr (string): The overlap attribute in left table.

            r_overlap_attr (string): The overlap attribute in right table.

            rem_stop_words (boolean): A flag to indicate whether stop words
                                      (e.g., a, an, the) should be removed
                                      from the token sets of the overlap
                                      attribute values (defaults to False).

            q_val (int): The value of q to use if the overlap attributes values
                         are to be tokenized as qgrams (defaults to None).

            word_level (boolean): A flag to indicate whether the overlap
                                  attributes should be tokenized as words
                                  (i.e, using whitespace as delimiter)
                                  (defaults to True).

            overlap_size (int): The minimum number of tokens that must overlap
                                (defaults to 1).

            allow_missing (boolean): A flag to indicate whether tuple pairs
                                     with missing value in at least one of the
                                     blocking attributes should be included in
                                     the output candidate set (defaults to
                                     False). If this flag is set to True, a
                                     tuple pair with missing value in either
                                     blocking attribute will be retained in the
                                     output candidate set.

            verbose (boolean): A flag to indicate whether the debug information

                should be logged (defaults to False).

            show_progress (boolean): A flag to indicate whether progress should
                                     be displayed to the user (defaults to True).

            n_chunks (int): The number of partitions to split the candidate set. If it
                            is set to -1, the number of partitions will be set to the
                            number of cores in the machine.

        Returns:
            A candidate set of tuple pairs that survived blocking (DataFrame).

        Raises:
            AssertionError: If `candset` is not of type pandas
                DataFrame.
            AssertionError: If `l_overlap_attr` is not of type string.
            AssertionError: If `r_overlap_attr` is not of type string.
            AssertionError: If `q_val` is not of type int.
            AssertionError: If `word_level` is not of type boolean.
            AssertionError: If `overlap_size` is not of type int.
            AssertionError: If `verbose` is not of type
                boolean.
            AssertionError: If `allow_missing` is not of type boolean.
            AssertionError: If `show_progress` is not of type
                boolean.
            AssertionError: If `n_chunks` is not of type
                int.
            AssertionError: If `l_overlap_attr` is not in the ltable
                columns.
            AssertionError: If `r_block_attr` is not in the rtable columns.
            SyntaxError: If `q_val` is set to a valid value and
                `word_level` is set to True.
            SyntaxError: If `q_val` is set to None and
                `word_level` is set to False.
        Examples:
            >>> import py_entitymatching as em
            >>> from py_entitymatching.dask.dask_overlap_blocker import DaskOverlapBlocker
            >>> A = em.read_csv_metadata('path_to_csv_dir/table_A.csv', key='ID')
            >>> B = em.read_csv_metadata('path_to_csv_dir/table_B.csv', key='ID')
            >>> ob = DaskOverlapBlocker()
            >>> C = ob.block_tables(A, B, 'address', 'address', l_output_attrs=['name'], r_output_attrs=['name'])

            >>> D1 = ob.block_candset(C, 'name', 'name', allow_missing=True)
            # Include all possible tuple pairs with missing values
            >>> D2 = ob.block_candset(C, 'name', 'name', allow_missing=True)
            # Execute blocking using multiple cores
            >>> D3 = ob.block_candset(C, 'name', 'name', n_chunks=-1)
            # Use q-gram tokenizer
            >>> D2 = ob.block_candset(C, 'name', 'name', word_level=False, q_val=2)

        """
        logger.warning(
            "WARNING THIS BLOCKER IS EXPERIMENTAL AND NOT TESTED. USE AT YOUR OWN "
            "RISK.")

        # Validate input parameters
        self.validate_types_params_candset(candset, verbose, show_progress, n_chunks)
        self.validate_types_other_params(l_overlap_attr, r_overlap_attr,
                                         rem_stop_words, q_val, word_level, overlap_size)

        # get and validate metadata
        log_info(logger,
                 'Required metadata: cand.set key, fk ltable, fk rtable, '
                 'ltable, rtable, ltable key, rtable key', verbose)

        # # get metadata
        key, fk_ltable, fk_rtable, ltable, rtable, l_key, r_key = cm.get_metadata_for_candset(
            candset, logger, verbose)

        # # validate metadata
        cm._validate_metadata_for_candset(candset, key, fk_ltable, fk_rtable,
                                          ltable, rtable, l_key, r_key,
                                          logger, verbose)

        # validate overlap attrs
        self.validate_overlap_attrs(ltable, rtable, l_overlap_attr,
                                    r_overlap_attr)

        # validate word_level and q_val
        self.validate_word_level_qval(word_level, q_val)

        # validate number of chunks
        validate_object_type(n_chunks, int, 'Parameter n_chunks')
        validate_chunks(n_chunks)

        # # do projection before merge
        l_df = ltable[[l_key, l_overlap_attr]]
        r_df = rtable[[r_key, r_overlap_attr]]

        # # set index for convenience
        l_df = l_df.set_index(l_key, drop=False)
        r_df = r_df.set_index(r_key, drop=False)

        # # case the overlap attribute to string if required.
        l_df.is_copy, r_df.is_copy = False, False  # to avoid setwithcopy warning
        ssj.dataframe_column_to_str(l_df, l_overlap_attr, inplace=True)
        ssj.dataframe_column_to_str(r_df, r_overlap_attr, inplace=True)

        if word_level == True:
            tokenizer = WhitespaceTokenizer(return_set=True)
        else:
            tokenizer = QgramTokenizer(return_set=True)

        n_chunks = get_num_partitions(n_chunks, len(candset))
        c_splits = np.array_split(candset, n_chunks)
        valid_splits = []

        # Create DAG
        for i in range(n_chunks):
            result = delayed(self._block_candset_split)(c_splits[i], l_df, r_df, l_key,
                                                       r_key, l_overlap_attr,
                                                       r_overlap_attr, fk_ltable,
                                                       fk_rtable, allow_missing,
                                                       rem_stop_words, tokenizer, overlap_size)
            valid_splits.append(result)
        valid_splits = delayed(wrap)(valid_splits)

        # Execute the DAG
        if show_progress:
            with ProgressBar():
                valid_splits = valid_splits.compute(scheduler="processes",
                                                    num_workers=get_num_cores())
        else:
            valid_splits = valid_splits.compute(scheduler="processes",
                                                num_workers=get_num_cores())

        valid = sum(valid_splits, [])

        # construct output table
        if len(candset) > 0:
            out_table = candset[valid]
        else:
            out_table = pd.DataFrame(columns=candset.columns)

        # update the catalog
        cm.set_candset_properties(out_table, key, fk_ltable, fk_rtable,
                                  ltable, rtable)

        # return the output table
        return out_table

    # ------------ helper functions ------- #

    def _block_candset_split(self, c_df, l_df, r_df, l_key, r_key, l_block_attr,
                             r_block_attr, fk_ltable, fk_rtable, allow_missing,
                             rem_stop_words, tokenizer, overlap_threshold):
        valid = []

        col_names = list(c_df.columns)
        lkey_idx = col_names.index(fk_ltable)
        rkey_idx = col_names.index(fk_rtable)

        l_dict = {}
        r_dict = {}

        for row in c_df.itertuples(index=False):
            row_lkey = row[lkey_idx]
            if row_lkey not in l_dict:
                l_dict[row_lkey] = l_df.loc[row_lkey, l_block_attr]
            l_val = l_dict[row_lkey]

            row_rkey = row[rkey_idx]
            if row_rkey not in r_dict:
                r_dict[row_rkey] = r_df.loc[row_rkey, r_block_attr]
            r_val = r_dict[row_rkey]
            if allow_missing:
                if pd.isnull(l_val) or pd.isnull(r_val) or l_val == r_val:
                    valid.append(True)
                else:
                    valid.append(False)
            else:
                if pd.notnull(l_val) and pd.notnull(r_val):
                    l_tokens = self._process_tokenize_block_str(l_val, rem_stop_words,
                                                                tokenizer)
                    r_tokens = self._process_tokenize_block_str(r_val, rem_stop_words,
                                                                tokenizer)
                    overlap = len(set(l_tokens).intersection(r_tokens))
                    if overlap > overlap_threshold:
                        valid.append(True)
                    else:
                        valid.append(False)
                else:
                    valid.append(False)
        return valid

    def process_tokenize_block_attr(self, block_column_values,
                                    start_row_id, should_rem_stop_words, tokenizer):
        result_vals = {}
        row_id = start_row_id
        for s in block_column_values:
            if not s or pd.isnull(s):
                row_id += 1
                continue
            val = self._process_tokenize_block_str(s, should_rem_stop_words, tokenizer)
            result_vals[row_id] = val
            row_id += 1
        return result_vals

    def build_inverted_index(self, process_tok_vals):
        index = collections.defaultdict(set)
        for key in process_tok_vals:
            for val in process_tok_vals[key]:
                index[val].add(key)
        return index

    def _find_candidates(self, probe_tokens, inverted_index):
        overlap_candidates = {}
        for token in probe_tokens:
            candidates = inverted_index.get(token, [])
            for candidate in candidates:
                overlap_candidates[candidate] = overlap_candidates.get(candidate, 0) + 1
        return overlap_candidates

    def probe(self, block_column_values, inverted_index, start_row_id,
              should_rem_stop_words, tokenizer, overlap_threshold):
        result = []
        row_id = start_row_id
        for s in block_column_values:
            if not s:
                row_id += 1
                continue
            probe_tokens = self._process_tokenize_block_str(s, should_rem_stop_words,
                                                            tokenizer)
            candidates = self._find_candidates(probe_tokens, inverted_index)
            for cand, overlap in candidates.items():
                if overlap >= overlap_threshold:
                    result.append((cand, row_id))
            row_id += 1
        return result

    def _process_tokenize_block_str(self, s, should_rem_stop_words, tokenizer):
        if not s or pd.isnull(s):
            return s
        if isinstance(s, bytes):
            s = s.decode('utf-8', 'ignore')
        s = s.lower()
        s = self.regex_punctuation.sub('', s)
        tokens = list(set(s.strip().split()))
        if should_rem_stop_words:
            tokens = [token for token in tokens if token not in self.stop_words]
        s = ' '.join(tokens)
        tokenized_str = tokenizer.tokenize(s)
        return tokenized_str

    def validate_types_params_tables(self, ltable, rtable,
                                     l_output_attrs, r_output_attrs, l_output_prefix,
                                     r_output_prefix, verbose, n_ltable_chunks,
                                     n_rtable_chunks):

        validate_object_type(ltable, pd.DataFrame, error_prefix='Input left table')
        validate_object_type(rtable, pd.DataFrame, error_prefix='Input right table')

        if l_output_attrs:
            validate_object_type(l_output_attrs, list, 'Output attributes of left table')
            for x in l_output_attrs:
                validate_object_type(x, six.string_types, 'An output attribute name of left table')

        if r_output_attrs:
            validate_object_type(r_output_attrs, list, 'Output attributes of right table')
            for x in r_output_attrs:
                validate_object_type(x, six.string_types, 'An output attribute name of right table')

        validate_object_type(l_output_prefix, six.string_types, 'Output prefix of left table')
        validate_object_type(r_output_prefix, six.string_types, 'Output prefix of right table')

        validate_object_type(verbose, bool, 'Parameter verbose')
        validate_object_type(n_ltable_chunks, int, 'Parameter n_ltable_chunks')
        validate_object_type(n_rtable_chunks, int, 'Parameter n_rtable_chunks')

    def validate_types_other_params(self, l_overlap_attr, r_overlap_attr,
                                    rem_stop_words, q_val,
                                    word_level, overlap_size):
        validate_object_type(l_overlap_attr, six.string_types, error_prefix='Overlap attribute name of left table')
        validate_object_type(r_overlap_attr, six.string_types, error_prefix='Overlap attribute name of right table')

        validate_object_type(rem_stop_words, bool, error_prefix='Parameter rem_stop_words')

        if q_val != None and not isinstance(q_val, int):
            logger.error('Parameter q_val is not of type int')
            raise AssertionError('Parameter q_val is not of type int')

        validate_object_type(word_level, bool, error_prefix='Parameter word_level')
        validate_object_type(overlap_size, int, error_prefix='Parameter overlap_size')

    # validate the overlap attrs
    def validate_overlap_attrs(self, ltable, rtable, l_overlap_attr,
                               r_overlap_attr):
        if not isinstance(l_overlap_attr, list):
            l_overlap_attr = [l_overlap_attr]
        assert set(l_overlap_attr).issubset(
            ltable.columns) is True, 'Left block attribute is not in the left table'

        if not isinstance(r_overlap_attr, list):
            r_overlap_attr = [r_overlap_attr]
        assert set(r_overlap_attr).issubset(
            rtable.columns) is True, 'Right block attribute is not in the right table'

    # validate word_level and q_val
    def validate_word_level_qval(self, word_level, q_val):
        if word_level == True and q_val != None:
            raise SyntaxError(
                'Parameters word_level and q_val cannot be set together; Note that word_level is '
                'set to True by default, so explicity set word_level=false to use qgram with the '
                'specified q_val')

        if word_level == False and q_val == None:
            raise SyntaxError(
                'Parameters word_level and q_val cannot be unset together; Note that q_val is '
                'set to None by default, so if you want to use qgram then '
                'explictiy set word_level=False and specify the q_val')

    def block_tuples(self, ltuple, rtuple, l_overlap_attr, r_overlap_attr,
                     rem_stop_words=False, q_val=None, word_level=True,
                     overlap_size=1, allow_missing=False):
        """Blocks a tuple pair based on the overlap of token sets of attribute values.

        Args:
            ltuple (Series): The input left tuple.

            rtuple (Series): The input right tuple.

            l_overlap_attr (string): The overlap attribute in left tuple.

            r_overlap_attr (string): The overlap attribute in right tuple.

            rem_stop_words (boolean): A flag to indicate whether stop words
                                      (e.g., a, an, the) should be removed
                                      from the token sets of the overlap
                                      attribute values (defaults to False).

            q_val (int): A value of q to use if the overlap attributes values
                         are to be tokenized as qgrams (defaults to None).

            word_level (boolean): A flag to indicate whether the overlap
                                  attributes should be tokenized as words
                                  (i.e, using whitespace as delimiter)
                                  (defaults to True).

            overlap_size (int): The minimum number of tokens that must overlap
                                (defaults to 1).

            allow_missing (boolean): A flag to indicate whether a tuple pair
                                     with missing value in at least one of the
                                     blocking attributes should be blocked
                                     (defaults to False). If this flag is set
                                     to True, the pair will be kept if either
                                     ltuple has missing value in l_block_attr
                                     or rtuple has missing value in r_block_attr
                                     or both.

        Returns:
            A status indicating if the tuple pair is blocked (boolean).

        Examples:
            >>> import py_entitymatching as em
            >>> A = em.read_csv_metadata('path_to_csv_dir/table_A.csv', key='ID')
            >>> B = em.read_csv_metadata('path_to_csv_dir/table_B.csv', key='ID')
            >>> ob = em.OverlapBlocker()
            >>> status = ob.block_tuples(A.loc[0], B.loc[0], 'address', 'address')

        """

        # validate data types of input parameters specific to overlap blocker
        self.validate_types_other_params(l_overlap_attr, r_overlap_attr,
                                         rem_stop_words, q_val,
                                         word_level, overlap_size)

        # validate word_level and q_val
        self.validate_word_level_qval(word_level, q_val)

        # determine which tokenizer to use
        if word_level == True:
            # # create a whitespace tokenizer
            tokenizer = WhitespaceTokenizer(return_set=True)
        else:
            # # create a qgram tokenizer
            tokenizer = QgramTokenizer(qval=q_val, return_set=True)

        # # cleanup the tuples from non-ascii characters, punctuations, and stop words
        l_val = self.cleanup_tuple_val(ltuple[l_overlap_attr], rem_stop_words)
        r_val = self.cleanup_tuple_val(rtuple[r_overlap_attr], rem_stop_words)

        # create a filter for overlap similarity
        overlap_filter = OverlapFilter(tokenizer, overlap_size,
                                       allow_missing=allow_missing)

        return overlap_filter.filter_pair(l_val, r_val)

    def cleanup_tuple_val(self, val, rem_stop_words):
        if pd.isnull(val):
            return val

        return self.process_string(val, rem_stop_words)

    def process_string(self, input_string, rem_stop_words):
        if not input_string:
            return input_string

        if isinstance(input_string, bytes):
            input_string = input_string.decode('utf-8', 'ignore')
        input_string = input_string.lower()

        input_string = self.rem_punctuations(input_string)

        # remove stopwords
        # chop the attribute values and convert into a set
        val_chopped = list(set(input_string.strip().split()))

        # remove stop words
        if rem_stop_words:
            val_chopped_no_stopwords = self.rem_stopwords(val_chopped)
            val_joined = ' '.join(val_chopped_no_stopwords)
        else:
            val_joined = ' '.join(val_chopped)

        return val_joined

    def rem_punctuations(self, s):
        return self.regex_punctuation.sub('', s)

    def rem_stopwords(self, lst):
        return [t for t in lst if t not in self.stop_words]


text:
package bench

import (
	"crypto/rand"
	"fmt"
	"os"
	"time"

	"github.com/allmad/madq/go/bio"
	"github.com/allmad/madq/go/fs"
	"github.com/allmad/madq/go/ptrace"
	"github.com/chzyer/flow"
	"github.com/chzyer/logex"
	"github.com/chzyer/test"
)

type FsFile struct {
	Trace     bool
	Mem       bool
	BenchCnt  int    `name:"count" desc:"bench size" default:"200"`
	BlockSize int    `name:"bs" desc:"block size" default:"200"`
	Stat      bool   `name:"stat"`
	RStat     bool   `name:"rstat"`
	Dir       string `desc:"test directory path" default:"/tmp/madq/bench/fsfile"`
}

func (f *FsFile) FlaglyDesc() string {
	return "benchmark file Read/Write"
}

func (cfg *FsFile) BenchWrite(f *flow.Flow, volcfg *fs.VolumeConfig, buf []byte) error {
	f = f.Fork(0)
	defer f.Close()

	if cfg.Trace {
		EnableTrace()
		defer DisableTrace()
	}

	now := time.Now()
	var size ptrace.Size
	size.AddInt(cfg.BlockSize * cfg.BenchCnt)

	defer func() {
		duration := time.Now().Sub(now)

		if cfg.Stat {
			println(fs.Stat.String())
		}
		println("write performance:", size.Rate(duration).String())
	}()

	vol, err := fs.NewVolume(f, volcfg)
	if err != nil {
		return err
	}
	defer vol.Close()

	fd, err := vol.Open("/hello", os.O_CREATE)
	if err != nil {
		return fmt.Errorf("error in openfile: %v", err)
	}
	defer fd.Close()

	for i := 0; i < cfg.BenchCnt; i++ {
		fd.Write(buf)
	}
	fd.Sync()
	return nil
}

func (cfg *FsFile) BenchRead(f *flow.Flow, volcfg *fs.VolumeConfig, expect []byte) error {
	f = f.Fork(0)
	defer f.Close()

	now := time.Now()
	var size ptrace.Size
	size.AddInt(cfg.BlockSize * cfg.BenchCnt)
	defer func() {
		duration := time.Now().Sub(now)

		if cfg.RStat {
			println(fs.Stat.String())
		}
		println("read performance:", size.Rate(duration).String())
	}()

	vol, err := fs.NewVolume(f, volcfg)
	if err != nil {
		return err
	}
	defer vol.Close()

	fd, err := vol.Open("/hello", 0)
	if err != nil {
		return fmt.Errorf("error in openfile: %v", err)
	}
	defer fd.Close()

	buf := make([]byte, len(expect))
	for i := 0; i < cfg.BenchCnt; i++ {
		_, err := fd.Read(buf)
		if err != nil {
			return err
		}
	}
	return nil
}

func (cfg *FsFile) FlaglyHandle(f *flow.Flow) error {
	defer f.Close()

	buf := make([]byte, cfg.BlockSize)
	rand.Read(buf)

	volcfg := &fs.VolumeConfig{}

	if cfg.Mem {
		volcfg.Delegate = bio.NewHybrid(test.NewMemDisk(), fs.BlockBit)
	} else {
		vs, err := fs.NewVolumeSource(cfg.Dir)
		if err != nil {
			return err
		}
		defer vs.Close()
		volcfg.Delegate = vs
	}

	f.Add(1)
	defer f.Done()

	if err := cfg.BenchWrite(f, volcfg, buf); err != nil {
		return err
	}
	fs.ResetStat()
	if err := cfg.BenchRead(f, volcfg, buf); err != nil {
		logex.Error(err)
		return err
	}

	return nil
}


text:
package com.github.jasminb.jsonapi.annotations;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
 * Annotation used to configure links field in JSON API resources.
 *
 * @author jbegic
 */

@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
public @interface Links {
}


text:
module.exports = {
  apiBaseUrl: ''
};


text:
        private Type arraySuperType = null;
        private Type arraySuperType() {
            // initialized lazily to avoid problems during compiler startup
            if (arraySuperType == null) {
                synchronized (this) {
                    if (arraySuperType == null) {
                        // JLS 10.8: all arrays implement Cloneable and Serializable.
                        arraySuperType = makeCompoundType(List.of(syms.serializableType,
                                                                  syms.cloneableType),
                                                          syms.objectType);
                    }
                }
            }
            return arraySuperType;
        }


text:
var querystring = require("querystring");

function start(response) {
    console.log("Request handler 'start' was called.");

    var body =
        '' +
        '' +
        '<meta http-equiv="Content-Type" content="text/html; ' +
        'charset=UTF-8" />' +
        '' +
        '' +
        '<form action="/upload" method="post">' +
        '<textarea name="text" rows="20" cols="60"></textarea>' +
        '<input type="submit" value="Submit text" />' +
        '</form>' +
        '' +
        '';

    response.writeHead(200, {
        "Content-Type" : "text/html"
    });
    response.write(body);
    response.end();
}

function upload(response, postData) {
    console.log("Request handler 'upload' was called.");
    response.writeHead(200, {
        "Content-Type" : "text/plain"
    });
    response.write("You've sent the text: "+ querystring.parse(postData).text);
    response.end();
}

exports.start = start;
exports.upload = upload;


text:
package com.davidbracewell.io;

import com.google.common.base.Preconditions;
import lombok.NonNull;

import java.io.ByteArrayInputStream;
import java.nio.charset.Charset;

/**
 * Convenience method for detecting the character set of a byte array.
 *
 * @author David B. Bracewell
 */
public class CharsetDetector {

   /**
    * Detects the character set for the buffer.
    *
    * @param buffer The buffer
    * @param offset where to start
    * @param length the length to read
    * @return The detected charset or null
    */
   public static Charset detect(@NonNull byte[] buffer, int offset, int length) {
      Preconditions.checkArgument(length > 0);
      Preconditions.checkArgument(offset >= 0);

      final com.ibm.icu.text.CharsetDetector detector = new com.ibm.icu.text.CharsetDetector();
      try {
         detector.setText(new ByteArrayInputStream(buffer, offset, length));
         return Charset.forName(detector.detect().getName());
      } catch (Exception e) {
         return null;
      }
   }

}// END OF CharsetDetector


text:
#pragma once
#include <aws/mediatailor/MediaTailor_EXPORTS.h>
#include <aws/core/utils/memory/stl/AWSString.h>
#include <aws/mediatailor/model/PrefetchConsumption.h>
#include <aws/mediatailor/model/PrefetchRetrieval.h>
#include <utility>

namespace Aws
{
template<typename RESULT_TYPE>
class AmazonWebServiceResult;

namespace Utils
{
namespace Json
{
  class JsonValue;
} // namespace Json
} // namespace Utils
namespace MediaTailor
{
namespace Model
{
  class AWS_MEDIATAILOR_API GetPrefetchScheduleResult
  {
  public:
    GetPrefetchScheduleResult();
    GetPrefetchScheduleResult(const Aws::AmazonWebServiceResult<Aws::Utils::Json::JsonValue>& result);
    GetPrefetchScheduleResult& operator=(const Aws::AmazonWebServiceResult<Aws::Utils::Json::JsonValue>& result);

    /**
     *
The Amazon Resource Name (ARN) of the prefetch schedule.

     */
    inline const Aws::String& GetArn() const{ return m_arn; }

    /**
     *
The Amazon Resource Name (ARN) of the prefetch schedule.

     */
    inline void SetArn(const Aws::String& value) { m_arn = value; }

    /**
     *
The Amazon Resource Name (ARN) of the prefetch schedule.

     */
    inline void SetArn(Aws::String&& value) { m_arn = std::move(value); }

    /**
     *
The Amazon Resource Name (ARN) of the prefetch schedule.

     */
    inline void SetArn(const char* value) { m_arn.assign(value); }

    /**
     *
The Amazon Resource Name (ARN) of the prefetch schedule.

     */
    inline GetPrefetchScheduleResult& WithArn(const Aws::String& value) { SetArn(value); return *this;}

    /**
     *
The Amazon Resource Name (ARN) of the prefetch schedule.

     */
    inline GetPrefetchScheduleResult& WithArn(Aws::String&& value) { SetArn(std::move(value)); return *this;}

    /**
     *
The Amazon Resource Name (ARN) of the prefetch schedule.

     */
    inline GetPrefetchScheduleResult& WithArn(const char* value) { SetArn(value); return *this;}

    /**
     *
Consumption settings determine how, and when, MediaTailor places the
     * prefetched ads into ad breaks. Ad consumption occurs within a span of time that
     * you define, called a consumption window. You can designate which ad
     * breaks that MediaTailor fills with prefetch ads by setting avail matching
     * criteria.

     */
    inline const PrefetchConsumption& GetConsumption() const{ return m_consumption; }

    /**
     *
Consumption settings determine how, and when, MediaTailor places the
     * prefetched ads into ad breaks. Ad consumption occurs within a span of time that
     * you define, called a consumption window. You can designate which ad
     * breaks that MediaTailor fills with prefetch ads by setting avail matching
     * criteria.

     */
    inline void SetConsumption(const PrefetchConsumption& value) { m_consumption = value; }

    /**
     *
Consumption settings determine how, and when, MediaTailor places the
     * prefetched ads into ad breaks. Ad consumption occurs within a span of time that
     * you define, called a consumption window. You can designate which ad
     * breaks that MediaTailor fills with prefetch ads by setting avail matching
     * criteria.

     */
    inline void SetConsumption(PrefetchConsumption&& value) { m_consumption = std::move(value); }

    /**
     *
Consumption settings determine how, and when, MediaTailor places the
     * prefetched ads into ad breaks. Ad consumption occurs within a span of time that
     * you define, called a consumption window. You can designate which ad
     * breaks that MediaTailor fills with prefetch ads by setting avail matching
     * criteria.

     */
    inline GetPrefetchScheduleResult& WithConsumption(const PrefetchConsumption& value) { SetConsumption(value); return *this;}

    /**
     *
Consumption settings determine how, and when, MediaTailor places the
     * prefetched ads into ad breaks. Ad consumption occurs within a span of time that
     * you define, called a consumption window. You can designate which ad
     * breaks that MediaTailor fills with prefetch ads by setting avail matching
     * criteria.

     */
    inline GetPrefetchScheduleResult& WithConsumption(PrefetchConsumption&& value) { SetConsumption(std::move(value)); return *this;}

    /**
     *
The name of the prefetch schedule. The name must be unique among all prefetch
     * schedules that are associated with the specified playback configuration.

     */
    inline const Aws::String& GetName() const{ return m_name; }

    /**
     *
The name of the prefetch schedule. The name must be unique among all prefetch
     * schedules that are associated with the specified playback configuration.

     */
    inline void SetName(const Aws::String& value) { m_name = value; }

    /**
     *
The name of the prefetch schedule. The name must be unique among all prefetch
     * schedules that are associated with the specified playback configuration.

     */
    inline void SetName(Aws::String&& value) { m_name = std::move(value); }

    /**
     *
The name of the prefetch schedule. The name must be unique among all prefetch
     * schedules that are associated with the specified playback configuration.

     */
    inline void SetName(const char* value) { m_name.assign(value); }

    /**
     *
The name of the prefetch schedule. The name must be unique among all prefetch
     * schedules that are associated with the specified playback configuration.

     */
    inline GetPrefetchScheduleResult& WithName(const Aws::String& value) { SetName(value); return *this;}

    /**
     *
The name of the prefetch schedule. The name must be unique among all prefetch
     * schedules that are associated with the specified playback configuration.

     */
    inline GetPrefetchScheduleResult& WithName(Aws::String&& value) { SetName(std::move(value)); return *this;}

    /**
     *
The name of the prefetch schedule. The name must be unique among all prefetch
     * schedules that are associated with the specified playback configuration.

     */
    inline GetPrefetchScheduleResult& WithName(const char* value) { SetName(value); return *this;}

    /**
     *
The name of the playback configuration to create the prefetch schedule
     * for.

     */
    inline const Aws::String& GetPlaybackConfigurationName() const{ return m_playbackConfigurationName; }

    /**
     *
The name of the playback configuration to create the prefetch schedule
     * for.

     */
    inline void SetPlaybackConfigurationName(const Aws::String& value) { m_playbackConfigurationName = value; }

    /**
     *
The name of the playback configuration to create the prefetch schedule
     * for.

     */
    inline void SetPlaybackConfigurationName(Aws::String&& value) { m_playbackConfigurationName = std::move(value); }

    /**
     *
The name of the playback configuration to create the prefetch schedule
     * for.

     */
    inline void SetPlaybackConfigurationName(const char* value) { m_playbackConfigurationName.assign(value); }

    /**
     *
The name of the playback configuration to create the prefetch schedule
     * for.

     */
    inline GetPrefetchScheduleResult& WithPlaybackConfigurationName(const Aws::String& value) { SetPlaybackConfigurationName(value); return *this;}

    /**
     *
The name of the playback configuration to create the prefetch schedule
     * for.

     */
    inline GetPrefetchScheduleResult& WithPlaybackConfigurationName(Aws::String&& value) { SetPlaybackConfigurationName(std::move(value)); return *this;}

    /**
     *
The name of the playback configuration to create the prefetch schedule
     * for.

     */
    inline GetPrefetchScheduleResult& WithPlaybackConfigurationName(const char* value) { SetPlaybackConfigurationName(value); return *this;}

    /**
     *
A complex type that contains settings for prefetch retrieval from the ad
     * decision server (ADS).

     */
    inline const PrefetchRetrieval& GetRetrieval() const{ return m_retrieval; }

    /**
     *
A complex type that contains settings for prefetch retrieval from the ad
     * decision server (ADS).

     */
    inline void SetRetrieval(const PrefetchRetrieval& value) { m_retrieval = value; }

    /**
     *
A complex type that contains settings for prefetch retrieval from the ad
     * decision server (ADS).

     */
    inline void SetRetrieval(PrefetchRetrieval&& value) { m_retrieval = std::move(value); }

    /**
     *
A complex type that contains settings for prefetch retrieval from the ad
     * decision server (ADS).

     */
    inline GetPrefetchScheduleResult& WithRetrieval(const PrefetchRetrieval& value) { SetRetrieval(value); return *this;}

    /**
     *
A complex type that contains settings for prefetch retrieval from the ad
     * decision server (ADS).

     */
    inline GetPrefetchScheduleResult& WithRetrieval(PrefetchRetrieval&& value) { SetRetrieval(std::move(value)); return *this;}

    /**
     *
An optional stream identifier that you can specify in order to prefetch for
     * multiple streams that use the same playback configuration.

     */
    inline const Aws::String& GetStreamId() const{ return m_streamId; }

    /**
     *
An optional stream identifier that you can specify in order to prefetch for
     * multiple streams that use the same playback configuration.

     */
    inline void SetStreamId(const Aws::String& value) { m_streamId = value; }

    /**
     *
An optional stream identifier that you can specify in order to prefetch for
     * multiple streams that use the same playback configuration.

     */
    inline void SetStreamId(Aws::String&& value) { m_streamId = std::move(value); }

    /**
     *
An optional stream identifier that you can specify in order to prefetch for
     * multiple streams that use the same playback configuration.

     */
    inline void SetStreamId(const char* value) { m_streamId.assign(value); }

    /**
     *
An optional stream identifier that you can specify in order to prefetch for
     * multiple streams that use the same playback configuration.

     */
    inline GetPrefetchScheduleResult& WithStreamId(const Aws::String& value) { SetStreamId(value); return *this;}

    /**
     *
An optional stream identifier that you can specify in order to prefetch for
     * multiple streams that use the same playback configuration.

     */
    inline GetPrefetchScheduleResult& WithStreamId(Aws::String&& value) { SetStreamId(std::move(value)); return *this;}

    /**
     *
An optional stream identifier that you can specify in order to prefetch for
     * multiple streams that use the same playback configuration.

     */
    inline GetPrefetchScheduleResult& WithStreamId(const char* value) { SetStreamId(value); return *this;}

  private:

    Aws::String m_arn;

    PrefetchConsumption m_consumption;

    Aws::String m_name;

    Aws::String m_playbackConfigurationName;

    PrefetchRetrieval m_retrieval;

    Aws::String m_streamId;
  };

} // namespace Model
} // namespace MediaTailor
} // namespace Aws


text:
static char s_szLastLeaderNetworkID[ASW_LEADERID_LEN] = {0};

LINK_ENTITY_TO_CLASS( asw_game_resource, CASW_Game_Resource );

extern void SendProxy_String_tToString( const SendProp *pProp, const void *pStruct, const void *pData, DVariant *pOut, int iElement, int objectID );

IMPLEMENT_SERVERCLASS_ST(CASW_Game_Resource, DT_ASW_Game_Resource)
	//SendPropArray( SendPropEHandle( SENDINFO_ARRAY(m_MarineResources) ), m_MarineResources ),
	SendPropArray3( SENDINFO_ARRAY3(m_MarineResources), SendPropEHandle( SENDINFO_ARRAY(m_MarineResources) ) ),
	//SendPropArray( SendPropEHandle( SENDINFO_ARRAY(m_Objectives) ),	m_Objectives ),
	SendPropArray3( SENDINFO_ARRAY3(m_Objectives), SendPropEHandle( SENDINFO_ARRAY(m_Objectives) ) ),
	//SendPropArray( SendPropBool( SENDINFO_ARRAY(m_bRosterSelected) ), m_bRosterSelected	),
	SendPropArray3	( SENDINFO_ARRAY3(m_iRosterSelected), SendPropInt( SENDINFO_ARRAY(m_iRosterSelected) ) ),
	SendPropArray3	( SENDINFO_ARRAY3(m_bPlayerReady), SendPropBool( SENDINFO_ARRAY(m_bPlayerReady) ) ),

	SendPropEHandle (SENDINFO(m_Leader) ),
	SendPropInt(SENDINFO(m_iLeaderIndex), 8),
	SendPropEHandle (SENDINFO(m_hScannerInfo) ),
	SendPropInt(SENDINFO(m_iCampaignGame), 4),
	SendPropEHandle (SENDINFO(m_hCampaignSave) ),
	SendPropBool (SENDINFO(m_bOneMarineEach)),
	SendPropInt(SENDINFO(m_iMaxMarines)),
	SendPropBool (SENDINFO(m_bOfflineGame)),

	// marine skills
	SendPropArray3( SENDINFO_ARRAY3( m_iSkillSlot0 ), SendPropInt( SENDINFO_ARRAY( m_iSkillSlot0 ), ASW_NUM_MARINE_PROFILES, SPROP_UNSIGNED ) ),
	SendPropArray3( SENDINFO_ARRAY3( m_iSkillSlot1 ), SendPropInt( SENDINFO_ARRAY( m_iSkillSlot1 ), ASW_NUM_MARINE_PROFILES, SPROP_UNSIGNED ) ),
	SendPropArray3( SENDINFO_ARRAY3( m_iSkillSlot2 ), SendPropInt( SENDINFO_ARRAY( m_iSkillSlot2 ), ASW_NUM_MARINE_PROFILES, SPROP_UNSIGNED ) ),
	SendPropArray3( SENDINFO_ARRAY3( m_iSkillSlot3 ), SendPropInt( SENDINFO_ARRAY( m_iSkillSlot3 ), ASW_NUM_MARINE_PROFILES, SPROP_UNSIGNED ) ),
	SendPropArray3( SENDINFO_ARRAY3( m_iSkillSlot4 ), SendPropInt( SENDINFO_ARRAY( m_iSkillSlot4 ), ASW_NUM_MARINE_PROFILES, SPROP_UNSIGNED ) ),
	SendPropArray3( SENDINFO_ARRAY3( m_iSkillSlotSpare ), SendPropInt( SENDINFO_ARRAY( m_iSkillSlotSpare ), ASW_NUM_MARINE_PROFILES, SPROP_UNSIGNED ) ),

	// player specific medals
	SendPropArray( SendPropString( SENDINFO_ARRAY( m_iszPlayerMedals ), 0, SendProxy_String_tToString ), m_iszPlayerMedals ),

	SendPropArray3( SENDINFO_ARRAY3( m_iKickVotes ), SendPropInt( SENDINFO_ARRAY( m_iKickVotes ), ASW_MAX_READY_PLAYERS, SPROP_UNSIGNED ) ),
	SendPropArray3( SENDINFO_ARRAY3( m_iLeaderVotes ), SendPropInt( SENDINFO_ARRAY( m_iLeaderVotes ), ASW_MAX_READY_PLAYERS, SPROP_UNSIGNED ) ),

	SendPropInt(SENDINFO(m_iMoney)),
	SendPropInt(SENDINFO(m_iNextCampaignMission)),

	SendPropInt(SENDINFO(m_nDifficultySuggestion)),

	SendPropFloat( SENDINFO(m_fMapGenerationProgress) ),
	SendPropString( SENDINFO(m_szMapGenerationStatus) ),
	SendPropInt( SENDINFO(m_iRandomMapSeed) ),
END_SEND_TABLE()

//---------------------------------------------------------
// Save/Restore
//---------------------------------------------------------
BEGIN_DATADESC( CASW_Game_Resource )
	DEFINE_AUTO_ARRAY( m_MarineResources, FIELD_EHANDLE ),
	DEFINE_AUTO_ARRAY( m_Objectives, FIELD_EHANDLE ),
	DEFINE_AUTO_ARRAY( m_iRosterSelected, FIELD_INTEGER ),
	DEFINE_AUTO_ARRAY( m_bPlayerReady, FIELD_BOOLEAN ),
	DEFINE_AUTO_ARRAY( m_iCampaignVote, FIELD_INTEGER ),
	DEFINE_FIELD( m_Leader, FIELD_EHANDLE ),
	DEFINE_FIELD( m_iLeaderIndex, FIELD_INTEGER ),
	DEFINE_FIELD( m_hScannerInfo, FIELD_EHANDLE ),
	DEFINE_FIELD( m_NumObjectives, FIELD_INTEGER ),
	DEFINE_FIELD( m_iCampaignGame, FIELD_INTEGER ),
	DEFINE_FIELD( m_hCampaignSave, FIELD_EHANDLE ),
	DEFINE_FIELD( m_iNumMarinesSelected, FIELD_INTEGER ),

	DEFINE_FIELD( m_iszPlayerMedals[0], FIELD_STRING ),
	DEFINE_FIELD( m_iszPlayerMedals[1], FIELD_STRING ),
	DEFINE_FIELD( m_iszPlayerMedals[2], FIELD_STRING ),
	DEFINE_FIELD( m_iszPlayerMedals[3], FIELD_STRING ),
	DEFINE_FIELD( m_iszPlayerMedals[4], FIELD_STRING ),
	DEFINE_FIELD( m_iszPlayerMedals[5], FIELD_STRING ),
	DEFINE_FIELD( m_iszPlayerMedals[6], FIELD_STRING ),
	DEFINE_FIELD( m_iszPlayerMedals[7], FIELD_STRING ),
END_DATADESC()

CASW_Game_Resource *g_pASWGameResource = NULL;

extern bool g_bOfflineGame;

int CASW_Game_Resource::s_nNumConsecutiveFailures = 0;
bool CASW_Game_Resource::s_bLeaderGivenDifficultySuggestion = false;

CASW_Game_Resource::CASW_Game_Resource()
{
	g_pASWGameResource = this;
	m_iNumEnumeratedMarines = NULL;
	m_pCampaignInfo = NULL;
	m_iMaxMarines = ASW_MAX_MARINE_RESOURCES;
	m_bOneMarineEach = false;
	m_iNumMarinesSelected = 0;
	m_iRandomMapSeed = 0;
	m_iNextCampaignMission = -1;
	m_iEggsHatched = 0;
	m_iEggsKilled = 0;
	m_iStartingEggsInMap = -1;
	m_bAwardedDamageAmpAchievement = false;
	m_iAliensKilledWithDamageAmp = 0;
	m_iElectroStunnedAliens = 0;
	for (int i=0;i<ASW_MAX_READY_PLAYERS;i++)
	{
		m_bPlayerReady.Set(i, false);
		m_iKickVotes.Set(i, 0);
		m_iLeaderVotes.Set(i, 0);
		m_iCampaignVote[i] = -1;
	}
	for (int i=0;i<ASW_MAX_MARINE_RESOURCES;i++)
	{
		m_MarineResources.Set(i, NULL);
	}
	for (int i=0;i<ASW_MAX_OBJECTIVES;i++)
	{
		m_Objectives.Set(i, NULL);
	}
	CASW_Scanner_Info* pScanner = (CASW_Scanner_Info*)CreateEntityByName("asw_scanner_info");
	if (pScanner)
	{
		pScanner->Spawn();
		m_hScannerInfo = pScanner;
	}

	m_bOfflineGame = g_bOfflineGame;
	m_iCampaignGame = 0;
	m_szCampaignSaveName[0] = '\0';
	// query map's launch options to see if we're in campaign mode
	KeyValues *pLaunchOptions = engine->GetLaunchOptions();
	if (pLaunchOptions)
	{
		KeyValues *pKey = pLaunchOptions->GetFirstSubKey();
		while ( pKey )
		{
			if ( !Q_stricmp( pKey->GetString(), "campaign" ) )
			{
				m_iCampaignGame = 1;
				KeyValues *pCampaignSaveName = pKey->GetNextKey();
				if (pCampaignSaveName)
					Q_snprintf(m_szCampaignSaveName, sizeof(m_szCampaignSaveName), "%s", pCampaignSaveName->GetString());
				break;
			}
			pKey = pKey->GetNextKey();
		}
	}

	// make skills default for single mission
	UpdateMarineSkills(NULL);
}

CASW_Game_Resource::~CASW_Game_Resource()
{
	if (m_hScannerInfo.Get())
		delete m_hScannerInfo.Get();

	if ( g_pASWGameResource == this )
	{
		g_pASWGameResource = NULL;
	}
}

// always send this info to players
int CASW_Game_Resource::ShouldTransmit( const CCheckTransmitInfo *pInfo )
{
	return FL_EDICT_ALWAYS;
}

CASW_Objective* CASW_Game_Resource::GetObjective(int i)
{
	if (i<0 || i>=ASW_MAX_OBJECTIVES)
		return NULL;

	if (m_Objectives[i] == NULL)
		return NULL;

	CBaseEntity* c = m_Objectives[i].Get();
	return static_cast<CASW_Objective*>(c);
}

void CASW_Game_Resource::FindObjectivesOfClass(const char *szClass)
{
	CBaseEntity* pEntity = NULL;
	while ((pEntity = gEntList.FindEntityByClassname( pEntity, szClass )) != NULL)
	{
		//CASW_Objective* pObjective = dynamic_cast<CASW_Objective*>(pEntity);
		m_Objectives.Set(m_NumObjectives, pEntity); //pObjective;
		m_NumObjectives++;
	}
}

void CASW_Game_Resource::FindObjectives()
{
	// search through all entities and populate the objectives array
	m_NumObjectives = 0;

	FindObjectivesOfClass("asw_objective_dummy");
	FindObjectivesOfClass("asw_objective_kill_eggs");
	FindObjectivesOfClass("asw_objective_destroy_goo");
	FindObjectivesOfClass("asw_objective_triggered");
	FindObjectivesOfClass("asw_objective_escape");
	FindObjectivesOfClass("asw_objective_survive");
	FindObjectivesOfClass("asw_objective_countdown");
	FindObjectivesOfClass("asw_objective_kill_aliens");

	// bubble sort objectives by their Y coord
	CASW_Objective* pObjective;
	//for (int k=0;k<m_NumObjectives-1;k++)
	//{
		//Msg("Objective %d has coord %f and title %s\n", k, GetObjective(k)->GetAbsOrigin().y,
				//GetObjective(k)->m_ObjectiveTitle);
	//}
	for (int i=0;i<m_NumObjectives;i++)
	{
		for (int j=i+1; j<m_NumObjectives; j++)
		{
			if (GetObjective(j)->GetAbsOrigin().y > GetObjective(i)->GetAbsOrigin().y)
			{
				//Msg("Swapping %d (%f) with %d (%f)\n", i, GetObjective(i)->GetAbsOrigin().y,
					//j, GetObjective(j)->GetAbsOrigin().y);
				pObjective = GetObjective(j);
				m_Objectives.Set(j, GetObjective(i));
				m_Objectives.Set(i, pObjective);
			}
		}
	}
}

bool CASW_Game_Resource::IsRosterSelected(int i)
{
	if (i<0 || i>=ASW_NUM_MARINE_PROFILES)
		return false;

	return m_iRosterSelected[i] == 1;
}

bool CASW_Game_Resource::IsRosterReserved(int i)
{
	if (i<0 || i>=ASW_NUM_MARINE_PROFILES)
		return false;

	return m_iRosterSelected[i] == 2;
}

void CASW_Game_Resource::SetRosterSelected(int i, int iSelected)
{
	if (i<0 || i>=ASW_NUM_MARINE_PROFILES)
		return;

	if (iSelected != m_iRosterSelected[i])
	{
		// update num selected
		if (iSelected == 1 && m_iRosterSelected[i] != 1)
		{
			m_iNumMarinesSelected++;
		}
		else if (iSelected != 1 && m_iRosterSelected[i] == 1)
		{
			m_iNumMarinesSelected--;
		}
		m_iRosterSelected.Set(i, iSelected);
	}
}

// unselects a single marine
// tries to pick the player with the most marines selected
void CASW_Game_Resource::RemoveAMarine()
{
	int iHighest = 0;
	CASW_Player *pChosen = NULL;

	for ( int i = 1; i <= gpGlobals->maxClients; i++ )
	{
		CASW_Player* pOtherPlayer = dynamic_cast<CASW_Player*>(UTIL_PlayerByIndex(i));

		if ( pOtherPlayer && pOtherPlayer->IsConnected() )
		{
			int iMarines = GetNumMarines(pOtherPlayer);
			if (iHighest == 0 || iMarines > iHighest)
			{
				iHighest = iMarines;
				pChosen = pOtherPlayer;
			}
		}
	}
	RemoveAMarineFor(pChosen);
}

void CASW_Game_Resource::RemoveAMarineFor(CASW_Player *pPlayer)
{
	if (!pPlayer || !ASWGameRules())
		return;

	// find this player's first marine and remove it
	int m = GetMaxMarineResources();
	for (int i=0;i<m;i++)
	{
		if (GetMarineResource(i) && GetMarineResource(i)->GetCommander() == pPlayer)
		{
			ASWGameRules()->RosterDeselect(pPlayer, GetMarineResource(i)->GetProfileIndex());
			return;
		}
	}
}

bool CASW_Game_Resource::AddMarineResource( CASW_Marine_Resource *m, int nPreferredSlot )
{
	if ( nPreferredSlot != -1 )
	{
		CASW_Marine_Resource *pExisting = static_cast<CASW_Marine_Resource*>( m_MarineResources[ nPreferredSlot ].Get() );
		if ( pExisting != NULL )
		{
			// if the existing is owned by someone else, then we abort
			if ( pExisting->GetCommander() != m->GetCommander() )
				return false;

			SetRosterSelected( pExisting->GetProfileIndex(), 0 );
			UTIL_Remove( pExisting );
		}

		m_MarineResources.Set( nPreferredSlot, m );

		// the above causes strange cases where the client copy of this networked array is incorrect
		// so we flag each element dirty to cause a complete update, which seems to fix the problem
		for (int k=0;k<ASW_MAX_MARINE_RESOURCES;k++)
		{
			m_MarineResources.GetForModify(k);
		}
		return true;
	}

	for (int i=0;i<ASW_MAX_MARINE_RESOURCES;i++)
	{
		if (m_MarineResources[i] == NULL)	// found a free slot
		{
			m_MarineResources.Set(i, m);

			// the above causes strange cases where the client copy of this networked array is incorrect
			// so we flag each element dirty to cause a complete update, which seems to fix the problem
			for (int k=0;k<ASW_MAX_MARINE_RESOURCES;k++)
			{
				m_MarineResources.GetForModify(k);
			}
			return true;
		}
	}
	Msg("Couldn't add new marine resource to list as no free slots\n");
	return false;
}

void CASW_Game_Resource::DeleteMarineResource(CASW_Marine_Resource *m)
{
	for (int i=0;i<ASW_MAX_MARINE_RESOURCES;i++)
	{
		if (GetMarineResource(i) == m)
		{
			m_MarineResources.Set(i, NULL);
			UTIL_Remove(m);
			// need to shuffle all others up
			for (int k=i;k<(ASW_MAX_MARINE_RESOURCES-1);k++)
			{
				CASW_Marine_Resource *other = GetMarineResource(k+1);
				m_MarineResources.Set(k, other);
				m_MarineResources.Set(k+1, NULL);
			}
		}
	}
	// the above causes strange cases where the client copy of this networked array is incorrect
	// so we flag each element dirty to cause a complete update, which seems to fix the problem
	for (int i=0;i<ASW_MAX_MARINE_RESOURCES;i++)
	{
		m_MarineResources.GetForModify(i);
	}
}

CASW_Marine_Resource* CASW_Game_Resource::GetMarineResource(int i)
{
	if (i<0 || i>=ASW_MAX_MARINE_RESOURCES || m_MarineResources[i] == NULL)
		return NULL;

	CBaseEntity* c = m_MarineResources[i].Get();
	return static_cast<CASW_Marine_Resource*>(c);
}

CASW_Scanner_Info* CASW_Game_Resource::GetScannerInfo()
{
	CASW_Scanner_Info* pScanner = dynamic_cast<CASW_Scanner_Info*>(m_hScannerInfo.Get());
	return pScanner;
}

void CASW_Game_Resource::SetLeader(CASW_Player *pPlayer)
{
	// check for auto-readying our old leader
	if (m_Leader.Get() && m_Leader.Get() != pPlayer && ASWGameRules() && m_Leader->m_bRequestedSpectator)
	{
		int iPlayerIndex = m_Leader->entindex() - 1;
		if (ASWGameRules()->GetGameState() == ASW_GS_BRIEFING || ASWGameRules()->GetGameState()==ASW_GS_DEBRIEF)
		{
			// player index is out of range
			if (iPlayerIndex >= 0 && iPlayerIndex < ASW_MAX_READY_PLAYERS)
			{
				Msg("Autoreadying old leader, who wanted to be a spectator\n");
				m_bPlayerReady.Set(iPlayerIndex, true);
			}
		}
		else if (ASWGameRules()->GetGameState() == ASW_GS_CAMPAIGNMAP && ASWGameRules()->GetCampaignSave())
		{
			Msg(" old leader telling campaign save that we're spectating\n", iPlayerIndex);
			ASWGameRules()->GetCampaignSave()->PlayerSpectating(m_Leader.Get());
		}
	}
	if (!pPlayer)
	{
		m_Leader = NULL;
		m_iLeaderIndex = -1;
		return;
	}

	m_Leader = pPlayer;
	m_iLeaderIndex = pPlayer->entindex();

	if (ASWGameRules()->GetGameState() == ASW_GS_BRIEFING || ASWGameRules()->GetGameState()==ASW_GS_DEBRIEF)
	{
		// unready leader
		int iPlayerIndex = m_iLeaderIndex - 1;

		// player index is out of range
		if (iPlayerIndex >= 0 && iPlayerIndex < ASW_MAX_READY_PLAYERS)
		{
			m_bPlayerReady.Set(iPlayerIndex, false);
		}
	}
}

CASW_Campaign_Save* CASW_Game_Resource::GetCampaignSave()
{
	CASW_Campaign_Save* pSave = dynamic_cast<CASW_Campaign_Save*>(m_hCampaignSave.Get());
	return pSave;
}

CASW_Campaign_Save* CASW_Game_Resource::CreateCampaignSave()
{
	CASW_Campaign_Save* pSave = (CASW_Campaign_Save*)CreateEntityByName("asw_campaign_save");
	if (pSave)
	{
		pSave->Spawn();
		m_hCampaignSave = pSave;
	}
	m_hCampaignSave = pSave;
	pSave = GetCampaignSave();
	return pSave;
}

void CASW_Game_Resource::UpdateMarineSkills( CASW_Campaign_Save *pCampaign )
{
	if ( !pCampaign || pCampaign->UsingFixedSkillPoints() )
	{
		// update skills with defaults
		for (int iProfileIndex=0;iProfileIndex<ASW_NUM_MARINE_PROFILES;iProfileIndex++)
		{
			CASW_Marine_Profile *pProfile = MarineProfileList()->m_Profiles[iProfileIndex];
			if (pProfile)
			{
				m_iSkillSlot0.Set( iProfileIndex, pProfile->m_iStaticSkills[ ASW_SKILL_SLOT_0 ] );
				m_iSkillSlot1.Set( iProfileIndex, pProfile->m_iStaticSkills[ ASW_SKILL_SLOT_1 ] );
				m_iSkillSlot2.Set( iProfileIndex, pProfile->m_iStaticSkills[ ASW_SKILL_SLOT_2 ] );
				m_iSkillSlot3.Set( iProfileIndex, pProfile->m_iStaticSkills[ ASW_SKILL_SLOT_3 ] );
				m_iSkillSlot4.Set( iProfileIndex, pProfile->m_iStaticSkills[ ASW_SKILL_SLOT_4 ] );
				m_iSkillSlotSpare.Set( iProfileIndex, pProfile->m_iStaticSkills[ ASW_SKILL_SLOT_SPARE ] );
			}
			else
			{
				m_iSkillSlot0.Set( iProfileIndex, 0 );
				m_iSkillSlot1.Set( iProfileIndex, 0 );
				m_iSkillSlot2.Set( iProfileIndex, 0 );
				m_iSkillSlot3.Set( iProfileIndex, 0 );
				m_iSkillSlot4.Set( iProfileIndex, 0 );
				m_iSkillSlotSpare.Set( iProfileIndex, 0 );
			}
		}
	}
	else
	{
		// get skills from the campaign save
		for (int iProfileIndex=0;iProfileIndex<ASW_NUM_MARINE_PROFILES;iProfileIndex++)
		{
			m_iSkillSlot0.Set( iProfileIndex, pCampaign->m_iMarineSkill[iProfileIndex][ASW_SKILL_SLOT_0] );
			m_iSkillSlot1.Set( iProfileIndex, pCampaign->m_iMarineSkill[iProfileIndex][ASW_SKILL_SLOT_1] );
			m_iSkillSlot2.Set( iProfileIndex, pCampaign->m_iMarineSkill[iProfileIndex][ASW_SKILL_SLOT_2] );
			m_iSkillSlot3.Set( iProfileIndex, pCampaign->m_iMarineSkill[iProfileIndex][ASW_SKILL_SLOT_3] );
			m_iSkillSlot4.Set( iProfileIndex, pCampaign->m_iMarineSkill[iProfileIndex][ASW_SKILL_SLOT_4] );
			m_iSkillSlotSpare.Set( iProfileIndex, pCampaign->m_iMarineSkill[iProfileIndex][ASW_SKILL_SLOT_SPARE] );
		}
	}
}

bool CASW_Game_Resource::SetMarineSkill( int nProfileIndex, int nSkillSlot, int nValue )
{
	if ( nProfileIndex < 0 || nProfileIndex > ASW_NUM_MARINE_PROFILES )
		return false;
	if ( nValue < 0 || nValue > 5 )
		return false;

	switch ( nSkillSlot )
	{
		case ASW_SKILL_SLOT_0: m_iSkillSlot0.Set( nProfileIndex, nValue ); break;
		case ASW_SKILL_SLOT_1: m_iSkillSlot1.Set( nProfileIndex, nValue ); break;
		case ASW_SKILL_SLOT_2: m_iSkillSlot2.Set( nProfileIndex, nValue ); break;
		case ASW_SKILL_SLOT_3: m_iSkillSlot3.Set( nProfileIndex, nValue ); break;
		case ASW_SKILL_SLOT_4: m_iSkillSlot4.Set( nProfileIndex, nValue ); break;
		case ASW_SKILL_SLOT_SPARE: m_iSkillSlotSpare.Set( nProfileIndex, nValue ); break;
		default: return false; break;
	}

	return true;
}

CASW_Player* CASW_Game_Resource::GetLeader()
{
	return dynamic_cast<CASW_Player*>(m_Leader.Get());
}

CASW_Marine* CASW_Game_Resource::FindMarineByVoiceType( ASW_Voice_Type voice )
{
	int m = GetMaxMarineResources();
	for (int i=0;i<m;i++)
	{
		if (GetMarineResource(i) && GetMarineResource(i)->GetProfile() && GetMarineResource(i)->GetProfile()->m_VoiceType == voice)
		{
			return GetMarineResource(i)->GetMarineEntity();
		}
	}
	return NULL;
}

void CASW_Game_Resource::RememberLeaderID()
{
	if (GetLeader())
	{
		SetLastLeaderNetworkID(GetLeader()->GetASWNetworkID());
	}
}

const char* CASW_Game_Resource::GetLastLeaderNetworkID()
{
	return s_szLastLeaderNetworkID;
}

void CASW_Game_Resource::SetLastLeaderNetworkID(const char* szID)
{
	Q_snprintf(s_szLastLeaderNetworkID, sizeof(s_szLastLeaderNetworkID), "%s", szID);
}

int CASW_Game_Resource::GetAliensKilledInThisMission()
{
	int nCount = 0;
	int m = GetMaxMarineResources();
	for ( int i = 0; i < m; i++ )
	{
		CASW_Marine_Resource *pMR = GetMarineResource( i );
		if ( !pMR )
			continue;

		nCount += pMR->m_iAliensKilled.Get();
	}
	return nCount;
}

void CASW_Game_Resource::OnMissionFailed( void )
{
	s_nNumConsecutiveFailures++;

	if ( !s_bLeaderGivenDifficultySuggestion && s_nNumConsecutiveFailures >= 4 )
	{
		s_bLeaderGivenDifficultySuggestion = true;
		m_nDifficultySuggestion = 2;
	}
}

void CASW_Game_Resource::OnMissionCompleted( bool bWellDone )
{
	if ( !s_bLeaderGivenDifficultySuggestion && s_nNumConsecutiveFailures == 0 && bWellDone )
	{
		s_bLeaderGivenDifficultySuggestion = true;
		m_nDifficultySuggestion = 1;
	}
	else
	{
		m_nDifficultySuggestion = 0;
	}

	s_nNumConsecutiveFailures = 0;
}