text:
(function () {

    "use strict";

//詳細画面保存時の処理
function submit_detail(event) {
	var record = event.record;
	var member=[];

	var members = record["members"]["value"]; //申し込み者テーブル

	if( !members ) return;//空欄ならなにもしない

	var appId = 94;//イベント名簿アプリのID
	var eventno = record["eventno"]["value"];
	var eventname = record["eventname"]["value"];

	for(var i=0;i < members.length; ++i ) {
			member = members[i]["value"];
			var query ="eventno="+eventno+" and "+"meibocd="+'"'+member["meibocd"]["value"]+'"';
            var appUrl = kintone.api.url('/k/v1/records') + '?app='+ appId  +  "&query="+encodeURI(query+'order by eventno desc limit 1');
            var xmlHttp = new XMLHttpRequest();
            var regno = 0;

			var objParam = {};
			objParam['app'] = appId;// アプリ番号
			objParam['record'] = {};
			objParam['record']['eventno'] = {}; // status
			objParam['record']['eventno']['value'] = eventno;
			objParam['record']['eventname'] = {}; // status
			objParam['record']['eventname']['value'] = eventname;
			objParam['record']['meibocd'] = {}; // status
			objParam['record']['meibocd']['value'] = member["meibocd"]["value"];
			objParam['record']['name'] = {}; // status
			objParam['record']['name']['value'] = member["name"]["value"];
			objParam['record']['kana'] = {}; // status
			objParam['record']['kana']['value'] = member["kana"]["value"];
			objParam['record']['tel'] = {}; // status
			objParam['record']['tel']['value'] = member["tel"]["value"];
			objParam['record']['company'] = {}; // status
			objParam['record']['company']['value'] = member["company"]["value"];
			objParam['record']['company_yomi'] = {}; // status
			objParam['record']['company_yomi']['value'] = member["company_yomi"]["value"];
			objParam['record']['dairi'] = {}; // status
			objParam['record']['dairi']['value'] = member["dairi"]["value"];
			objParam['record']['others'] = {}; // status
			objParam['record']['others']['value'] = member["others"]["value"];

            // 同期リクエストを行う
            xmlHttp.open("GET", appUrl, false);
            xmlHttp.setRequestHeader('X-Requested-With','XMLHttpRequest');
            xmlHttp.send(null);

			var method = "POST";
            if (xmlHttp.status == 200 ){
                    if(window.JSON){
                   		 var obj = JSON.parse(xmlHttp.responseText);
				   		 if (obj.records[0] != null) { //レコードがあった場合は更新
								objParam['id'] = obj.records[0]["レコード番号"]["value"];
				   		 	    method = "PUT";
					   			}
					   		}
				  }

				// レコードがない場合は追加
				kintone.api('/k/v1/record', method, objParam, function(resp){
                // 成功時は画面をリロードします
                //location.reload(true);
                    }, function(resp) {
                        // エラー時はメッセージを表示して、処理を中断します
                        //alert('error->' + resp.message);

                    });

                }

	}

//レコード表示の場合
	    //保存前の処理
	    kintone.events.on('app.record.edit.submit', submit_detail );
        })();


text:
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<!-- 用户信息持久层继承IBaseDao -->
<mapper namespace="com.mingsoft.people.dao.IPeopleUserDao">
	<!-- 表字段开始-->
	<sql id="column_list">
	 	PU.PU_PEOPLE_ID,PU.PU_REAL_NAME,PU.PU_ADDRESS,PU.PU_ICON,PU.PU_CARD,
	 	PU.PU_NICKNAME,PU.PU_SEX,PU.PU_BIRTHDAY,PU.PU_APP_ID,PU.PU_PROVINCE,PU.PU_CITY,PU.PU_DISTRICT,PU.PU_STREET,PU.PU_COUNTRY,PU.PU_EDUCATION,PU.PU_JOB,PU.PU_RISK
	</sql>

	<sql id="column_people_list">
	 	PU.PU_PEOPLE_ID,PU.PU_REAL_NAME,PU.PU_ADDRESS,PU.PU_ICON,PU.PU_NICKNAME,PU.PU_SEX,PU.PU_BIRTHDAY,PU.PU_APP_ID,PU.PU_CARD,PU.PU_PROVINCE,PU.PU_CITY,PU.PU_DISTRICT,PU.PU_STREET,PU.PU_COUNTRY,PU.PU_EDUCATION,PU.PU_JOB,PU.PU_RISK,
		P.PEOPLE_ID,P.PEOPLE_STATE,P.PEOPLE_PHONE,P.PEOPLE_NAME,P.PEOPLE_PASSWORD,P.PEOPLE_DATETIME,P.PEOPLE_APP_ID,P.PEOPLE_MAIL,P.PEOPLE_PHONECHECK,P.PEOPLE_MAILLCHECK
	</sql>
	<!-- 表字段结束 -->

	<!-- 将实体属性与表字段对接开始 -->
	<resultMap  id="resultMap" type="com.mingsoft.people.entity.PeopleUserEntity">
		<id column="PU_PEOPLE_ID" property="peopleUserPeopleId"/><!-- 将实体的ID属性与表的ID字段对接 -->
		<result column="PU_REAL_NAME" property="peopleUserRealName"/><!--用户的真实名称 -->
		<result column="PU_ADDRESS" property="peopleUserAddress"/><!-- 用户地址 -->
		<result column="PU_ICON" property="peopleUserIcon"/><!-- 用户头像 -->
		<result column="PU_NICKNAME" property="peopleUserNickName"/><!-- 用户昵称 -->
		<result column="PU_SEX" property="peopleUserSex"/><!-- 用户性别0.未知;1.男;2.女 -->
		<result column="PU_BIRTHDAY" property="peopleUserBirthday"/><!-- 用户生日 -->
		<result column="PU_APP_ID" property="peopleUserAppId"/><!-- 用户应用ID -->
		<result column="PU_CARD" property="peopleUserCard"/><!-- 用户身份证号码 -->
		<result column="PU_PROVINCE" property="peopleUserProvince"/><!-- 省 -->
		<result column="PU_CITY" property="peopleUserCity"/><!-- 市 -->
		<result column="PU_DISTRICT" property="peopleUserDistrict"/><!-- 区县 -->
		<result column="PU_STREET" property="peopleUserStreet"/><!-- 街道&镇 -->
		<!--加入新字段-->
		<result column="PU_COUNTRY" property="peopleUserCountry"/><!-- 目前居住 -->
		<result column="PU_EDUCATION" property="peopleUserEducation"/><!-- 学历 -->
		<result column="PU_JOB" property="peopleUserJob"/><!-- 工作 -->
		<result column="PU_RISK" property="peopleRisk"/><!-- 工作 -->

		<result column="PEOPLE_ID" property="peopleId"/><!-- 将实体的ID属性与表的ID字段对接 -->
		<result column="PEOPLE_PHONE" property="peoplePhone"/><!-- 将实体的peoplePhone属性与表的peoplePhone字段对接 -->
		<result column="PEOPLE_NAME" property="peopleName"/><!-- 将实体的peopleName属性与表的peopleName字段对接 -->
		<result column="PEOPLE_PASSWORD" property="peoplePassword"/><!-- 将实体的peoplePwd属性与表的peoplePwd字段对接 -->
		<result column="PEOPLE_DATETIME" property="peopleDateTime"/><!-- 将实体的peopleDateTime属性与表的peopleDateTime字段对接 -->
		<result column="PEOPLE_APP_ID" property="peopleAppId"/><!-- 用户所属的应用ID -->
		<result column="PEOPLE_MAIL" property="peopleMail"/><!-- 用户邮箱 -->
		<result column="PEOPLE_MAILLCHECK" property="peopleMailCheck"/>
		<result column="PEOPLE_PHONECHECK" property="peoplePhoneCheck"/>
	</resultMap>
	<!-- 将实体属性与表字段对接结束 -->

	<!-- 新增用户信息开始 -->
	<insert id="saveEntity" useGeneratedKeys="true" keyProperty="peopleId" parameterType="com.mingsoft.base.entity.BaseEntity">
		insert into people_user
		<trim prefix="(" suffix=")" suffixOverrides=",">
			<if test="peopleId != null">PU_PEOPLE_ID,</if>
			<if test="peopleUserRealName != null">PU_REAL_NAME,</if>
			<if test="peopleUserAddress != null">PU_ADDRESS,</if>
			<if test="peopleUserIcon != null">PU_ICON,</if>
			<if test="peopleUserNickName != null">PU_NICKNAME,</if>
			<if test="peopleUserSex != null">PU_SEX,</if>
			<if test="peopleUserBirthday != null">PU_BIRTHDAY,</if>
			<if test="peopleUserAppId != null">PU_APP_ID,</if>
			<if test="peopleUserCard != null">PU_CARD,</if>
			<if test="peopleUserProvince != null">PU_PROVINCE,</if>
			<if test="peopleUserCity != null">PU_CITY,</if>
			<if test="peopleUserDistrict != null">PU_DISTRICT,</if>
			<if test="peopleUserStreet != null">PU_STREET,</if>
			<if test="peopleUserCountry != null">PU_COUNTRY,</if>
			<if test="peopleUserEducation != null">PU_EDUCATION,</if>
			<if test="peopleUserJob != null">PU_JOB,</if>
			<if test="peopleRisk != null">PU_RISK,</if>
		</trim>
		<trim prefix="values (" suffix=")" suffixOverrides=",">
			<if test="peopleId != null">#{peopleId},</if>
			<if test="peopleUserRealName != null">#{peopleUserRealName},</if>
			<if test="peopleUserAddress != null">#{peopleUserAddress},</if>
			<if test="peopleUserIcon != null">#{peopleUserIcon},</if>
			<if test="peopleUserNickName != null">#{peopleUserNickName},</if>
			<if test="peopleUserSex != null">#{peopleUserSex},</if>
			<if test="peopleUserBirthday != null">#{peopleUserBirthday},</if>
			<if test="peopleUserAppId != null">#{peopleUserAppId},</if>
			<if test="peopleUserCard != null">#{peopleUserCard},</if>
			<if test="peopleUserProvince != null">#{peopleUserProvince},</if>
			<if test="peopleUserCity != null">#{peopleUserCity},</if>
			<if test="peopleUserDistrict != null">#{peopleUserDistrict},</if>
			<if test="peopleUserStreet != null">#{peopleUserStreet},</if>
			<if test="peopleUserCountry != null">#{peopleUserCountry},</if>
			<if test="peopleUserEducation != null">#{peopleUserEducation},</if>
			<if test="peopleUserJob != null">#{peopleUserJob},</if>
			<if test="peopleRisk != null">#{peopleRisk},</if>
		</trim>
	</insert>
	<!-- 新增用户信息结束 -->

	<!-- 更新用户开始 -->
	<update id="updateEntity" parameterType="com.mingsoft.base.entity.BaseEntity">
		update people_user
		<set>
			<if test="peopleUserRealName != null">PU_REAL_NAME=#{peopleUserRealName},</if>
			<if test="peopleUserAddress != null">PU_ADDRESS=#{peopleUserAddress},</if>
			<if test="peopleUserIcon != null">PU_ICON=#{peopleUserIcon},</if>
			<if test="peopleUserNickName != null">PU_NICKNAME=#{peopleUserNickName},</if>
			<if test="peopleUserSex != null">PU_SEX=#{peopleUserSex},</if>
			<if test="peopleUserBirthday != null">PU_BIRTHDAY=#{peopleUserBirthday},</if>
			<if test="peopleUserCard != null">PU_CARD=#{peopleUserCard},</if>
			<if test="peopleUserProvince != null">PU_PROVINCE=#{peopleUserProvince},</if>
			<if test="peopleUserCity != null">PU_CITY=#{peopleUserCity},</if>
			<if test="peopleUserDistrict != null">PU_DISTRICT=#{peopleUserDistrict},</if>
			<if test="peopleUserStreet != null">PU_STREET=#{peopleUserStreet},</if>
			<if test="peopleUserCountry != null">PU_COUNTRY=#{peopleUserCountry},</if>
			<if test="peopleUserEducation != null">PU_EDUCATION=#{peopleUserEducation},</if>
			<if test="peopleUserJob != null">PU_JOB=#{peopleUserJob},</if>
			<if test="peopleRisk != null">PU_RISK=#{peopleRisk},</if>
		</set>
		where PU_PEOPLE_ID = #{peopleId}
	</update>
	<!-- 更新用户结束 -->

	<!-- 根据Id查询实体开始 -->
	<select id="getEntity" resultMap="resultMap" parameterType="int">
		select <include refid="column_people_list"/>
		from  people p
		left join people_user pu on p.people_id=pu.pu_people_id
		where
		p.people_id = #{peopleId}
	</select>
	<!-- 根据Id查询实体结束 -->

	<!-- 删除用户开始 -->
	<delete id="deleteEntity" parameterType="int">
		delete from people_user where pu_people_id = #{id}
	</delete>
	<!-- 删除用户结束 -->

	<!-- 批量删除用户开始 -->
	<delete id="deletePeopleUsers" parameterType="int">
		delete from people_user
		<where>
			pu_people_id  in <foreach collection="peopleIds" item="item" index="index"
			open="(" separator="," close=")">#{item}</foreach>
		</where>
	</delete>
	<!-- 批量删除用户结束 -->
</mapper>

redpajama_set_name: RedPajamaGithub


text:
class CreateNizbelReleaseRegexes < ActiveRecord::Migration
  def change
    create_table :nizbel_release_regexes do |t|
      t.text :regex, :null => false, :limit => false
      t.boolean :active, :null => false, :default => true
      t.integer :options, :null => false, :default => 0
      t.string :description
      t.integer :category_id

      t.timestamps
    end

    add_index :nizbel_release_regexes, :regex, :unique => true
  end
end


text:
#include "third_party/tensorflow/core/framework/common_shape_fns.h"
#include "third_party/tensorflow/core/framework/op.h"
#include "third_party/tensorflow/core/framework/shape_inference.h"
#include "x_ops_helper.h"

namespace tensorflow {
namespace {

REGISTER_OP("AssertShapeMatch")
    .Input("x: int32")
    .Input("y: int32")
    .Attr("msg: string = ''")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      return Status::OK();
    })
    .Doc(R"doc(
Asserts that shape vector x and y matches.

The i-th dimension matches iff x[i] == y[i] || x[i] == -1 || y[i] == -1.

x: A shape vector.
y: A shape vector.
msg: The error message generated when the assertion failed.
)doc");

REGISTER_OP("AssertSameDim0")
    .Input("x: types")
    .Attr("msg: string = ''")
    .Attr("types: list(type)")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      return Status::OK();
    })
    .Doc(R"doc(
Asserts that all input tensors are non-scalar and have the same 0-th dim size.

x: The list of tensors.
msg: The error message generated when the assertion failed.
)doc");

REGISTER_OP("RandomPermutationSequence")
    .Attr("num: int = 1")
    .Attr("batch: int = 1")
    .Attr("repeat: bool = false")
    .Attr("seed: int = 0")
    .Output("out: int32")
    .SetIsStateful()
    .Doc(R"doc(
Generate random samples from [0..num-1] without replacements.

num: The number of ids.
batch: Each output is a vector of size up to batch. Right now,
    the last batch from one epoch is the only one that can be
    smaller than a full batch.
repeat: If true, this op keep generating random samples after one
    epoch. If false, this op errors with `tf.errors.OutOfRangeError` when an
    epoch finishes.
seed: The random seed.
out: Each output is a vector of size up to batch.
)doc");

REGISTER_OP("BestStep")
    .Output("best_step: int64")
    .Attr("hist_file: string")
    .Attr("tol: float = 0.0")
    .Attr("minimize: bool = true")
    .Attr("metric: string = \"\"")
    .SetIsStateful()
    .Doc(R"doc(

Determines the best global step from a history file.

best_step: Shape [2]. best_step[0] is scalar value for best global step.
  best_step[1] is scalar value for last global step.
hist_file: A text file containing 'step score' records, or a file pattern that
    matches tf event files in the format of /path_to_file/events.out.tfevents*.
tol: Difference between previous best score and current score must be greater
than this amount to trigger update.
minimize: If the metric is being minimized. Recorded in hist_file, smaller
    scores are better if True, and bigger scores are better if False.
metric: The name of the metric being tracked.
)doc");

REGISTER_OP("BeamSearchStep")
    .Input("scores: float32")
    .Input("atten_probs: float32")
    .Input("best_scores: float32")
    .Input("cumulative_scores: float32")
    .Input("in_scores: float32")
    .Input("in_hyps: int32")
    .Input("in_prev_hyps: int32")
    .Input("in_done_hyps: string")
    .Input("in_atten_probs: float32")
    .Input("is_last_chunk: bool")
    .Input("cur_step: int32")
    .Output("out_best_scores: float32")
    .Output("out_cumulative_scores: float32")
    .Output("out_scores: float32")
    .Output("out_hyps: int32")
    .Output("out_prev_hyps: int32")
    .Output("out_done_hyps: string")
    .Output("out_atten_probs: float32")
    .Output("all_done: bool")
    .Attr("eoc_id: int = -1")
    .Attr("eos_id: int")
    .Attr("beam_size: float")
    .Attr("num_hyps_per_beam: int")
    .Attr("valid_eos_max_logit_delta: float = 5.0")
    .Attr("local_eos_threshold: float = -100.0")
    .Attr("merge_paths: bool = false")
    .Attr("allow_empty_terminated_hyp: bool = true")
    .Attr("ensure_full_beam: bool = false")
    .Attr("force_eos_in_last_step: bool = false")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      c->set_output(0, c->input(2));
      c->set_output(1, c->input(3));
      c->set_output(2, c->input(4));
      c->set_output(3, c->input(5));
      c->set_output(4, c->input(6));
      c->set_output(5, c->input(7));
      c->set_output(6, c->input(8));
      c->set_output(7, c->Scalar());
      return Status::OK();
    })
    .Doc(R"doc(

Move forward one step in beam search.

Let "b" be the number of beams, "k" be the number hyps in each beam, "t" be the
maximum decoding steps.

The following data structures are allocated before the first decoding step and
are passed along from cur step to the next step:

in_scores
    A tensor of shape [t, b * k]. in_scores[i, j] is the local
    score of the j-th hyp at the i-th decoding step.
in_hyps
    A tensor of shape [t, b * k]. in_hyps[i, j] is the token id of the
    j-th hyp at the i-th decoding step.
in_prev_hyps
    A tensor of shape [t, b * k]. in_prev_hyps[i, j] stores a
    pointer of the j-th hyp at time step i to the hyp at previous timestep
    (i - 1). 0 <= in_prev_hyps[i, j] < b * k.
in_done_hyps
    A tensor of shape [t, b * k]. in_done_hyps[i, j] can be either an
    empty string, or a serialized Hypothesis proto. Terminated hyps are removed
    from the beam and are moved to the corresponding in_done_hyps slot.
in_atten_probs
    A tensor of shape [t, b * k, s_len]. in_atten_probs[i, j, ...]
    is the attention probs over the source words for the j-th hyp at the i-th
    timestep.

Those tensors are modified (with content for the cur_step timestep being filled
in) within this op invocation and are passed to the corresponding output
tensors.

is_last_chunk: A tensor of shape [b * k]. Used by neural transducer, determine
    whether the current hypothesis reaches the last chunk and should treat the
    next end-of-chunk symbol as end-of-sentence.
scores: A matrix of shape [b * k, vocab_size], where b is the number of
    active beams, and k is the number of hyps in each beam. Local scores for the
    current timestep.
atten_probs: A matrix of shape [b * k, source_len]. Attention probabilities
    for the current timestep.
best_scores: A vector of size [b], best scores of terminated hyps so far in
    each of the beams.
cumulative_scores: A vector of size [b * k]. The cumulative score of each
    active hyp before the current step.
in_scores: As explained above.
in_hyps: As explained above.
in_prev_hyps: As explained above.
in_done_hyps: As explained above.
in_atten_probs: As explained above.
cur_step: Current step id.
out_best_scores:
    Updated best scores for each of the beams.
out_cumulative_scores:
    A vector of size [b * k]. The cumulative score of the new hyps after the
    current decoding step.
out_scores:
    As explained above.
out_hyps:
    As explained above.
out_prev_hyps:
    As explained above.
out_done_hyps:
    As explained above.
out_atten_probs:
    As explained above.
all_done:
    A scalar, whether decoding should terminate for all beams.
eoc_id: Token id of the special end of chunk token.
eos_id: Token id of the special end of sequence token.
beam_size: Search terminates if the delta between the scores of the active hyps
    in a beam and the best scores exceeds this threashold.
num_hyps_per_beam: Number of hyps in a beam.
valid_eos_max_logit_delta: We allow </s> to terminate a hyp only if its logit
    is no more than `valid_eos_max_logit_delta` away from the logit of the best
    candidate.
local_eos_threshold: We allow </s> to terminate a hyp if the local score for
    </s> is greater than local_eos_threshold.
merge_paths: If true, hyps which are identical when epsilons are removed will
    be combined into a single hyp.  The probability for that combined hyp will
    be the sum of the probabilities of the component hyps.  This can only be
    applied for epsilon-emitting models (RNN-T and NT).
allow_empty_terminated_hyp: Whether it is okay to consider a hyp that consists
    only of epsilons as terminated.  By default this is true, as an
    utterance may consist of silence.  It should be set to false when EMBR
    training epsilon-emitting models (e.g., RNN-T), which are prone to emit
    all-epsilon hyps even in the absence of silence.  Note that a hyp that
    terminates in EOS is not considered empty, so this flag has no effect for
    non-epsilon-emitting models.
ensure_full_beam: If True, we will not set the all_done output to True until we
     have found 'num_hyps_per_beam' terminated hyps AND no active hyps have a
     score within 'beam_size' of the best terminated hyp.  If False, only the
     second condition must be satisfied.  Generally this should be False unless
     beam search is being run as part of minimum word error rate training.
force_eos_in_last_step: If true, then if decode does not terminate even after
    (max - 1) steps, eos symbol is injected into the result and partial
    hypotheses (with a valid eos symbol in the end) are returned. all_done
    is set to true for these partials. If false, which is the default behavior,
    empty hypothesis are returned and all_done is set to false at termination.
)doc");

REGISTER_OP("TopKTerminatedHyps")
    .Input("in_done_hyps: string")
    .Input("src_seq_lengths: int32")
    .Output("out_topk_hyps: string")
    .Attr("k: int")
    .Attr("num_hyps_per_beam: int")
    .Attr("length_normalization: float")
    .Attr("coverage_penalty: float")
    .Attr("target_seq_length_ratio: float=1.0")
    .Attr("eoc_id: int=-1")
    .Attr("merge_paths: bool = false")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      auto batch_size = c->Dim(c->input(1), 0);
      int k;
      TF_RETURN_IF_ERROR(c->GetAttr("k", &k));
      shape_inference::DimensionOrConstant k_dim = c->UnknownDim();
      if (k > 0) {
        k_dim = k;
      }
      c->set_output(0, c->Matrix(batch_size, k_dim));
      return Status::OK();
    })
    .Doc(R"doc(

Compute the top k terminated hyps based on normalized score for each beam.

Let "b" be the number of beams, "h" be the number hyps in each beam, "t" be the
maximum decoding steps.

in_done_hyps: A tensor of shape [t, h * b]. Each string in in_done_hyps can be
    either an empty string, or a serialized Hypothesis proto. If not empty,
    in_done_hyps[t, i * num_beams + j] represents the i-th hypothesis for beam
    j that terminates at step t.
src_seq_lengths: A tensor of shape [b] of the src sequence lengths.
out_topk_hyps: A string tensor of shape [b, k]. topk_hyps[i: ] contains
    top k terminated hyp for beam 'i', each hyp could be either an empty string
    or a serialized `Hypothesis` proto.
k: number of highest scoring hyps to be returned for each beam.
num_hyps_per_beam: Number of hyps per beam in the input `in_done_hyps`.
length_normalization: The length normalization ratio.
coverage_penalty: The alpha value for coverage penalty.
target_seq_length_ratio: Ratio of the average target sequence length
    over the average source sequence length.
eoc_id: Token id of the special end of chunk or blank (epsilon) token. -1 means
    this model does not use epsilon.
merge_paths: If true, hyps which are identical when epsilons are removed will
    be combined into a single hyp. The probability for that combined hyp will
    be the sum of the probabilities of the component hyps. This can only be
    applied for epsilon-emitting models (RNN-T and NT).
)doc");

REGISTER_OP("UnpackHyp")
    .Input("in_hyps: string")
    .Output("out_ids: int32")
    .Output("out_seq_lens: int32")
    .Output("out_scores: float32")
    .Attr("max_seq_length: int = 0")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      auto batch_size = c->NumElements(c->input(0));
      int k;
      TF_RETURN_IF_ERROR(c->GetAttr("max_seq_length", &k));
      shape_inference::DimensionOrConstant k_dim = c->UnknownDim();
      if (k > 0) {
        k_dim = k;
      }
      c->set_output(0, c->Matrix(batch_size, k_dim));
      c->set_output(1, c->Vector(batch_size));
      c->set_output(2, c->Vector(batch_size));
      return Status::OK();
    })
    .Doc(R"doc(
Unpacks hyps into tensors of ids, seq_len and scores.

in_hyps: A vector of serialized `Hypothesis` protos.
out_ids:
    Output sequences, a matrix of shape (batch_size, max_seq_length).
    Sequences shorter than max_seq_length are padded with 0s. If max_seq_length is 0, derive it from the longest sequence in input_hyps.
out_seq_lens:
    Length of each of the output sequence, a vector of size `batch_size`.
out_scores:
    Scores for each of the output sequence, a vector of `batch_size`.
)doc");

REGISTER_OP("HypsFromBeamSearchOuts")
    .Input("hyps: int32")
    .Input("prev_hyps: int32")
    .Input("done_hyps: bool")
    .Input("scores: T")
    .Input("atten_probs: T")
    .Input("eos_scores: T")
    .Input("eos_atten_probs: T")
    .Output("out_hyps: string")
    .Attr("T: {float, bfloat16} = DT_FLOAT")
    .Attr("eos_id: int")
    .Attr("num_hyps_per_beam: int")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      c->set_output(0, c->input(0));
      return ::tensorflow::Status::OK();
    })
    .Doc(R"doc(

Generates `Hypothesis` protos from output of a beam search step.

hyps: A tensor of shape [t, b * k] with ids of the token selected.
prev_hyps: A tensor of shape [t, b * k] with index to the previous hyps which
    was selected.
done_hyps: A boolean tensor of shape [t, b * k] where value indicates if hyps
    was terminated.
scores: A tensor of shape [t, b * k]. in_scores[i, j] is the local score of
    the j-th hyp at the i-th decoding step.
atten_probs:  A tensor of shape [t, b * k, s_len]. atten_probs[i, j, ...]
    is the attention probs over the source words for the j-th hyp at the i-th
    timestep.
eos_scores: A tensor of shape [t, b * k]. eos_scores[i, j] is the local
    score of the EOS token at the j-th hyp at the i-th decoding step.
eos_atten_probs: A tensor of shape [t, b * k, s_len].
    eos_atten_probs[i, j, ...] is the attention probs over the source words
    for the j-th terminated hyp at the i-th timestep.
out_hyps: A tensor of shape [t, b * k] with terminated hyps.
eos_id: Token id of the special end of sequence token.
num_hyps_per_beam: Number of hyps per beam.
)doc");

REGISTER_OP("CachedCall")
    .Output("output: T")
    .Attr("f: func")
    .Attr("T: list(type) >= 0")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnknownShape)
    .Doc(R"doc(
Invokes function f once and memorize its output.

output: A list of output tensors whose types are T.
f: A function that returns a list of tensors (T).
)doc");

REGISTER_OP("VocabTokenToId")
    .Input("token: string")
    .Output("id: int32")
    .Attr("vocab: list(string)")
    .Attr("load_token_ids_from_vocab: bool = false")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnchangedShape)
    .Doc(R"doc(
Looks up the token in the vocab and return its id.

token: A scalar or list of strings.
id: A scalar or list of ints.
vocab: A list of strings.
load_token_ids_from_vocab: Whether token ids are present in vocab (i.e. vocab
    contains two colums, one for IDs and one for words).  If false, line numbers
    are used.
)doc");

REGISTER_OP("VocabIdToToken")
    .Input("id: int32")
    .Output("token: string")
    .Attr("vocab: list(string)")
    .Attr("load_token_ids_from_vocab: bool = false")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnchangedShape)
    .Doc(R"doc(
Looks up the token at the given id from a vocab.

id: A scalar or list of ints.
token: A scalar or list of strings.
vocab: A list of strings.
load_token_ids_from_vocab: Whether token ids are present in vocab (i.e. vocab
    contains two colums, one for IDs and one for words).  If false, line numbers
    are used.
)doc");

REGISTER_OP("TokenInVocab")
    .Input("token: string")
    .Output("result: bool")
    .Attr("vocab: list(string)")
    .Attr("load_token_ids_from_vocab: bool = false")
    .SetIsStateful()
    .SetShapeFn(shape_inference::UnchangedShape)
    .Doc(R"doc(
Checks whether the provided token is in the vocab.

token: A scalar or list of strings.
result: A scalar or list of bools.
vocab: A list of strings.
load_token_ids_from_vocab: Whether token ids are present in vocab (i.e. vocab
    contains two colums, one for IDs and one for words).  If false, line numbers
    are used.
)doc");

REGISTER_OP("AsciiToTokenId")
    .Input("labels: string")
    .Output("token_ids: int32")
    .Output("target_ids: int32")
    .Output("paddings: float")
    .Attr("append_eos: bool = true")
    .Attr("maxlen: int = 300")
    .Attr("pad_to_maxlen: bool = true")
    .Doc(R"doc(
Converts ASCII label strings into token ids.

labels: A vector of shape [batch].
token_ids:
    A matrix of shape [batch, maxlen].
    token_ids[i, j] is the i-th sample's j-th token id.
    token_ids[i, 0] is always <s>.
target_ids:
    A matrix of shape [batch, maxlen].
    target_ids[i, j] is the i-th sample's j-th prediction label id.
paddings:
    A matrix of shape [batch, maxlen].
    paddings[i, j] == 1.0 indicates that i-th training example'
    j-th target token is padded and should be ignored.
append_eos: Whether to append </s> at the end and treat it as a non-padded
    label.
maxlen: an integer, sequence length of the output tensors.
pad_to_maxlen: Whether to pad the output to maxlen.
)doc");

REGISTER_OP("StrToVocabTokens")
    .Input("labels: string")
    .Output("token_ids: int32")
    .Output("target_ids: int32")
    .Output("paddings: float")
    .Attr("append_eos: bool = true")
    .Attr("maxlen: int = 300")
    .Attr("pad_to_maxlen: bool = true")
    .Attr("vocab_filepath: string")
    .Attr("load_token_ids_from_vocab: bool = true")
    .Attr("delimiter: string = ' '")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      auto batch_size = c->Dim(c->input(0), 0);
      int maxlen;
      TF_RETURN_IF_ERROR(c->GetAttr("maxlen", &maxlen));
      c->set_output(0, c->Matrix(batch_size, maxlen));
      c->set_output(1, c->Matrix(batch_size, maxlen));
      c->set_output(2, c->Matrix(batch_size, maxlen));
      return Status::OK();
    })
    .Doc(R"doc(
Tokenizes string into white space separated tokens according to a vocab file.

labels: A vector of shape [batch].
token_ids:
    A matrix of shape [batch, maxlen].
    token_ids[i, j] is the i-th sample's j-th token id.
    token_ids[i, 0] is always <s>.
target_ids:
    A matrix of shape [batch, maxlen].
    target_ids[i, j] is the i-th sample's j-th prediction label id.
paddings:
    A matrix of shape [batch, maxlen].
    paddings[i, j] == 1.0 indicates that i-th training example's
    j-th target token is padded and should be ignored.
append_eos: Whether to append </s> at the end and treat it as a non-padded
    label.
maxlen: an integer, sequence length of the output tensors.
pad_to_maxlen: Whether to pad the output to maxlen.
vocab_filepath: a string, filepath to the vocab file.
load_token_ids_from_vocab: Whether token ids are present in vocab (i.e. vocab
    contains two colums, one for IDs and one for words).  If false, line numbers
    are used.
delimiter: The delimiter to split the labels to tokens by.
)doc");

REGISTER_OP("IdToAscii")
    .Input("token_ids: int32")
    .Input("seq_lengths: int32")
    .Output("sequence: string")
    .Doc(R"doc(
Converts sequences from token ids to actual ASCII tokens.

token_ids: A matrix of shape [batch, seq_len].
seq_lengths: A vector of shape [batch]. seq_lengths[i] is the length of the
    i-th sequence. Only the first seq_lengths[i] tokens in token_ids[i] are
    valid tokens for the i-th sequence.
sequence: A vector of shape [batch]. The converted string sequence.
)doc");

REGISTER_OP("NgramIdToToken")
    .Input("token_ids: int32")
    .Input("seq_lengths: int32")
    .Output("sequences: string")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      c->set_output(0, c->input(1));
      return Status::OK();
    })
    .Attr("ngram_vocab_filepath: string")
    .Attr("ngram_separator: string = \"\"")
    .Doc(R"doc(
Converts sequences from token ids to actual tokens.

token_ids: A matrix of shape [batch, seq_len].
seq_lengths: A vector of shape [batch]. seq_lengths[i] is the length of the
    i-th sequence. Only the first seq_lengths[i] tokens in token_ids[i] are
    valid tokens for the i-th sequence.
sequences: A vector of shape [batch]. The converted string sequence.
ngram_vocab_filepath: filepath to the ngram vocab file.
ngram_separator: separator to use when joining ngrams into string.
)doc");

REGISTER_OP("BpeWordsToIds")
    .Input("labels: string")
    .Output("token_ids: int32")
    .Output("target_ids: int32")
    .Output("paddings: float")
    .Attr("append_eos: bool = true")
    .Attr("maxlen: int = 300")
    .Attr("sos_id: int = 1")
    .Attr("eos_id: int = 2")
    .Attr("tokenization_filepath: string")
    .SetShapeFn([](shape_inference::InferenceContext* ctx) {
      const auto batch_size = ctx->Dim(ctx->input(0), 0);
      int maxlen;
      TF_RETURN_IF_ERROR(ctx->GetAttr("maxlen", &maxlen));
      ctx->set_output(0, ctx->Matrix(batch_size, maxlen));
      ctx->set_output(1, ctx->Matrix(batch_size, maxlen));
      ctx->set_output(2, ctx->Matrix(batch_size, maxlen));
      return Status::OK();
    })
    .Doc(R"doc(
A tokenizer to convert string to BPE ids.

This op is especially convenient for mapping string text to a sequenec of word BPE
ids. The `labels` strings are tokenized via BPE. This op is typically used in conjunction with BpeIdsToWords.
As the vocabulary file it receives the mapping from each word to the series of BPE ids.
An example of lines in the vocabulary file:
...
AARON 10005,16,29
AARON'S 10005,16,3466
AARONSON 10005,16,17,447
...

The output tensor `token_ids` is a sequence of integer ids with <s> prepended.
The output tensor `target_ids` is a sequence of integer ids with </s> appended.

labels: The batch of tf.String tensors. Expected shape is [batch_size].
token_ids:
    The ids with <s>. The shape is [batch_size, maxlen].
target_ids:
    The ids with </s>. The shape is [batch_size, maxlen].
paddings:
    The paddings. The shape is [batch_size, maxlen].
maxlen: Maximum length of token_ids/target_ids/paddings.
tokenization_filepath: A path to a text file where each line is a word separated with space form a list of ids which are separated by ','.
)doc");

REGISTER_OP("BpeIdsToWords")
    .Input("token_ids: int32")
    .Input("seq_lengths: int32")
    .Output("sequences: string")
    .Attr("vocab_filepath: string")
    .SetShapeFn([](shape_inference::InferenceContext* ctx) {
      const auto batch_size = ctx->Dim(ctx->input(0), 0);
      ctx->set_output(0, ctx->Vector(batch_size));
      return Status::OK();
    })
    .Doc(R"doc(
A tokenizer to map BPE ids to strings.

This op is to map a sequence of integer ids to a string. The op is typically
used in conjunction with BpeWordsToIds.

The op will consume `seq_lengths` of tokens from `token_ids` and convert it to
string `sequences`. A space character will be interested inbetween tokens. We do
not filter any tokens (i.e., <s> and </s> are not treated specially).

token_ids: The ids (can include paddings; length is determined by seq_lengths). The shape is [batch_size, maxlen].
seq_lengths: The length of the ids. The shape is [batch_size].
sequences: The string sequences. The shape is [batch_size].
vocab_filepath: A path to a text file where each line is a BPE string token.
)doc");

REGISTER_OP("GenericInput")
    .Output("out: out_types")
    .INPUT_ATTRS  // Common input attributes.
    .Attr("out_types: list(type)")
    .Attr("processor: func")
    .Attr("dynamic_padding_dimensions: list(int) = []")
    .Attr("dynamic_padding_constants: list(int) = []")
    .Doc(R"doc(
Produces examples from processed from records.

out: The 1st dimension of every tensor is the batch dimension.
)doc" INPUT_DOCS
         R"doc(
out_types: A list of tensor types.
processor: A function that processes a string (one record) and returns
    a list of tensors. The last tensor must be a int32 scalar, which is
    used in conjunction with `bucket_upper_bound` to bucket all the
    samples.  The other tensors belongs to one sample. They have the
    respective `out_types`.  These tensors' first dimension are _not_ the
    batch dimension. Instead, when multiple samples are merged into a
    batch, GenericInput's implementation expand the batch dimension (dim
    0) and concatenate the corresponding tensors into one tensor.
dynamic_padding_dimensions: If not empty, must be the same length as out.
    Specifies the 0-indexed dimension to pad dynamically for each output.
    The output is padded to the longest tensor in the batch along the dimension.
    The first (0-th) dimension is _not_ the batch dimension. A value of -1
    indicates the specified output should not be padded, eg. if the output is a
    scalar rather than a sequence.
dynamic_padding_constants: Must be set if `dynamic_padding_dimension` is
    provided. The constant value to use for padding.
)doc");

REGISTER_OP("StaticMapStringInt")
    .Input("x: string")
    .Output("y: int32")
    .Attr("keys: list(string)")
    .Attr("vals: list(int) = []")
    .Attr("unk: int = -1")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      c->set_output(0, c->input(0));
      return ::tensorflow::Status::OK();
    })
    .Doc(R"doc(
Maps every element of x according a static mapping.

x: A Tensor of type string.
y: A Tensor of type int32. Same shape of x.
keys: The list of keys.
vals: The list of values. If empty, defaults to [0 .. len(keys)).
unk: The value when the key is not found.
)doc");

REGISTER_OP("StaticMapIntString")
    .Input("x: int32")
    .Output("y: string")
    .Attr("keys: list(int) = []")
    .Attr("vals: list(string)")
    .Attr("unk: string = ''")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      c->set_output(0, c->input(0));
      return ::tensorflow::Status::OK();
    })
    .Doc(R"doc(
Maps every element of x according a static mapping.

x: A Tensor of type int32.
y: A Tensor of type string. Same shape of x.
keys: The list of keys. If empty, defaults to [0 .. len(keys)).
vals: The list of values.
unk: The value when the key is not found.
)doc");

REGISTER_OP("ComputePreconditioners")
    .Input("inputs: num_tensors * float32")
    .Input("exponents: num_tensors * float32")
    .Input("global_step: int32")
    .Attr("preconditioner_compute_graphdef: string")
    .Attr("keys: list(string)")
    .Attr("sync: bool = false")
    .Attr("num_tensors: int >= 1")
    .SetIsStateful()
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      return Status::OK();
    })
    .Doc(R"doc(
Compute preconditioners for Shampoo optimizer.

inputs: A list of Tensors of type float32, of statistic matrices.
exponents: A list of scalar Tensors of type float32, exponent for matrix power.
global_step: A scalar Tensor of type int32 which indicates the global step.
preconditioner_compute_graphdef: A graphdef which indicates the function to run.
keys: A list of keys indicating the name of preconditioners.
sync: Boolean indicating whether to run preconditioning in synchronous mode.
num_tensors: Number of tensor inputs.
)doc");

REGISTER_OP("GetPreconditioners")
    .Input("shapes: num_tensors * Tshape")
    .Output("outputs: num_tensors * float32")
    .Output("statuses: num_tensors * bool")
    .Attr("preconditioner_compute_graphdef: string")
    .Attr("keys: list(string)")
    .Attr("Tshape: {int32, int64} = DT_INT32")
    .Attr("num_tensors: int >= 1")
    .SetIsStateful()
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      std::vector<shape_inference::ShapeHandle> shapes;
      if (c->input("shapes", &shapes).ok()) {
        for (int i = 0; i < shapes.size(); ++i) {
          shape_inference::ShapeHandle out;
          TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(i, &out));
          c->set_output(i, out);
          c->set_output(shapes.size() + i, c->Scalar());
        }
      }
      return Status::OK();
    })
    .Doc(R"doc(
Get preconditioners for Shampoo optimizer.

shapes: A list of Tensors of type Tshape indicating the size of preconditioner.
outputs: A list of Tensors of type float32 which are the preconditioners.
statuses: A list of Tensors of type bool which are the preconditioner status.
preconditioner_compute_graphdef: A graphdef which indicates the function to run.
keys: A list of keys indicating the name of preconditioners.
Tshape: The data-type to use for shape.
num_tensors: Number of tensor inputs.
)doc");

REGISTER_OP("MlPerfSubwordIdToString")
    .Input("token_ids: int32")
    .Input("seq_lengths: int32")
    .Output("sequences: string")
    .SetShapeFn([](shape_inference::InferenceContext* c) {
      c->set_output(0, c->input(1));
      return Status::OK();
    })
    .Attr("vocab_filepath: string")
    .Doc(R"doc(
Converts sequences from subword token ids to strings

token_ids: A matrix of shape [batch, seq_len].
seq_lengths: A vector of shape [batch]. seq_lengths[i] is the length of the
    i-th sequence. Only the first seq_lengths[i] tokens in token_ids[i] are
    valid tokens for the i-th sequence.
sequences: A vector of shape [batch]. The converted string sequence.
vocab_filepath: filepath to the MLPerf subword vocab file.
)doc");

REGISTER_OP("PackSequences")
    .Input("src_actual_seq_len: int32")
    .Input("tgt_actual_seq_len: int32")
    .Input("packed_batch_size: int32")
    .Input("packed_src_seq_len: int32")
    .Input("packed_tgt_seq_len: int32")
    .SetIsStateful()
    .Output("src_segment_ids: int32")
    .Output("src_segment_pos: int32")
    .Output("src_indices_in_input: int32")
    .Output("tgt_segment_ids: int32")
    .Output("tgt_segment_pos: int32")
    .Output("tgt_indices_in_input: int32")
    .Attr("seed: int = 0")
    .Doc(R"doc(
Produces a packing pattern for the (src, tgt) input pair with the provided
lengths, according to the given packed shape.

src_actual_seq_len: A tensor of shape [N], where N is the input batch size.
  This tensor contains the actual lengths for the src sequence.
tgt_actual_seq_len: A tensor of shape [N], where N is the input batch size.
  This tensor contains the actual lengths for the tgt sequence.
packed_batch_size: A scalar. The output batch size. The packed output will
  be of shape [packed_batch_size, packed_{src,tgt}_seq_len] for src and tgt,
  respectively.
packed_src_seq_len: A scalar. The output sequence length for src. A src input
  with shape [N, src_input_seq_len] will be packed into an output with shape
  [packed_batch_size, packed_src_seq_len].
packed_tgt_seq_len: A scalar. The output sequence length for tgt. A tgt input
  with shape [N, tgt_input_seq_len] will be packed into an output with shape
  [packed_batch_size, packed_tgt_seq_len].

{src,tgt}_segment_ids: A tensor of shape [packed_batch_size,
  packed_{src,tgt}_seq_len]. Incrementing from 1 to indicate each segment in the
  packed output, for src and tgt, respectively. Zero is reserved for indicating
  padding at the end of each row.
{src,tgt}_segment_pos: A tensor of shape [packed_batch_size,
  packed_{src,tgt}_seq_len]. Zero-based index to indicate relative position
  within each segment, for src and tgt, respectively. Zero is also used to
  indicate padding. When needed, use `{src,tgt}_segment_ids` to disambiguate.
{src,tgt}_indices_in_input: A tensor of shape [packed_batch_size,
  packed_{src,tgt}_seq_len]. For each segment in the packed output, it contains
  the original (zero-based) row index of each segment found in
  `{src,tgt}_actual_seq_len`, respectively. Zero is also used to indicate
  padding. When needed, use `{src,tgt}_segment_ids` to disambiguate.

seed: Seed for random number generator, which is used when we need to drop
  excessive input sequences. If seed is zero, use completely random seed.

For example, the following input:
  src_actual_seq_len = [3, 2, 1]
  tgt_actual_seq_len = [4, 1, 5]
  packed_batch_size = 2
  packed_src_seq_len = 5
  packed_tgt_seq_len = 5
will result in:
  src_segment_ids = [ [1, 1, 1, 2, 2], [1, 0, 0, 0, 0] ]
  src_segment_pos = [ [0, 1, 2, 0, 1], [0, 0, 0, 0, 0] ]
  src_indices_in_input = [ [0, 0, 0, 1, 1], [2, 0, 0, 0, 0] ]
  tgt_segment_ids = [ [1, 1, 1, 1, 2], [1, 1, 1, 1, 1] ]
  tgt_segment_pos = [ [0, 1, 2, 3, 0], [0, 1, 2, 3, 4] ]
  tgt_indices_in_input = [ [0, 0, 0, 0, 1], [2, 2, 2, 2, 2] ]

The packed sequence length can be different between src and tgt. For example,
the following input:
  src_actual_seq_len = [3, 2, 1]
  tgt_actual_seq_len = [4, 1, 5]
  packed_batch_size = 2
  packed_src_seq_len = 4
  packed_tgt_seq_len = 6
will result in:
  src_segment_ids = [ [1, 1, 1, 0], [1, 1, 2, 0] ]
  src_segment_pos = [ [0, 1, 2, 0], [0, 1, 0, 0] ]
  src_indices_in_input = [ [0, 0, 0, 0], [1, 1, 2, 0] ]
  tgt_segment_ids = [ [1, 1, 1, 1, 0, 0], [1, 2, 2, 2, 2, 2] ]
  tgt_segment_pos = [ [0, 1, 2, 3, 0, 0], [0, 0, 1, 2, 3, 4] ]
  tgt_indices_in_input = [ [0, 0, 0, 0, 0, 0], [1, 2, 2, 2, 2, 2] ]

If there are too few input sequences to pack into `output_shape`, the op pads
the remaining elements in the output.

If there are too many input sequences to pack into `output_shape`, the op drops
input sequences. The dropping is done randomly uniformly on the input sequences
to not bias the distribution of sequence lengths in the packed output.
 )doc");

REGISTER_OP("ApplyPacking")
    .Input("input: T")
    .Input("padding: T")
    .Input("segment_ids: int32")
    .Input("indices_in_input: int32")
    .Output("output: T")
    .Attr("T: type")
    .Doc(R"doc(
Applies a packing pattern on the input to obtain a packed output.

A slightly different semantics when T is tf.string type: the output joins the
strings that are packed on the same row, separated by `padding`.

input: A tensor of shape [N, seq_len]. The input to apply the packing to.
  For tf.string typed input, a vector of shape [N] is expected.
padding: A scalar to indicate the padding value. This is typically the zero
  value of T, but may not always be the case, e.g. when the input is a paddings
  tensor, in which case caller should set padding=1.
  For tf.string typed input, padding is used as a separator to join all the
  strings on the same row in the output.
segment_ids: A rank 2 tensor of shape `output_shape`.
indices_in_input: A rank 2 tensor of shape `output_shape`.

output: A tensor of shape `output_shape`. For tf.string typed input, the output
  is a vector of strings where its length is the same as the number of rows in
  `output_shape`.

The inputs `segment_ids` and `indices_in_input` can be obtained from the outputs
of an `PackSequence` op (though only the src or the tgt tensors are needed).

Note that ApplyPacking is done on a per column basis (either on the src or on
the tgt), as opposed to in PackSequences, when both src and tgt columns must be
processed together within the same op.
 )doc");

REGISTER_OP("Mass")
    .Input("ids: int32")
    .Input("weights: float32")
    .Input("actual_seq_len: int32")
    .Attr("mask_id: int")
    .Attr("mask_ratio: float = 0.5")
    .Attr("mask_minlen: int = 0")
    .Attr("span_len: int = 100000")
    .Attr("random_start_prob: float = 0.6")
    .Attr("keep_prob: float = 0.1")
    .Attr("rand_prob: float = 0.1")
    .Attr("mask_prob: float = 0.8")
    // TODO(alisonlui): This flag is rarely used; remove after verification.
    .Attr("mask_target: bool = True")
    .Attr("vocab_size: int")
    .Attr("first_unreserved_id: int = 4")
    .Output("src_ids: int32")
    .Output("tgt_ids: int32")
    .Output("tgt_labels: int32")
    .Output("tgt_weights: float32")
    .Doc(R"doc(
Applies masking to implement MASS.

ids: Tensor of shape [batch_size, max_seq_len] containing the token ids.
  Should include EOS token </s>.
weights: Tensor of shape [batch_size, max_seq_len].
actual_seq_len: Tensor of shape [batch_size].

mask_id: The id to use for the mask token.
mask_ratio: Proportion of src to mask.
mask_minlen: Skip sentences too short to mask at least this many tokens.
span_len: Split mask_len into segments of this size and randomly distribute
those across the src.
random_start_prob: The probability that the placement of masked segments will be
  entirely random. The remaining cases are split evenly between masking at the
  beginning and at the end of the src.
keep_prob/rand_prob/mask_prob: The probability that a token to be masked will
be unchanged, replaced with a random token in the vocab, or replaced with the
mask_id, respectively. Must sum to 1.
mask_target: whether to mask the target (the mask will be the inverse of that of
  the src).
vocab_size: Vocab size used when selecting a random token to replace a masked
  token.
first_unreserved_id: Tokens greater than or equal to this may be selected at
  random to replace a masked token.

src_ids: Masked ids. E.g. s1 s2 s3 m m </s>
tgt_ids: Right-shifted ids with BOS token added, where the mask is the
  positional inverse of that of the source unless mask_target=False.
  E.g. m m m s3 s4 m
tgt_labels: E.g. s1 s2 s3 s4 s5 </s>
tgt_weights: weights are zeroed wherever the target is masked.
)doc");

}  // namespace
}  // namespace tensorflow


text:
<ion-view class="itemView" view-title="Ordine inviato - {{tavolo.descrizione}}" >
    <ion-content>

                Totale: {{totale  | number:2}}€

                A persona: {{quotaPersona  | number:2}}€

          <button  class="button button-positive button-block button-light animated bounceInUp" ng-click="nuovoOrdine()">Torna ai tavoli</button>

    </ion-content>
</ion-view>


text:
html, body {
    background-color: #eee;
}
body {
    padding-top: 40px; /* 40px to make the container go all the way to the bottom of the topbar */
}
.container > footer p {
    text-align: center; /* center align it with the container */
}
.container {
    width: 820px; /* downsize our container to make the content feel a bit tighter and more cohesive. NOTE: this removes two full columns from the grid, meaning you only go to 14 columns and not 16. */
}

/* The white background content wrapper */
.content {
    background-color: #fff;
    padding: 20px;
    margin: 0 -20px; /* negative indent the amount of the padding to maintain the grid system */
    -webkit-border-radius: 0 0 6px 6px;
       -moz-border-radius: 0 0 6px 6px;
            border-radius: 0 0 6px 6px;
    -webkit-box-shadow: 0 1px 2px rgba(0,0,0,.15);
       -moz-box-shadow: 0 1px 2px rgba(0,0,0,.15);
            box-shadow: 0 1px 2px rgba(0,0,0,.15);
    min-height: 10px;
}

/* Page header tweaks */
.page-header {
    background-color: #f5f5f5;
    padding: 20px 20px 10px;
    margin: -20px -20px 20px;
}

/* Styles you shouldn't keep as they are for displaying this base example only */
.content .span10,
.content .span4 {
    min-height: 500px;
}

/* Give a quick and non-cross-browser friendly divider */
.content .span4 {
    padding-left: 9px;
    margin-left: 10px;
    border-left: 1px solid #eee;
}

.topbar .btn {
    border: 0;
}

.topbar p {
    color: #888;
}

#username {
    width: 200px;
}

#main {
    position: relative;
    overflow: hidden;
}

#onError, #onChat {
    display: none;
}

#messages {
    position: absolute;
    bottom: 60px;
}

#talk {
    position: absolute;
    bottom: 0;
    left: 0;
    right: 0;
    width: auto;
    height: 40px
}

.message {
    padding: 4px 0;
    border-bottom: 1px solid #eee;
    position: relative;
}

.message span {
    width: 70px;
    overflow: hidden;
    text-align: right;
    font-weight: bold;
    margin-right: 5px;
    display: inline-block;
    vertical-align: top;
    position: relative;
    top: -1px;
}

.message p {
    display: inline-block;
    margin: 0;
    width: 500px;
}

.message.me {
    background: #FFC;
}

.message.join, .message.quit {
    background: #D9E7FB;
}


text:
import Entypo from 'react-native-vector-icons/Entypo';
import MaterialCommunityIcons from 'react-native-vector-icons/MaterialCommunityIcons';
import {PixelRatio} from 'react-native';

const ICON_SIZE = __DEV__ ? 24 : PixelRatio.getPixelSizeForLayoutSize(24);
const iconsToLoad = {
  "plus": [Entypo, 'plus', ICON_SIZE, '#FFFFFF'],
  "pencil": [MaterialCommunityIcons, 'pencil', ICON_SIZE, '#FFFFFF'],
  "delete": [MaterialCommunityIcons, 'delete', ICON_SIZE, '#FFFFFF'],
};

let Icons = {};
let LoadIcons = new Promise((resolve, reject) => {
  Promise.all(
    Object.keys(iconsToLoad).map(iconName =>
      iconsToLoad[iconName][0].getImageSource(
        iconsToLoad[iconName][1],
        iconsToLoad[iconName][2],
        iconsToLoad[iconName][3]
      ))
  ).then(sources => {
    Object.keys(iconsToLoad)
      .forEach((iconName, idx) => Icons[iconName] = sources[idx]);
    resolve(true);
  })
});

export {
  Icons,
  LoadIcons
};


text:
from difflib import SequenceMatcher

import csv

authornames = set()
allrows = list()

with open('amplifiedficmeta.tsv', encoding = 'utf-8') as f:
    reader = csv.DictReader(f, delimiter = '\t')
    fieldnames = reader.fieldnames

    for row in reader:
        allrows.append(row)
        authornames.add(row['author'])

authornames = list(authornames)

synonyms = dict()

for name in authornames:
    for anothername in authornames:
        if name == anothername:
            continue

        if name in synonyms:
            if synonyms[name] == anothername:
                continue

        if anothername in synonyms:
            if synonyms[anothername] == name:
                continue

        m = SequenceMatcher(None, name, anothername)
        match = m.ratio()

        if match > 0.85:
            print(name + " | " + anothername + " | " + str(match))

            user = input('Are these synonymous? ')

            if not user.lower().startswith('y'):
                continue

            user = input("Prefer 1) first or 2) second: ")
            if user == "1":
                synonyms[anothername] = name
            elif user == '2':
                synonyms[name] = anothername

with open('namefusedficmeta.tsv', mode = 'w', encoding = 'utf-8') as f:
    writer = csv.DictWriter(f, fieldnames = fieldnames, delimiter = '\t')
    writer.writeheader()
    for row in allrows:
        author = row['author']
        if author in synonyms:
            row['author'] = synonyms[author]

        writer.writerow(row)


text:
// Object.h - Copyright 2013-2016 Will Cassella, All Rights Reserved
#pragma once

#include "Reflection/Reflection.h"
#include "Memory/ReferenceCounter.h"
#include "Utility/UnMoveable.h"

/** Base of any polymorphic type that has to act within a larger system.
* May be referenced with 'Weak<T>' and 'Borrowed<T>' smart pointers. */
class CORE_API Object : public willow::UnMoveable
{
	///////////////////////
	///   Inner Types   ///
protected:

	/** Flags for constructing Objects. */
	enum ObjectConstructionFlags
	{
		/** This object is constructed without a reference count block. */
		NoReferenceCount
	};

	///////////////////////
	///   Information   ///
public:

	REFLECTION_DECL(ClassInfo)
	IMPLEMENTS()

	////////////////////////
	///   Constructors   ///
public:

	/** Constructs an Object with a reference counter. */
	Object();

	/** Constructs an Object without a reference counter. */
	explicit Object(ObjectConstructionFlags flags);

	/** Marks Object as being destroyed (if it's refcounted). */
	virtual ~Object();

	///////////////////
	///   Methods   ///
public:

	/** Returns whether this Object is reference counted. */
	FORCEINLINE bool IsReferenceCounted() const
	{
		return _referenceCounter != nullptr;
	}

	/** Returns a pointer to the reference counter for this Object.
	* NOTE: Returns 'null' if this Object is not reference counted. */
	FORCEINLINE ReferenceCounter* GetReferenceCounter() const
	{
		return _referenceCounter;
	}

	/** Returns the type information for this Object. */
	virtual const ClassInfo& GetType() const = 0;

	/** Formats the state of this Object as a String. */
	virtual String ToString() const;

	////////////////
	///   Data   ///
private:

	ReferenceCounter* _referenceCounter;
};


text:
using System;
using System.Collections.Generic;
using System.Linq;
using FluentAssertions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using NClap.ConsoleInput;
using NSubstitute;

namespace NClap.Tests.ConsoleInput
{
    [TestClass]
    public class ConsoleReaderUnitTests
    {
        [TestMethod]
        public void SimpleKeyPress()
        {
            Check(
                'h',
                input =>
                {
                    input.Insert('h');
                    input.MoveCursorForward(1);
                },
                input => { input.AtEnd.Returns(true); });
        }

        [TestMethod]
        public void Movement()
        {
            Check(ConsoleKey.LeftArrow, input => { input.MoveCursorBackward(1); });
            Check(ConsoleKey.RightArrow, input => { input.MoveCursorForward(1); });
            Check(ConsoleKey.UpArrow, input => { input.ReplaceWithLastLineInHistory(); });
            Check(ConsoleKey.DownArrow, input => { input.ReplaceWithNextLineInHistory(); });
            Check(ConsoleKey.Escape, input => { input.ClearLine(false); });
            Check(ConsoleKey.Backspace, input => { input.DeletePrecedingChar(); });
            Check(ConsoleKey.Delete, input => { input.Delete(); });
            Check(ConsoleKey.Home, input => { input.MoveCursorToStart(); });
            Check(ConsoleKey.End, input => { input.MoveCursorToEnd(); });
            Check(ConsoleKey.Enter, input => { }, expectedResult: ConsoleInputOperationResult.EndOfInputLine);

            var lessThan = new ConsoleKeyInfo('<', ConsoleKey.OemComma, true, true, false);
            Check(lessThan, input => { input.ReplaceWithOldestLineInHistory(); });

            var greaterThan = new ConsoleKeyInfo('>', ConsoleKey.OemPeriod, true, true, false);
            Check(greaterThan, input => { input.ReplaceWithYoungestLineInHistory(); });
        }

        [TestMethod]
        public void Capitalize()
        {
            ConsoleReader.Capitalize(string.Empty).Should().BeEmpty();
            ConsoleReader.Capitalize("a").Should().Be("A");
            ConsoleReader.Capitalize("A").Should().Be("A");
            ConsoleReader.Capitalize(" a").Should().Be(" a");
            ConsoleReader.Capitalize("abcd").Should().Be("Abcd");
            ConsoleReader.Capitalize("ab cd").Should().Be("Ab cd");
            ConsoleReader.Capitalize("AB CD").Should().Be("Ab cd");
        }

        [TestMethod]
        public void BogusOp()
        {
            var reader = CreateReader();

            reader.Invoking(r => r.Process((ConsoleInputOperation)0xFFFF, new ConsoleKeyInfo()))
                  .Should().Throw<ArgumentOutOfRangeException>();
        }

        [TestMethod]
        public void NoOp()
        {
            var anyKey = Any.Enum<ConsoleKey>();

            var bindingSet = ConsoleKeyBindingSet.CreateDefaultSet();
            bindingSet.Bind(anyKey, (ConsoleModifiers)0, ConsoleInputOperation.NoOp);

            var reader = CreateReader(bindingSet: bindingSet);
            reader.ProcessKey(anyKey.ToKeyInfo()).Should().Be(ConsoleInputOperationResult.Normal);
        }

        [TestMethod]
        public void UnimplementedOp()
        {
            var reader = CreateReader(
                ConsoleKey.RightArrow.AsInfo().Concat(ConsoleKey.Enter.AsInfo()));

            reader.LineInput.MoveCursorForward(Arg.Any<int>()).ReturnsForAnyArgs(
                call => { throw new NotImplementedException(); });

            reader.ReadLine().Should().BeEmpty();
            reader.LineInput.Received().MoveCursorForward(1);
        }

        private static void Check(
            char value,
            Action<IConsoleLineInput> expectedCalls,
            Action<IConsoleLineInput> inputSetup = null,
            ConsoleInputOperationResult expectedResult = ConsoleInputOperationResult.Normal)
        {
            Check(value.ToKeyInfo(), expectedCalls, inputSetup, expectedResult);
        }

        private static void Check(
            ConsoleKey key,
            Action<IConsoleLineInput> expectedCalls,
            Action<IConsoleLineInput> inputSetup = null,
            ConsoleInputOperationResult expectedResult = ConsoleInputOperationResult.Normal)
        {
            Check(key.ToKeyInfo(), expectedCalls, inputSetup, expectedResult);
        }

        private static void Check(
            ConsoleKeyInfo keyInfo,
            Action<IConsoleLineInput> expectedCalls,
            Action<IConsoleLineInput> inputSetup = null,
            ConsoleInputOperationResult expectedResult = ConsoleInputOperationResult.Normal)
        {
            var reader = CreateReader();
            inputSetup?.Invoke(reader.LineInput);
            reader.ProcessKey(keyInfo).Should().Be(expectedResult);
            Received.InOrder(() => expectedCalls(reader.LineInput));
        }

        private static ConsoleReader CreateReader(IEnumerable<ConsoleKeyInfo> keyStream = null, ConsoleKeyBindingSet bindingSet = null)
        {
            var consoleOutput = new SimulatedConsoleOutput();
            var consoleInput = new SimulatedConsoleInput(keyStream ?? Enumerable.Empty<ConsoleKeyInfo>());
            var input = Substitute.For<IConsoleLineInput>();
            return new ConsoleReader(input, consoleInput, consoleOutput, bindingSet);
        }
    }
}


text:
/**
*============================================================================
*  The Ohio State University Research Foundation, Emory University,
*  the University of Minnesota Supercomputing Institute
*
*  Distributed under the OSI-approved BSD 3-Clause License.
*  See http://ncip.github.com/cagrid-grid-incubation/LICENSE.txt for details.
*============================================================================
**/
/**
*============================================================================
*============================================================================
**/
package org.cagrid.workflow.helper.service.globus;

import java.rmi.RemoteException;

import org.cagrid.workflow.helper.service.WorkflowHelperImpl;

/**
 * DO NOT EDIT:  This class is autogenerated!
 *
 * This class implements each method in the portType of the service.  Each method call represented
 * in the port type will be then mapped into the unwrapped implementation which the user provides
 * in the WorkflowHelperImpl class.  This class handles the boxing and unboxing of each method call
 * so that it can be correclty mapped in the unboxed interface that the developer has designed and
 * has implemented.  Authorization callbacks are automatically made for each method based
 * on each methods authorization requirements.
 *
 * @created by Introduce Toolkit version 1.2
 *
 */
public class WorkflowHelperProviderImpl{

	WorkflowHelperImpl impl;

	public WorkflowHelperProviderImpl() throws RemoteException {
		impl = new WorkflowHelperImpl();
	}

    public org.cagrid.workflow.helper.stubs.CreateWorkflowInstanceHelperResponse createWorkflowInstanceHelper(org.cagrid.workflow.helper.stubs.CreateWorkflowInstanceHelperRequest params) throws RemoteException {
    org.cagrid.workflow.helper.stubs.CreateWorkflowInstanceHelperResponse boxedResult = new org.cagrid.workflow.helper.stubs.CreateWorkflowInstanceHelperResponse();
    boxedResult.setWorkflowInstanceHelperReference(impl.createWorkflowInstanceHelper(params.getWorkflowInstanceHelperDescriptor().getWorkflowInstanceHelperDescriptor()));
    return boxedResult;
  }

    public org.cagrid.workflow.helper.stubs.GetIdentityResponse getIdentity(org.cagrid.workflow.helper.stubs.GetIdentityRequest params) throws RemoteException {
    org.cagrid.workflow.helper.stubs.GetIdentityResponse boxedResult = new org.cagrid.workflow.helper.stubs.GetIdentityResponse();
    boxedResult.setResponse(impl.getIdentity());
    return boxedResult;
  }

}


text:
/* global WebFont */
(function() {
  function vendorModule() {
    'use strict';

    return {
      'default': self['webfontloader'],
      __esModule: true,
      'WebFont': self.WebFont
    };
  }

  define('webfontloader', [], vendorModule);
})();


text:
package io.crate.metadata.doc;

import io.crate.Constants;
import io.crate.exceptions.RelationUnknown;
import io.crate.exceptions.UnhandledServerException;
import io.crate.metadata.IndexParts;
import io.crate.metadata.NodeContext;
import io.crate.metadata.PartitionName;
import io.crate.metadata.RelationName;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.Version;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.apache.logging.log4j.LogManager;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;

public class DocTableInfoBuilder {

    private final RelationName ident;
    private final ClusterState state;
    private final NodeContext nodeCtx;
    private final IndexNameExpressionResolver indexNameExpressionResolver;
    private final Metadata metadata;
    private String[] concreteIndices;
    private String[] concreteOpenIndices;
    private static final Logger LOGGER = LogManager.getLogger(DocTableInfoBuilder.class);

    public DocTableInfoBuilder(NodeContext nodeCtx,
                               RelationName ident,
                               ClusterState state,
                               IndexNameExpressionResolver indexNameExpressionResolver) {
        this.nodeCtx = nodeCtx;
        this.indexNameExpressionResolver = indexNameExpressionResolver;
        this.ident = ident;
        this.state = state;
        this.metadata = state.metadata();
    }

    private DocIndexMetadata docIndexMetadata() {
        DocIndexMetadata docIndexMetadata;
        String templateName = PartitionName.templateName(ident.schema(), ident.name());
        if (metadata.getTemplates().containsKey(templateName)) {
            docIndexMetadata = buildDocIndexMetadataFromTemplate(ident.indexNameOrAlias(), templateName);
            // We need all concrete indices, regardless of their state, for operations such as reopening.
            concreteIndices = indexNameExpressionResolver.concreteIndexNames(
                state, IndicesOptions.lenientExpandOpen(), ident.indexNameOrAlias());
            // We need all concrete open indices, as closed indices must not appear in the routing.
            concreteOpenIndices = indexNameExpressionResolver.concreteIndexNames(
                state, IndicesOptions.fromOptions(true, true, true,
                    false, IndicesOptions.strictExpandOpenAndForbidClosed()), ident.indexNameOrAlias());
        } else {
            try {
                concreteIndices = indexNameExpressionResolver.concreteIndexNames(
                    state, IndicesOptions.strictExpandOpen(), ident.indexNameOrAlias());
                concreteOpenIndices = concreteIndices;
                if (concreteIndices.length == 0) {
                    // no matching index found
                    throw new RelationUnknown(ident);
                }
                docIndexMetadata = buildDocIndexMetadata(concreteIndices[0]);
            } catch (IndexNotFoundException ex) {
                throw new RelationUnknown(ident.fqn(), ex);
            }
        }
        return docIndexMetadata;
    }

    private DocIndexMetadata buildDocIndexMetadata(String indexName) {
        DocIndexMetadata docIndexMetadata;
        IndexMetadata indexMetadata = metadata.index(indexName);
        try {
            docIndexMetadata = new DocIndexMetadata(nodeCtx, indexMetadata, ident);
        } catch (IOException e) {
            throw new UnhandledServerException("Unable to build DocIndexMetadata", e);
        }
        try {
            return docIndexMetadata.build();
        } catch (Exception e) {
            try {
                LOGGER.error(
                    "Could not build DocIndexMetadata from: {}", indexMetadata.mapping().getSourceAsMap());
            } catch (Exception ignored) {
            }
            throw e;
        }
    }

    private DocIndexMetadata buildDocIndexMetadataFromTemplate(String index, String templateName) {
        IndexTemplateMetadata indexTemplateMetadata = metadata.getTemplates().get(templateName);
        DocIndexMetadata docIndexMetadata;
        try {
            IndexMetadata.Builder builder = new IndexMetadata.Builder(index);
            builder.putMapping(Constants.DEFAULT_MAPPING_TYPE,
                indexTemplateMetadata.getMappings().get(Constants.DEFAULT_MAPPING_TYPE).toString());

            Settings.Builder settingsBuilder = Settings.builder()
                .put(indexTemplateMetadata.settings())
                .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT);

            Settings settings = settingsBuilder.build();
            builder.settings(settings);
            builder.numberOfShards(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, 5));
            builder.numberOfReplicas(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, 1));
            docIndexMetadata = new DocIndexMetadata(nodeCtx, builder.build(), ident);
        } catch (IOException e) {
            throw new UnhandledServerException("Unable to build DocIndexMetadata from template", e);
        }
        return docIndexMetadata.build();
    }

    private List<PartitionName> buildPartitions(DocIndexMetadata md) {
        List<PartitionName> partitions = new ArrayList<>();
        if (md.partitionedBy().size() > 0) {
            for (String indexName : concreteIndices) {
                if (IndexParts.isPartitioned(indexName)) {
                    try {
                        PartitionName partitionName = PartitionName.fromIndexOrTemplate(indexName);
                        assert partitionName.relationName().equals(ident) : "ident must equal partitionName";
                        partitions.add(partitionName);
                    } catch (IllegalArgumentException e) {
                        // ignore
                        LOGGER.warn(String.format(Locale.ENGLISH, "Cannot build partition %s of index %s", indexName, ident.indexNameOrAlias()));
                    }
                }
            }
        }
        return partitions;
    }

    public DocTableInfo build() {
        DocIndexMetadata md = docIndexMetadata();
        List<PartitionName> partitions = buildPartitions(md);

        return new DocTableInfo(
            ident,
            md.columns(),
            md.partitionedByColumns(),
            md.generatedColumnReferences(),
            md.notNullColumns(),
            md.indices(),
            md.references(),
            md.analyzers(),
            md.primaryKey(),
            md.checkConstraints(),
            md.routingCol(),
            md.hasAutoGeneratedPrimaryKey(),
            concreteIndices,
            concreteOpenIndices,
            indexNameExpressionResolver,
            md.numberOfShards(), md.numberOfReplicas(),
            md.tableParameters(),
            md.partitionedBy(),
            partitions,
            md.columnPolicy(),
            md.versionCreated(),
            md.versionUpgraded(),
            md.isClosed(),
            md.supportedOperations());
    }
}


text:
/* This program is free software: you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public License
   as published by the Free Software Foundation, either version 3 of
   the License, or (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

otp.namespace("otp.modules.fieldtrip");

otp.modules.fieldtrip.FieldTripModule =
    otp.Class(otp.modules.multimodal.MultimodalPlannerModule, {

    moduleName  : "Field Trip Planner",

    itinWidget  : null,

    requestWidgets : null, // object, maps request ids to widget instances

    groupSize   : 100,
    bannedSegments : null,
    //itineraries : null,
    groupPlan : null,

    datastoreUrl : otp.config.datastoreUrl,

    geocoderWidgets : null,
    geocodedOrigins : null,
    geocodedDestinations : null,

    sessionManager : null,

    showIntermediateStops : true,

    initialize : function(webapp, id, options) {
        otp.modules.multimodal.MultimodalPlannerModule.prototype.initialize.apply(this, arguments);
        this.templateFiles.push('otp/modules/fieldtrip/fieldtrip-templates.html');

        this.requiresAuth = true;
        this.authUserRoles = ['fieldtrip', 'all'];

        this.planTripFunction = this.ftPlanTrip;
        this.requestWidgets = {};
        this.geocoderWidgets = {};
        this.geocodedOrigins = {};
        this.geocodedDestinations = {};
    },

    activate : function() {
        if(this.activated) return;
        console.log("activate "+this.id);
        otp.modules.multimodal.MultimodalPlannerModule.prototype.activate.apply(this);

        var modeSelector = this.optionsWidget.controls['mode'];
        modeSelector.addModeControl(new otp.widgets.tripoptions.GroupTripOptions(this.optionsWidget, "Group Size: "));
        modeSelector.refreshModeControls();

        // use app-wide session manager
        this.sessionManager = this.webapp.sessionManager;

        this.fieldTripManager = new otp.modules.fieldtrip.FieldTripManagerWidget('otp-'+this.id+'-fieldTripWidget', this);
        this.searchWidget = new otp.modules.fieldtrip.SearchWidget(this.id+'-searchWidget', this);

    },

    restore : function() {
        if(_.has(this.webapp.urlParams, 'groupSize')) {
            this.groupSize = parseInt(this.webapp.urlParams['groupSize']);
        }
        if("fromPlace" in this.webapp.urlParams && "toPlace" in this.webapp.urlParams) {
            this.optionsWidget.restorePlan({queryParams : this.webapp.urlParams});
        }
        otp.modules.multimodal.MultimodalPlannerModule.prototype.restore.apply(this);
    },

    getExtendedQueryParams : function() {
        return {
            numItineraries : 1,
        };
    },

    getAdditionalUrlParams : function() {
        return { groupSize : this.groupSize };
    },

    ftPlanTrip : function(queryParams) {

        if(this.updateActiveOnly) { // single itin modified by first/last/previous/next

            if(this.groupPlan) {
                var bannedTripArr = [];
                for(var i = 0; i < this.groupPlan.itineraries.length; i++) {
                    bannedTripArr = bannedTripArr.concat(this.groupPlan.itineraries[i].getTripIds());
                }
                queryParams.bannedTrips = bannedTripArr.join(',');
            }

            this.planTrip(queryParams);
            return;
        }

        var planDate = moment(this.optionsWidget.controls['time'].getDate()).format("MM/DD/YYYY");
        this.currentGroupSize = this.groupSize;
        this.bannedSegments = [];
        this.groupPlan = null;

        var this_ = this;

        // query for trips in use by other field trip itineraries in the DB
        $.ajax(this.datastoreUrl+'/fieldtrip/getTrips', {
            data: {
                sessionId : this.sessionManager.sessionId,
                date : planDate,
                //limit : 100,
            },

            success: function(data) {
                if((typeof data) == "string") data = jQuery.parseJSON(data);

                // store the trips in use for reference by the checkTripValidity() method
                this_.tripsInUse = [];

                for(var t = 0; t < data.length; t++) {
                    var fieldTrip = data[t];
                    for(var i = 0; i < fieldTrip.groupItineraries.length; i++) {
                        var grpItin = fieldTrip.groupItineraries[i];
                        for(var gt =0 ; gt < grpItin.trips.length; gt++) {
                            var gtfsTrip = grpItin.trips[gt];
                            // (gtfsTrip already includes fields agencyAndId, fromStopIndex, and toStopIndex)
                            gtfsTrip.passengers = grpItin.passengers;
                            this_.tripsInUse.push(gtfsTrip);
                        }
                    }
                }

                // kick off the main planTrip request
                this.itinCapacity = null;
                this_.planTrip();
            },

            error: function(data) {
                console.log("error retrieving trips");
                console.log(data);
            }
        });
        this.setBannedTrips();
    },

    processPlan : function(tripPlan, restoring) {
        if(this.updateActiveOnly) {
            var itinIndex = this.itinWidget.activeIndex;
            tripPlan.itineraries[0].groupSize = this.groupPlan.itineraries[itinIndex].groupSize;
            this.itinWidget.updateItineraries(tripPlan.itineraries);
            this.updateActiveOnly = false;
            this.drawItinerary(tripPlan.itineraries[0]);
            return;
        }

        if(this.groupPlan == null)
            this.groupPlan = new otp.modules.planner.TripPlan(null, _.extend(tripPlan.queryParams, { groupSize : this.groupSize }));

        if(this.itinWidget == null) this.createItinerariesWidget();

        var itin = tripPlan.itineraries[0];
        var capacity = itin.getGroupTripCapacity();

        // if this itin shares a vehicle trip with another one already in use, only use the remainingCapacity (as set in checkTripValidity())
        if(this.itinCapacity) capacity = Math.min(capacity, this.itinCapacity);

        this.groupPlan.addItinerary(itin);

        var transitLegs = itin.getTransitLegs();
        for(var i = 0; i < transitLegs.length; i++) {
            var leg = transitLegs[i];
            this.bannedSegments.push({
                agencyAndId : leg.agencyId + "_" + leg.tripId,
                fromStopIndex : leg.from.stopIndex,
                toStopIndex : leg.to.stopIndex,
            });
        }

        this.setBannedTrips();

        if(this.currentGroupSize > capacity) { // group members remain; plan another trip
            this.currentGroupSize -= capacity;
            itin.groupSize = capacity;
            //console.log("remaining: "+this.currentGroupSize);
            this.itinCapacity = null;
            this.planTrip();
        }
        else { // we're done; show the results
            itin.groupSize = this.currentGroupSize;
            this.showResults();
        }

    },

    showResults : function() {
        this.itinWidget.show();
        this.itinWidget.bringToFront();
        this.itinWidget.updatePlan(this.groupPlan);
        this.drawItinerary(this.groupPlan.itineraries[0]);

        var requestWidgets = _.values(this.requestWidgets);
        for(var i = 0; i <= requestWidgets.length; i++) {
            if(requestWidgets[i]) {
                requestWidgets[i].tripPlanned();
            }
        }
    },

    createItinerariesWidget : function() {
        this.itinWidget = new otp.widgets.ItinerariesWidget(this.id+"-itinWidget", this);
        this.itinWidget.showItineraryLink = false;
    },

    setBannedTrips : function() {
        var tripIds = [];
        for(var i=0; i<this.bannedSegments.length; i++) {
            tripIds.push(this.bannedSegments[i].agencyAndId);
        }

        this.bannedTrips = tripIds.length > 0 ? tripIds.join(',') : null;
        //console.log("set bannedTrips: "+this.bannedTrips);
    },

    refreshTrips : function(date) {
        var this_ = this;
        $.ajax(this.datastoreUrl+'/fieldtrip/getTrips', {
            data: {
                sessionId : this.sessionManager.sessionId,
                //date : this.fieldTripManager.selectedDate,
                //limit : 100,
            },

            success: function(data) {
                if((typeof data) == "string") data = jQuery.parseJSON(data);
                //this_.fieldTripManager.updateTrips(data);
            },

            error: function(data) {
                console.log("error retrieving trips");
                console.log(data);
            }
        });
    },

    showSaveWidget : function() {
        new otp.modules.fieldtrip.SaveFieldTripWidget('otp-'+this.id+'-saveFieldTripWidget', this);
    },

    saveTrip : function(request, requestOrder, successCallback) {
        var this_ = this;

        var data = {
            sessionId : this.sessionManager.sessionId,
            requestId : request.id,
            'trip.requestOrder' : requestOrder,
            'trip.origin' : this.getStartOTPString(),
            'trip.destination' : this.getEndOTPString(),
            'trip.createdBy' : this.userName,
            'trip.passengers' : this.groupSize,
            'trip.departure' : moment(this.groupPlan.earliestStartTime).add("hours", otp.config.timeOffset).format("YYYY-MM-DDTHH:mm:ss"),
            'trip.queryParams' : JSON.stringify(this.groupPlan.queryParams),
        };

        for(var i = 0; i < this.groupPlan.itineraries.length; i++) {
            var itin = this.groupPlan.itineraries[i];
            //console.log("saving itin for trip "+tripId);
            //console.log(itin.itinData);

            data['itins['+i+'].passengers'] = itin.groupSize;
            data['itins['+i+'].itinData'] = otp.util.Text.lzwEncode(JSON.stringify(itin.itinData));
            data['itins['+i+'].timeOffset'] = otp.config.timeOffset || 0;

            var legs = itin.getTransitLegs();

            for(var l = 0; l < legs.length; l++) {
                var leg = legs[l];
                var routeName = (leg.routeShortName !== null ? ('(' + leg.routeShortName + ') ') : '') + (leg.routeLongName || "");

                data['gtfsTrips['+i+']['+l+'].depart'] = moment(leg.startTime).format("HH:mm:ss");
                data['gtfsTrips['+i+']['+l+'].arrive'] = moment(leg.endTime).format("HH:mm:ss");
                data['gtfsTrips['+i+']['+l+'].agencyAndId'] = leg.agencyId + "_" + leg.tripId;
                data['gtfsTrips['+i+']['+l+'].routeName'] = routeName;
                data['gtfsTrips['+i+']['+l+'].fromStopIndex'] = leg.from.stopIndex;
                data['gtfsTrips['+i+']['+l+'].toStopIndex'] = leg.to.stopIndex;
                data['gtfsTrips['+i+']['+l+'].fromStopName'] = leg.from.name;
                data['gtfsTrips['+i+']['+l+'].toStopName'] = leg.to.name;
                data['gtfsTrips['+i+']['+l+'].headsign'] = leg.headsign;
                data['gtfsTrips['+i+']['+l+'].capacity'] = itin.getModeCapacity(leg.mode);
                if(leg.tripBlockId) data['gtfsTrips['+i+']['+l+'].blockId'] = leg.tripBlockId;
            }
        }

        //console.log(data);
        $.ajax(this.datastoreUrl+'/fieldtrip/newTrip', {
            type: 'POST',

            data: data,

            success: function(data) {
                if((typeof data) == "string") data = jQuery.parseJSON(data);
                if(data === -1) {
                    otp.widgets.Dialogs.showOkDialog("This plan could not be saved due to a lack of capacity on one or more vehicles. Please re-plan your trip.", "Cannot Save Plan");
                }
                else successCallback.call(this_, data);
            },

            error: function(data) {
                console.log("error saving trip");
                console.log(data);
            }
        });
    },

    checkTripValidity : function(tripId, leg, itin) {
        var capacityInUse = 0;
        for(var i = 0; i < this.tripsInUse.length; i++) {
            var tripInUse  = this.tripsInUse[i];

            // first test: are these the same vehicle trip? if not, we're ok
            if(tripId !== tripInUse.agencyAndId) continue;

            // second test: do the stop ranges overlap? if not, we're ok
            if(leg.from.stopIndex > tripInUse.toStopIndex || leg.to.stopIndex < tripInUse.fromStopIndex) continue;

            // if ranges overlap, calculate remaining capacity
            capacityInUse += tripInUse.passengers;
        }

        var remainingCapacity = itin.getModeCapacity(leg.mode) - capacityInUse;
        if(remainingCapacity < 10) return false; // consider trip 'full' if < 10 spots remain
        this.itinCapacity = this.itinCapacity ? Math.min(this.itinCapacity, remainingCapacity) : remainingCapacity;

        return true;
    },

    deleteTrip : function(trip) {
        var this_ = this;

        var data = {
                sessionId : this.sessionManager.sessionId,
                id : trip.id
        };

        console.log("delete");
        console.log(data);
        $.ajax(this.datastoreUrl+'/fieldtrip/deleteTrip', {
            type: 'POST',

            data: data,

            success: function(data) {
                this_.refreshTrips();
            },

            error: function(data) {
                console.log("error deleting trip:");
                console.log(data);
            }
        });
    },

    renderTripFromId : function(tripId) {
        var this_ = this;
        $.ajax(this.datastoreUrl+'/fieldTrip', {
            data: {
                sessionId : this.sessionManager.sessionId,
                id : trip.id
            },

            success: function(data) {
                if((typeof data) == "string") data = jQuery.parseJSON(data);
                this_.renderTrip(data);
            },

            error: function(data) {
                console.log("error retrieving trip "+trip.id);
                console.log(data);
            }
        });

    },

    renderTrip : function(tripData) {
        var queryParams = JSON.parse(tripData.queryParams);
        this.groupPlan = new otp.modules.planner.TripPlan(null, queryParams);//_.extend(queryParams, { groupSize : this.groupSize }));
        for(var i = 0; i < tripData.groupItineraries.length; i++) {
            var itinData = otp.util.FieldTrip.readItinData(tripData.groupItineraries[i]);
            var itin = new otp.modules.planner.Itinerary(itinData, this.groupPlan);
            itin.groupSize = tripData.groupItineraries[i].passengers;
            this.groupPlan.addItinerary(itin);
        }
        if(this.itinWidget == null) this.createItinerariesWidget();
        this.showResults();
        this.restoreMarkers(queryParams);
        this.optionsWidget.restorePlan({ queryParams : queryParams });
    },

    //** requests functions **//

    loadRequests : function() {
        var this_ = this;
        $.ajax(this.datastoreUrl+'/fieldtrip/getRequests', {
            data: {
                sessionId : this.sessionManager.sessionId,
                //limit : 100,
            },

            success: function(data) {
                if((typeof data) == "string") data = jQuery.parseJSON(data);
                this_.fieldTripManager.updateRequests(data);

                for(var i = 0; i < data.length; i++) {
                    var req = data[i];
                    if(_.has(this_.requestWidgets, req.id)) {
                        var widget = this_.requestWidgets[req.id];
                        widget.request = req;
                        widget.render();
                    }
                }
            },

            error: function(data) {
                console.log("error retrieving requests");
                console.log(data);
            }
        });
    },

    setRequestStatus : function(request, status) {
        var this_ = this;

        var data = {
            sessionId : this.sessionManager.sessionId,
            requestId : request.id,
            status : status
        };

        $.ajax(this.datastoreUrl+'/fieldtrip/setRequestStatus', {
            type: 'POST',

            data: data,

            success: function(data) {
                this_.loadRequests();
            },

            error: function(data) {
                console.log("error setting trip status:");
                console.log(data);
            }
        });
    },

    cancelRequest : function(request) {
        this.setRequestStatus(request, "cancelled");
    },

    setClasspassId : function(request, classpassId) {
        var this_ = this;

        $.ajax(this.datastoreUrl+'/fieldtrip/setRequestClasspassId', {
            type: 'POST',

            data: {
                sessionId : this.sessionManager.sessionId,
                requestId : request.id,
                classpassId : classpassId
            },

            success: function(data) {
                this_.loadRequests();
            },

            error: function(data) {
                console.log("error setting classpass id:");
                console.log(data);
            }
        });
    },

    showRequest : function(request) {
        if(_.has(this.requestWidgets, request.id)) {
            var widget = this.requestWidgets[request.id];
            if(widget.minimized) widget.unminimize();
        }
        else {
            var widget = new otp.modules.fieldtrip.FieldTripRequestWidget('otp-'+this.id+'-requestWidget-'+request.id, this, request);
            this.requestWidgets[request.id] = widget;
        }
        widget.bringToFront();
    },

    planOutbound : function(request) {
        this.clearTrip();
        var queryParams = {
            time : moment(request.arriveDestinationTime).format("h:mma"),
            date : moment(request.travelDate).format("MM-DD-YYYY"),
            arriveBy : true,
            groupSize : otp.util.FieldTrip.getGroupSize(request),
        };

        this.optionsWidget.applyQueryParams(queryParams);

        var geocodedOrigin = this.geocodedOrigins[request.id];
        var geocodedDestination = this.geocodedDestinations[request.id];

        if(geocodedOrigin) {
            this.setStartPoint(new L.LatLng(geocodedOrigin.lat, geocodedOrigin.lng),
                               false, geocodedOrigin.description);
        }
        if(geocodedDestination) {
            this.setEndPoint(new L.LatLng(geocodedDestination.lat, geocodedDestination.lng),
                             false, geocodedDestination.description);
        }
        if(!geocodedOrigin || !geocodedDestination) {
            this.showGeocoder(request, "outbound");
        }
    },

    planInbound : function(request) {
        this.clearTrip();
        var queryParams = {
            time : moment(request.leaveDestinationTime).format("h:mma"),
            date : moment(request.travelDate).format("MM-DD-YYYY"),
            arriveBy : false,
            groupSize : otp.util.FieldTrip.getGroupSize(request),
        };

        this.optionsWidget.applyQueryParams(queryParams);

        var geocodedOrigin = this.geocodedOrigins[request.id];
        var geocodedDestination = this.geocodedDestinations[request.id];

        if(geocodedOrigin) {
            this.setEndPoint(new L.LatLng(geocodedOrigin.lat, geocodedOrigin.lng),
                             false, geocodedOrigin.description);
        }
        if(geocodedDestination) {
            this.setStartPoint(new L.LatLng(geocodedDestination.lat, geocodedDestination.lng),
                               false, geocodedDestination.description);
        }
        if(!geocodedOrigin || !geocodedDestination) {
            this.showGeocoder(request, "inbound");
        }
    },

    showGeocoder : function(request, mode) {
        if(_.has(this.geocoderWidgets, request.id)) {
            var widget = this.geocoderWidgets[request.id];
            widget.mode = mode;
            if(widget.minimized) widget.unminimize();
            widget.show();
        }
        else {
            var widget = new otp.modules.fieldtrip.FieldTripGeocoderWidget('otp-'+this.id+'-geocoderWidget-'+request.id, this, request, mode);
            this.geocoderWidgets[request.id] = widget;
        }
        widget.bringToFront();

    },

    saveRequestTrip : function(request, type) {
        if(!this.checkPlanValidity(request)) return;

        this.itinWidget.close();

        var this_ = this;

        var outboundTrip = otp.util.FieldTrip.getOutboundTrip(request);
        if(type === 'outbound') {
            if(outboundTrip) {
                var msg = "This action will overwrite a previously planned outbound itinerary for this request. Do you wish to continue?";
                otp.widgets.Dialogs.showYesNoDialog(msg, "Overwrite Outbound Itinerary?", function() {
                    this_.saveTrip(request, 0, function(tripId) {
                        //this_.deleteTrip(outboundTrip);
                        this_.loadRequests();
                    });
                });
            }
            else {
                this_.saveTrip(request, 0, function(tripId) {
                    this_.loadRequests();
                });
            }
        }

        var inboundTrip = otp.util.FieldTrip.getInboundTrip(request);
        if(type === 'inbound') {
            if(inboundTrip) {
                var msg = "This action will overwrite a previously planned inbound itinerary for this request. Do you wish to continue?";
                otp.widgets.Dialogs.showYesNoDialog(msg, "Overwrite Inbound Itinerary?", function() {
                    this_.saveTrip(request, 1, function(tripId) {
                        //this_.deleteTrip(inboundTrip);
                        this_.loadRequests();
                    });
                });
            }
            else {
                this_.saveTrip(request, 1, function(tripId) {
                    this_.loadRequests();
                });
            }
        }
    },

    checkPlanValidity : function(request) {
        if(this.groupPlan == null) {
            otp.widgets.Dialogs.showOkDialog("No active plan to save", "Cannot Save Plan");
            return false;
        }

        var planDeparture = moment(this.groupPlan.earliestStartTime).add("hours", otp.config.timeOffset);
        var requestDate = moment(request.travelDate);

        if(planDeparture.date() != requestDate.date() ||
                planDeparture.month() != requestDate.month() ||
                planDeparture.year() != requestDate.year()) {
            var msg = "Planned trip date (" + planDeparture.format("MM/DD/YYYY") + ") is not the requested day of travel (" + requestDate.format("MM/DD/YYYY") + ")";
            otp.widgets.Dialogs.showOkDialog(msg, "Cannot Save Plan");
            return false;
        }

        return true;
    },

    addNote: function(request, note, type) {
        var this_ = this;

        $.ajax(this.datastoreUrl+'/fieldtrip/addNote', {
            type: 'POST',

            data: {
                sessionId : this.sessionManager.sessionId,
                requestId : request.id,
                'note.note' : note,
                'note.type' : type,
                'note.userName' : this.sessionManager.username,
            },

            success: function(data) {
                this_.loadRequests();
            },

            error: function(data) {
                console.log("error adding note:");
                console.log(data);
            }
        });
    },

    deleteNote: function(note) {
        var this_ = this;

        $.ajax(this.datastoreUrl+'/fieldtrip/deleteNote', {
            type: 'POST',

            data: {
                sessionId : this.sessionManager.sessionId,
                noteId : note.id,
            },

            success: function(data) {
                this_.loadRequests();
            },

            error: function(data) {
                console.log("error deleting note:");
                console.log(data);
            }
        });
    },

    setRequestDate: function(request, date) {
        var this_ = this;

        $.ajax(this.datastoreUrl+'/fieldtrip/setRequestDate', {
            type: 'POST',

            data: {
                sessionId : this.sessionManager.sessionId,
                requestId : request.id,
                date: moment(date).format("MM/DD/YYYY")
            },

            success: function(data) {
                this_.loadRequests();
            },

            error: function(data) {
                console.log("error updating request date");
                console.log(data);
            }
        });
    },

});

otp.util.FieldTrip = {

    constructPlanInfo : function(trip) {
        return trip.groupItineraries.length + " group itineraries, planned by " + trip.createdBy + " at " + trip.timeStamp;
    },

    getOutboundTrip : function(request) {
        if(request.outboundTrip) return request.outboundTrip;
        return otp.util.FieldTrip.tripAtIndex(request, 0);
    },

    getInboundTrip : function(request) {
        if(request.inboundTrip) return request.inboundTrip;
        return otp.util.FieldTrip.tripAtIndex(request, 1);
    },

    tripAtIndex : function(request, index) {
        if(request.trips) {
            for(var i = 0; i < request.trips.length; i++) {
                if(request.trips[i].requestOrder === index) return request.trips[i];
            }
        }
        return null;
    },

    getGroupSize : function(request) {
        var groupSize = 0;
        if(request.numStudents) groupSize += request.numStudents;
        if(request.numChaperones) groupSize += request.numChaperones;
        return groupSize;
    },

    readItinData : function(groupItin) {
        if(groupItin.timeOffset) otp.config.timeOffset = groupItin.timeOffset;
        return JSON.parse(otp.util.Text.lzwDecode(groupItin.itinData));
    },

    getRequestContext : function(req) {
        var context = _.clone(req);
        if(req.travelDate) context.formattedDate = moment(req.travelDate).format("MMM Do YYYY");
        var outboundTrip = otp.util.FieldTrip.getOutboundTrip(req);
        if(outboundTrip) context.outboundDesc = otp.util.FieldTrip.constructPlanInfo(outboundTrip);
        var inboundTrip = otp.util.FieldTrip.getInboundTrip(req);
        if(inboundTrip) context.inboundDesc = otp.util.FieldTrip.constructPlanInfo(inboundTrip);
        return context;
    }

};