<|start_filename|>internal/pkg/store/stores.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/pkg/db"
	"github.com/lf-edge/ekuiper/internal/pkg/ts"
	kv2 "github.com/lf-edge/ekuiper/pkg/kv"
	"sync"
)

type stores struct {
	kv        map[string]kv2.KeyValue
	ts        map[string]kv2.Tskv
	mu        sync.Mutex
	kvBuilder Builder
	tsBuilder ts.Builder
}

func newStores(db db.Database) (error, *stores) {
	var err error
	var kvBuilder Builder
	var tsBuilder ts.Builder
	kvBuilder, err = CreateStoreBuilder(db)
	if err != nil {
		return err, nil
	}
	err, tsBuilder = ts.CreateTsBuilder(db)
	if err != nil {
		return err, nil
	}
	return nil, &stores{
		kv:        make(map[string]kv2.KeyValue),
		ts:        make(map[string]kv2.Tskv),
		mu:        sync.Mutex{},
		kvBuilder: kvBuilder,
		tsBuilder: tsBuilder,
	}
}

func (s *stores) GetKV(table string) (error, kv2.KeyValue) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if ks, contains := s.kv[table]; contains {
		return nil, ks
	}
	ks, err := s.kvBuilder.CreateStore(table)
	if err != nil {
		return err, nil
	}
	s.kv[table] = ks
	return nil, ks
}

func (s *stores) GetTS(table string) (error, kv2.Tskv) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if tts, contains := s.ts[table]; contains {
		return nil, tts
	}
	err, tts := s.tsBuilder.CreateTs(table)
	if err != nil {
		return err, nil
	}
	s.ts[table] = tts
	return nil, tts
}

var globalStores *stores = nil

func InitGlobalStores(db db.Database) error {
	var err error
	err, globalStores = newStores(db)
	return err
}

func GetKV(table string) (error, kv2.KeyValue) {
	if globalStores == nil {
		return fmt.Errorf("global stores are not initialized"), nil
	}
	return globalStores.GetKV(table)
}

func GetTS(table string) (error, kv2.Tskv) {
	if globalStores == nil {
		return fmt.Errorf("global stores are not initialized"), nil
	}
	return globalStores.GetTS(table)
}


<|start_filename|>internal/topo/node/join_align_node.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/xsql"
	"github.com/lf-edge/ekuiper/pkg/api"
)

// JoinAlignNode will block the stream and buffer all the table tuples. Once buffered, it will combine the later input with the buffer
// The input for batch table MUST be *WindowTuples
type JoinAlignNode struct {
	*defaultSinkNode
	statManager StatManager
	emitters    map[string]int
	// states
	batch *xsql.WindowTuplesSet
}

const BatchKey = "$$batchInputs"

func NewJoinAlignNode(name string, emitters []string, options *api.RuleOption) (*JoinAlignNode, error) {
	emap := make(map[string]int, len(emitters))
	for i, e := range emitters {
		emap[e] = i
	}
	n := &JoinAlignNode{
		emitters: emap,
	}
	n.defaultSinkNode = &defaultSinkNode{
		input: make(chan interface{}, options.BufferLength),
		defaultNode: &defaultNode{
			outputs:   make(map[string]chan<- interface{}),
			name:      name,
			sendError: options.SendError,
		},
	}
	return n, nil
}

func (n *JoinAlignNode) Exec(ctx api.StreamContext, errCh chan<- error) {
	n.ctx = ctx
	log := ctx.GetLogger()
	log.Debugf("JoinAlignNode %s is started", n.name)

	if len(n.outputs) <= 0 {
		go func() { errCh <- fmt.Errorf("no output channel found") }()
		return
	}
	stats, err := NewStatManager("op", ctx)
	if err != nil {
		go func() { errCh <- err }()
		return
	}
	n.statManager = stats
	go func() {
		// restore batch state
		if s, err := ctx.GetState(BatchKey); err == nil {
			switch st := s.(type) {
			case []xsql.WindowTuples:
				if len(st) == len(n.emitters) {
					n.batch = &xsql.WindowTuplesSet{Content: st}
					log.Infof("Restore batch state %+v", st)
				} else {
					log.Warnf("Restore batch state got different emitter length so discarded: %+v", st)
				}
			case nil:
				log.Debugf("Restore batch state, nothing")
			default:
				errCh <- fmt.Errorf("restore batch state %v error, invalid type", st)
			}
		} else {
			log.Warnf("Restore batch state fails: %s", err)
		}
		if n.batch == nil {
			n.batch = &xsql.WindowTuplesSet{
				Content: make([]xsql.WindowTuples, len(n.emitters)),
			}
		}

		for {
			log.Debugf("JoinAlignNode %s is looping", n.name)
			select {
			// process incoming item from both streams(transformed) and tables
			case item, opened := <-n.input:
				processed := false
				if item, processed = n.preprocess(item); processed {
					break
				}
				n.statManager.IncTotalRecordsIn()
				n.statManager.ProcessTimeStart()
				if !opened {
					n.statManager.IncTotalExceptions()
					break
				}
				switch d := item.(type) {
				case error:
					n.Broadcast(d)
					n.statManager.IncTotalExceptions()
				case *xsql.Tuple:
					log.Debugf("JoinAlignNode receive tuple input %s", d)
					temp := xsql.WindowTuplesSet{
						Content: make([]xsql.WindowTuples, 0),
					}
					temp = temp.AddTuple(d)
					n.alignBatch(ctx, temp)
				case xsql.WindowTuplesSet:
					log.Debugf("JoinAlignNode receive window input %s", d)
					n.alignBatch(ctx, d)
				case xsql.WindowTuples: // batch input
					log.Debugf("JoinAlignNode receive batch source %s", d)
					// Buffer and update batch inputs
					index, ok := n.emitters[d.Emitter]
					if !ok {
						n.Broadcast(fmt.Errorf("run JoinAlignNode error: receive batch input from unknown emitter %[1]T(%[1]v)", d))
						n.statManager.IncTotalExceptions()
					}
					if n.batch != nil && len(n.batch.Content) > index {
						n.batch.Content[index] = d
						ctx.PutState(BatchKey, n.batch)
					} else {
						log.Errorf("Invalid index %d for batch %v", index, n.batch)
					}
				default:
					n.Broadcast(fmt.Errorf("run JoinAlignNode error: invalid input type but got %[1]T(%[1]v)", d))
					n.statManager.IncTotalExceptions()
				}
			case <-ctx.Done():
				log.Infoln("Cancelling join align node....")
				return
			}
		}
	}()
}

func (n *JoinAlignNode) alignBatch(_ api.StreamContext, w xsql.WindowTuplesSet) {
	n.statManager.ProcessTimeStart()
	w.Content = append(w.Content, n.batch.Content...)
	n.Broadcast(w)
	n.statManager.ProcessTimeEnd()
	n.statManager.IncTotalRecordsOut()
	n.statManager.SetBufferLength(int64(len(n.input)))
}

func (n *JoinAlignNode) GetMetrics() [][]interface{} {
	if n.statManager != nil {
		return [][]interface{}{
			n.statManager.GetMetrics(),
		}
	} else {
		return nil
	}
}


<|start_filename|>internal/pkg/ts/sql/sqlTs.go<|end_filename|>
// Copyright 2021 INTECH Process Automation Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"bytes"
	"database/sql"
	"encoding/gob"
	"fmt"
	dbSql "github.com/lf-edge/ekuiper/internal/pkg/db/sql"
	kvEncoding "github.com/lf-edge/ekuiper/internal/pkg/store/encoding"
)

type ts struct {
	database dbSql.Database
	table    string
	last     int64
}

func init() {
	gob.Register(make(map[string]interface{}))
}

func createSqlTs(database dbSql.Database, table string) (error, *ts) {
	store := &ts{
		database: database,
		table:    table,
		last:     getLast(database, table),
	}
	err := store.database.Apply(func(db *sql.DB) error {
		query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS '%s'('key' INTEGER PRIMARY KEY, 'val' BLOB);", table)
		_, err := db.Exec(query)
		return err
	})
	if err != nil {
		return err, nil
	}
	return nil, store
}

func (t *ts) Set(key int64, value interface{}) (bool, error) {
	if key <= t.last {
		return false, nil
	}
	err, b := kvEncoding.Encode(value)
	if err != nil {
		return false, err
	}
	err = t.database.Apply(func(db *sql.DB) error {
		query := fmt.Sprintf("INSERT INTO %s(key,val) values(?,?);", t.table)
		stmt, err := db.Prepare(query)
		if err != nil {
			return err
		}
		defer stmt.Close()
		_, err = stmt.Exec(key, b)
		if err != nil {
			return err
		}
		t.last = key
		return nil
	})
	if err != nil {
		return false, err
	}
	return true, nil
}

func (t ts) Get(key int64, value interface{}) (bool, error) {
	result := false
	err := t.database.Apply(func(db *sql.DB) error {
		query := fmt.Sprintf("SELECT val FROM %s WHERE key=%d;", t.table, key)
		row := db.QueryRow(query)
		var tmp []byte
		switch err := row.Scan(&tmp); err {
		case sql.ErrNoRows:
			return nil
		case nil:
		default:
			return err
		}

		dec := gob.NewDecoder(bytes.NewBuffer(tmp))
		if err := dec.Decode(value); err != nil {
			return err
		}
		result = true
		return nil
	})
	if err != nil {
		return false, err
	}
	return result, nil
}

func (t ts) Last(value interface{}) (int64, error) {
	_, err := t.Get(t.last, value)
	if err != nil {
		return 0, err
	}
	return t.last, nil
}

func (t ts) Delete(key int64) error {
	return t.database.Apply(func(db *sql.DB) error {
		query := fmt.Sprintf("DELETE FROM %s WHERE key=%d;", t.table, key)
		_, err := db.Exec(query)
		return err
	})
}

func (t ts) DeleteBefore(key int64) error {
	return t.database.Apply(func(db *sql.DB) error {
		query := fmt.Sprintf("DELETE FROM %s WHERE key<%d;", t.table, key)
		_, err := db.Exec(query)
		return err
	})
}

func (t ts) Close() error {
	return nil
}

func (t ts) Drop() error {
	return t.database.Apply(func(db *sql.DB) error {
		query := fmt.Sprintf("Drop table %s;", t.table)
		_, err := db.Exec(query)
		return err
	})
}

func getLast(d dbSql.Database, table string) int64 {
	var last int64 = 0
	_ = d.Apply(func(db *sql.DB) error {
		query := fmt.Sprintf("SELECT key FROM %s Order by key DESC Limit 1;", table)
		row := db.QueryRow(query)
		err := row.Scan(&last)
		return err
	})
	return last
}


<|start_filename|>pkg/ast/visitor.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ast

import (
	"reflect"
)

type Visitor interface {
	Visit(Node) bool
}

func Walk(v Visitor, node Node) {
	if node == nil || reflect.ValueOf(node).IsNil() {
		return
	}

	if !v.Visit(node) {
		return
	}

	switch n := node.(type) {
	case *SelectStatement:
		Walk(v, n.Fields)
		Walk(v, n.Sources)
		Walk(v, n.Joins)
		Walk(v, n.Condition)
		Walk(v, n.Dimensions)
		Walk(v, n.Having)
		Walk(v, n.SortFields)

	case Fields:
		for _, f := range n {
			Walk(v, &f)
		}

	case *Field:
		Walk(v, n.Expr)
		if fr, ok := n.Expr.(*FieldRef); ok && fr.IsAlias() {
			Walk(v, fr.Expression)
		}

	case Sources:
		for _, s := range n {
			Walk(v, s)
		}

	//case *Table:

	case Joins:
		for _, s := range n {
			Walk(v, &s)
		}

	case *Join:
		Walk(v, n.Expr)

	case Dimensions:
		Walk(v, n.GetWindow())
		for _, dimension := range n.GetGroups() {
			Walk(v, dimension.Expr)
		}

	case *Window:
		Walk(v, n.Length)
		Walk(v, n.Interval)
		Walk(v, n.Filter)

	case SortFields:
		for _, sf := range n {
			Walk(v, &sf)
		}

	//case *SortField:

	case *BinaryExpr:
		Walk(v, n.LHS)
		Walk(v, n.RHS)

	case *Call:
		for _, expr := range n.Args {
			Walk(v, expr)
		}

	case *ParenExpr:
		Walk(v, n.Expr)

	case *CaseExpr:
		Walk(v, n.Value)
		for _, w := range n.WhenClauses {
			Walk(v, w)
		}
		Walk(v, n.ElseClause)

	case *WhenClause:
		Walk(v, n.Expr)
		Walk(v, n.Result)

	case *IndexExpr:
		Walk(v, n.Index)
	}
}

// WalkFunc traverses a node hierarchy in depth-first order.
func WalkFunc(node Node, fn func(Node) bool) {
	Walk(walkFuncVisitor(fn), node)
}

type walkFuncVisitor func(Node) bool

func (fn walkFuncVisitor) Visit(n Node) bool { return fn(n) }


<|start_filename|>internal/server/middleware/auth.go<|end_filename|>
package middleware

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/pkg/jwt"
	"net/http"
)

var notAuth = []string{"/", "/ping"}

var Auth = func(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		requestPath := r.URL.Path
		for _, value := range notAuth {
			if value == requestPath {
				next.ServeHTTP(w, r)
				return
			}
		}

		tokenHeader := r.Header.Get("Authorization")

		if tokenHeader == "" {
			http.Error(w, "missing_token", http.StatusUnauthorized)
			return
		}
		tk, err := jwt.ParseToken(tokenHeader)
		if err != nil {
			http.Error(w, err.Error(), http.StatusUnauthorized)
			return
		}
		if tk.StandardClaims.Audience != "eKuiper" {
			http.Error(w, fmt.Sprintf("audience field should be eKuiper, but got %s", tk.StandardClaims.Audience), http.StatusUnauthorized)
			return
		}

		next.ServeHTTP(w, r)
	})
}


<|start_filename|>internal/service/test/schemas/helloworld/hw.pb.go<|end_filename|>
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// 	protoc-gen-go v1.26.0-devel
// 	protoc        v3.6.1
// source: hw.proto

package helloworld

import (
	empty "github.com/golang/protobuf/ptypes/empty"
	wrappers "github.com/golang/protobuf/ptypes/wrappers"
	_ "google.golang.org/genproto/googleapis/api/annotations"
	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
	reflect "reflect"
	sync "sync"
)

const (
	// Verify that this generated code is sufficiently up-to-date.
	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
	// Verify that runtime/protoimpl is sufficiently up-to-date.
	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)

// The request message containing the user's name.
type HelloRequest struct {
	state         protoimpl.MessageState
	sizeCache     protoimpl.SizeCache
	unknownFields protoimpl.UnknownFields

	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
}

func (x *HelloRequest) Reset() {
	*x = HelloRequest{}
	if protoimpl.UnsafeEnabled {
		mi := &file_hw_proto_msgTypes[0]
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		ms.StoreMessageInfo(mi)
	}
}

func (x *HelloRequest) String() string {
	return protoimpl.X.MessageStringOf(x)
}

func (*HelloRequest) ProtoMessage() {}

func (x *HelloRequest) ProtoReflect() protoreflect.Message {
	mi := &file_hw_proto_msgTypes[0]
	if protoimpl.UnsafeEnabled && x != nil {
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		if ms.LoadMessageInfo() == nil {
			ms.StoreMessageInfo(mi)
		}
		return ms
	}
	return mi.MessageOf(x)
}

// Deprecated: Use HelloRequest.ProtoReflect.Descriptor instead.
func (*HelloRequest) Descriptor() ([]byte, []int) {
	return file_hw_proto_rawDescGZIP(), []int{0}
}

func (x *HelloRequest) GetName() string {
	if x != nil {
		return x.Name
	}
	return ""
}

// The response message containing the greetings
type HelloReply struct {
	state         protoimpl.MessageState
	sizeCache     protoimpl.SizeCache
	unknownFields protoimpl.UnknownFields

	Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
}

func (x *HelloReply) Reset() {
	*x = HelloReply{}
	if protoimpl.UnsafeEnabled {
		mi := &file_hw_proto_msgTypes[1]
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		ms.StoreMessageInfo(mi)
	}
}

func (x *HelloReply) String() string {
	return protoimpl.X.MessageStringOf(x)
}

func (*HelloReply) ProtoMessage() {}

func (x *HelloReply) ProtoReflect() protoreflect.Message {
	mi := &file_hw_proto_msgTypes[1]
	if protoimpl.UnsafeEnabled && x != nil {
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		if ms.LoadMessageInfo() == nil {
			ms.StoreMessageInfo(mi)
		}
		return ms
	}
	return mi.MessageOf(x)
}

// Deprecated: Use HelloReply.ProtoReflect.Descriptor instead.
func (*HelloReply) Descriptor() ([]byte, []int) {
	return file_hw_proto_rawDescGZIP(), []int{1}
}

func (x *HelloReply) GetMessage() string {
	if x != nil {
		return x.Message
	}
	return ""
}

type Response struct {
	state         protoimpl.MessageState
	sizeCache     protoimpl.SizeCache
	unknownFields protoimpl.UnknownFields

	Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
	Msg  string `protobuf:"bytes,2,opt,name=msg,proto3" json:"msg,omitempty"`
}

func (x *Response) Reset() {
	*x = Response{}
	if protoimpl.UnsafeEnabled {
		mi := &file_hw_proto_msgTypes[2]
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		ms.StoreMessageInfo(mi)
	}
}

func (x *Response) String() string {
	return protoimpl.X.MessageStringOf(x)
}

func (*Response) ProtoMessage() {}

func (x *Response) ProtoReflect() protoreflect.Message {
	mi := &file_hw_proto_msgTypes[2]
	if protoimpl.UnsafeEnabled && x != nil {
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		if ms.LoadMessageInfo() == nil {
			ms.StoreMessageInfo(mi)
		}
		return ms
	}
	return mi.MessageOf(x)
}

// Deprecated: Use Response.ProtoReflect.Descriptor instead.
func (*Response) Descriptor() ([]byte, []int) {
	return file_hw_proto_rawDescGZIP(), []int{2}
}

func (x *Response) GetCode() uint32 {
	if x != nil {
		return x.Code
	}
	return 0
}

func (x *Response) GetMsg() string {
	if x != nil {
		return x.Msg
	}
	return ""
}

type InferRequest struct {
	state         protoimpl.MessageState
	sizeCache     protoimpl.SizeCache
	unknownFields protoimpl.UnknownFields

	Rid    string `protobuf:"bytes,1,opt,name=rid,proto3" json:"rid,omitempty"`
	Uuid   string `protobuf:"bytes,2,opt,name=uuid,proto3" json:"uuid,omitempty"`
	Outlet string `protobuf:"bytes,3,opt,name=outlet,proto3" json:"outlet,omitempty"`
	Path   string `protobuf:"bytes,4,opt,name=path,proto3" json:"path,omitempty"`
	Data   []byte `protobuf:"bytes,5,opt,name=data,proto3" json:"data,omitempty"`
	Extra  string `protobuf:"bytes,6,opt,name=extra,proto3" json:"extra,omitempty"`
}

func (x *InferRequest) Reset() {
	*x = InferRequest{}
	if protoimpl.UnsafeEnabled {
		mi := &file_hw_proto_msgTypes[3]
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		ms.StoreMessageInfo(mi)
	}
}

func (x *InferRequest) String() string {
	return protoimpl.X.MessageStringOf(x)
}

func (*InferRequest) ProtoMessage() {}

func (x *InferRequest) ProtoReflect() protoreflect.Message {
	mi := &file_hw_proto_msgTypes[3]
	if protoimpl.UnsafeEnabled && x != nil {
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		if ms.LoadMessageInfo() == nil {
			ms.StoreMessageInfo(mi)
		}
		return ms
	}
	return mi.MessageOf(x)
}

// Deprecated: Use InferRequest.ProtoReflect.Descriptor instead.
func (*InferRequest) Descriptor() ([]byte, []int) {
	return file_hw_proto_rawDescGZIP(), []int{3}
}

func (x *InferRequest) GetRid() string {
	if x != nil {
		return x.Rid
	}
	return ""
}

func (x *InferRequest) GetUuid() string {
	if x != nil {
		return x.Uuid
	}
	return ""
}

func (x *InferRequest) GetOutlet() string {
	if x != nil {
		return x.Outlet
	}
	return ""
}

func (x *InferRequest) GetPath() string {
	if x != nil {
		return x.Path
	}
	return ""
}

func (x *InferRequest) GetData() []byte {
	if x != nil {
		return x.Data
	}
	return nil
}

func (x *InferRequest) GetExtra() string {
	if x != nil {
		return x.Extra
	}
	return ""
}

type Box struct {
	state         protoimpl.MessageState
	sizeCache     protoimpl.SizeCache
	unknownFields protoimpl.UnknownFields

	X int32 `protobuf:"varint,1,opt,name=x,proto3" json:"x,omitempty"`
	Y int32 `protobuf:"varint,2,opt,name=y,proto3" json:"y,omitempty"`
	W int32 `protobuf:"varint,3,opt,name=w,proto3" json:"w,omitempty"`
	H int32 `protobuf:"varint,4,opt,name=h,proto3" json:"h,omitempty"`
}

func (x *Box) Reset() {
	*x = Box{}
	if protoimpl.UnsafeEnabled {
		mi := &file_hw_proto_msgTypes[4]
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		ms.StoreMessageInfo(mi)
	}
}

func (x *Box) String() string {
	return protoimpl.X.MessageStringOf(x)
}

func (*Box) ProtoMessage() {}

func (x *Box) ProtoReflect() protoreflect.Message {
	mi := &file_hw_proto_msgTypes[4]
	if protoimpl.UnsafeEnabled && x != nil {
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		if ms.LoadMessageInfo() == nil {
			ms.StoreMessageInfo(mi)
		}
		return ms
	}
	return mi.MessageOf(x)
}

// Deprecated: Use Box.ProtoReflect.Descriptor instead.
func (*Box) Descriptor() ([]byte, []int) {
	return file_hw_proto_rawDescGZIP(), []int{4}
}

func (x *Box) GetX() int32 {
	if x != nil {
		return x.X
	}
	return 0
}

func (x *Box) GetY() int32 {
	if x != nil {
		return x.Y
	}
	return 0
}

func (x *Box) GetW() int32 {
	if x != nil {
		return x.W
	}
	return 0
}

func (x *Box) GetH() int32 {
	if x != nil {
		return x.H
	}
	return 0
}

type FeatureResult struct {
	state         protoimpl.MessageState
	sizeCache     protoimpl.SizeCache
	unknownFields protoimpl.UnknownFields

	Features []float32 `protobuf:"fixed32,1,rep,packed,name=features,proto3" json:"features,omitempty"`
	Box      *Box      `protobuf:"bytes,2,opt,name=box,proto3" json:"box,omitempty"`
}

func (x *FeatureResult) Reset() {
	*x = FeatureResult{}
	if protoimpl.UnsafeEnabled {
		mi := &file_hw_proto_msgTypes[5]
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		ms.StoreMessageInfo(mi)
	}
}

func (x *FeatureResult) String() string {
	return protoimpl.X.MessageStringOf(x)
}

func (*FeatureResult) ProtoMessage() {}

func (x *FeatureResult) ProtoReflect() protoreflect.Message {
	mi := &file_hw_proto_msgTypes[5]
	if protoimpl.UnsafeEnabled && x != nil {
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		if ms.LoadMessageInfo() == nil {
			ms.StoreMessageInfo(mi)
		}
		return ms
	}
	return mi.MessageOf(x)
}

// Deprecated: Use FeatureResult.ProtoReflect.Descriptor instead.
func (*FeatureResult) Descriptor() ([]byte, []int) {
	return file_hw_proto_rawDescGZIP(), []int{5}
}

func (x *FeatureResult) GetFeatures() []float32 {
	if x != nil {
		return x.Features
	}
	return nil
}

func (x *FeatureResult) GetBox() *Box {
	if x != nil {
		return x.Box
	}
	return nil
}

type FeatureResponse struct {
	state         protoimpl.MessageState
	sizeCache     protoimpl.SizeCache
	unknownFields protoimpl.UnknownFields

	Feature []*FeatureResult `protobuf:"bytes,1,rep,name=feature,proto3" json:"feature,omitempty"`
}

func (x *FeatureResponse) Reset() {
	*x = FeatureResponse{}
	if protoimpl.UnsafeEnabled {
		mi := &file_hw_proto_msgTypes[6]
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		ms.StoreMessageInfo(mi)
	}
}

func (x *FeatureResponse) String() string {
	return protoimpl.X.MessageStringOf(x)
}

func (*FeatureResponse) ProtoMessage() {}

func (x *FeatureResponse) ProtoReflect() protoreflect.Message {
	mi := &file_hw_proto_msgTypes[6]
	if protoimpl.UnsafeEnabled && x != nil {
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		if ms.LoadMessageInfo() == nil {
			ms.StoreMessageInfo(mi)
		}
		return ms
	}
	return mi.MessageOf(x)
}

// Deprecated: Use FeatureResponse.ProtoReflect.Descriptor instead.
func (*FeatureResponse) Descriptor() ([]byte, []int) {
	return file_hw_proto_rawDescGZIP(), []int{6}
}

func (x *FeatureResponse) GetFeature() []*FeatureResult {
	if x != nil {
		return x.Feature
	}
	return nil
}

type ObjectDetectionRequest struct {
	state         protoimpl.MessageState
	sizeCache     protoimpl.SizeCache
	unknownFields protoimpl.UnknownFields

	Cmd       string `protobuf:"bytes,1,opt,name=cmd,proto3" json:"cmd,omitempty"`
	Base64Img string `protobuf:"bytes,2,opt,name=base64_img,json=base64Img,proto3" json:"base64_img,omitempty"`
}

func (x *ObjectDetectionRequest) Reset() {
	*x = ObjectDetectionRequest{}
	if protoimpl.UnsafeEnabled {
		mi := &file_hw_proto_msgTypes[7]
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		ms.StoreMessageInfo(mi)
	}
}

func (x *ObjectDetectionRequest) String() string {
	return protoimpl.X.MessageStringOf(x)
}

func (*ObjectDetectionRequest) ProtoMessage() {}

func (x *ObjectDetectionRequest) ProtoReflect() protoreflect.Message {
	mi := &file_hw_proto_msgTypes[7]
	if protoimpl.UnsafeEnabled && x != nil {
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		if ms.LoadMessageInfo() == nil {
			ms.StoreMessageInfo(mi)
		}
		return ms
	}
	return mi.MessageOf(x)
}

// Deprecated: Use ObjectDetectionRequest.ProtoReflect.Descriptor instead.
func (*ObjectDetectionRequest) Descriptor() ([]byte, []int) {
	return file_hw_proto_rawDescGZIP(), []int{7}
}

func (x *ObjectDetectionRequest) GetCmd() string {
	if x != nil {
		return x.Cmd
	}
	return ""
}

func (x *ObjectDetectionRequest) GetBase64Img() string {
	if x != nil {
		return x.Base64Img
	}
	return ""
}

type ObjectDetectionResponse struct {
	state         protoimpl.MessageState
	sizeCache     protoimpl.SizeCache
	unknownFields protoimpl.UnknownFields

	Info   string `protobuf:"bytes,1,opt,name=info,proto3" json:"info,omitempty"`
	Code   int32  `protobuf:"varint,2,opt,name=code,proto3" json:"code,omitempty"`
	Image  string `protobuf:"bytes,3,opt,name=image,proto3" json:"image,omitempty"`
	Result string `protobuf:"bytes,4,opt,name=result,proto3" json:"result,omitempty"`
	Type   string `protobuf:"bytes,5,opt,name=type,proto3" json:"type,omitempty"`
}

func (x *ObjectDetectionResponse) Reset() {
	*x = ObjectDetectionResponse{}
	if protoimpl.UnsafeEnabled {
		mi := &file_hw_proto_msgTypes[8]
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		ms.StoreMessageInfo(mi)
	}
}

func (x *ObjectDetectionResponse) String() string {
	return protoimpl.X.MessageStringOf(x)
}

func (*ObjectDetectionResponse) ProtoMessage() {}

func (x *ObjectDetectionResponse) ProtoReflect() protoreflect.Message {
	mi := &file_hw_proto_msgTypes[8]
	if protoimpl.UnsafeEnabled && x != nil {
		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
		if ms.LoadMessageInfo() == nil {
			ms.StoreMessageInfo(mi)
		}
		return ms
	}
	return mi.MessageOf(x)
}

// Deprecated: Use ObjectDetectionResponse.ProtoReflect.Descriptor instead.
func (*ObjectDetectionResponse) Descriptor() ([]byte, []int) {
	return file_hw_proto_rawDescGZIP(), []int{8}
}

func (x *ObjectDetectionResponse) GetInfo() string {
	if x != nil {
		return x.Info
	}
	return ""
}

func (x *ObjectDetectionResponse) GetCode() int32 {
	if x != nil {
		return x.Code
	}
	return 0
}

func (x *ObjectDetectionResponse) GetImage() string {
	if x != nil {
		return x.Image
	}
	return ""
}

func (x *ObjectDetectionResponse) GetResult() string {
	if x != nil {
		return x.Result
	}
	return ""
}

func (x *ObjectDetectionResponse) GetType() string {
	if x != nil {
		return x.Type
	}
	return ""
}

var File_hw_proto protoreflect.FileDescriptor

var file_hw_proto_rawDesc = []byte{
	0x0a, 0x08, 0x68, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x68, 0x65, 0x6c, 0x6c,
	0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61,
	0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70,
	0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f,
	0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74,
	0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
	0x75, 0x66, 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74,
	0x6f, 0x22, 0x22, 0x0a, 0x0c, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
	0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
	0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x26, 0x0a, 0x0a, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65,
	0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01,
	0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x30, 0x0a,
	0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64,
	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x10, 0x0a,
	0x03, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x22,
	0x8a, 0x01, 0x0a, 0x0c, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
	0x12, 0x10, 0x0a, 0x03, 0x72, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x72,
	0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
	0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x6c, 0x65, 0x74,
	0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x6c, 0x65, 0x74, 0x12, 0x12,
	0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61,
	0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c,
	0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x18,
	0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x22, 0x3d, 0x0a, 0x03,
	0x42, 0x6f, 0x78, 0x12, 0x0c, 0x0a, 0x01, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01,
	0x78, 0x12, 0x0c, 0x0a, 0x01, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x79, 0x12,
	0x0c, 0x0a, 0x01, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x77, 0x12, 0x0c, 0x0a,
	0x01, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x01, 0x68, 0x22, 0x4e, 0x0a, 0x0d, 0x46,
	0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x1a, 0x0a, 0x08,
	0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x02, 0x52, 0x08,
	0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x03, 0x62, 0x6f, 0x78, 0x18,
	0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72,
	0x6c, 0x64, 0x2e, 0x42, 0x6f, 0x78, 0x52, 0x03, 0x62, 0x6f, 0x78, 0x22, 0x46, 0x0a, 0x0f, 0x46,
	0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33,
	0x0a, 0x07, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
	0x19, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x46, 0x65, 0x61,
	0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x66, 0x65, 0x61, 0x74,
	0x75, 0x72, 0x65, 0x22, 0x49, 0x0a, 0x16, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x44, 0x65, 0x74,
	0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a,
	0x03, 0x63, 0x6d, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x63, 0x6d, 0x64, 0x12,
	0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x65, 0x36, 0x34, 0x5f, 0x69, 0x6d, 0x67, 0x18, 0x02, 0x20,
	0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x61, 0x73, 0x65, 0x36, 0x34, 0x49, 0x6d, 0x67, 0x22, 0x83,
	0x01, 0x0a, 0x17, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69,
	0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6e,
	0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x12,
	0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x63, 0x6f,
	0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28,
	0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75,
	0x6c, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74,
	0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
	0x74, 0x79, 0x70, 0x65, 0x32, 0x90, 0x03, 0x0a, 0x07, 0x47, 0x72, 0x65, 0x65, 0x74, 0x65, 0x72,
	0x12, 0x3e, 0x0a, 0x08, 0x53, 0x61, 0x79, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x12, 0x18, 0x2e, 0x68,
	0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52,
	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f,
	0x72, 0x6c, 0x64, 0x2e, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00,
	0x12, 0x3b, 0x0a, 0x07, 0x43, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x68, 0x65,
	0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x49, 0x6e, 0x66, 0x65, 0x72, 0x52, 0x65,
	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72,
	0x6c, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a,
	0x0b, 0x67, 0x65, 0x74, 0x5f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x1b, 0x2e, 0x67,
	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42,
	0x79, 0x74, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x1b, 0x2e, 0x68, 0x65, 0x6c, 0x6c,
	0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65,
	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7a, 0x0a, 0x10, 0x6f, 0x62, 0x6a, 0x65,
	0x63, 0x74, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x2e, 0x68,
	0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74,
	0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
	0x1a, 0x23, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x4f, 0x62,
	0x6a, 0x65, 0x63, 0x74, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73,
	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1d, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x17, 0x22, 0x15, 0x2f,
	0x76, 0x31, 0x2f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
	0x69, 0x6f, 0x6e, 0x2f, 0x12, 0x41, 0x0a, 0x09, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75,
	0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
	0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c,
	0x56, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x00, 0x42, 0x41, 0x0a, 0x1b, 0x69, 0x6f, 0x2e, 0x67, 0x72,
	0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x65, 0x6c, 0x6c,
	0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x42, 0x0f, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x57, 0x6f, 0x72,
	0x6c, 0x64, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x0f, 0x67, 0x72, 0x70, 0x63, 0x2f,
	0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
	0x6f, 0x33,
}

var (
	file_hw_proto_rawDescOnce sync.Once
	file_hw_proto_rawDescData = file_hw_proto_rawDesc
)

func file_hw_proto_rawDescGZIP() []byte {
	file_hw_proto_rawDescOnce.Do(func() {
		file_hw_proto_rawDescData = protoimpl.X.CompressGZIP(file_hw_proto_rawDescData)
	})
	return file_hw_proto_rawDescData
}

var file_hw_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
var file_hw_proto_goTypes = []interface{}{
	(*HelloRequest)(nil),            // 0: helloworld.HelloRequest
	(*HelloReply)(nil),              // 1: helloworld.HelloReply
	(*Response)(nil),                // 2: helloworld.Response
	(*InferRequest)(nil),            // 3: helloworld.InferRequest
	(*Box)(nil),                     // 4: helloworld.Box
	(*FeatureResult)(nil),           // 5: helloworld.FeatureResult
	(*FeatureResponse)(nil),         // 6: helloworld.FeatureResponse
	(*ObjectDetectionRequest)(nil),  // 7: helloworld.ObjectDetectionRequest
	(*ObjectDetectionResponse)(nil), // 8: helloworld.ObjectDetectionResponse
	(*wrappers.BytesValue)(nil),     // 9: google.protobuf.BytesValue
	(*empty.Empty)(nil),             // 10: google.protobuf.Empty
	(*wrappers.BoolValue)(nil),      // 11: google.protobuf.BoolValue
}
var file_hw_proto_depIdxs = []int32{
	4,  // 0: helloworld.FeatureResult.box:type_name -> helloworld.Box
	5,  // 1: helloworld.FeatureResponse.feature:type_name -> helloworld.FeatureResult
	0,  // 2: helloworld.Greeter.SayHello:input_type -> helloworld.HelloRequest
	3,  // 3: helloworld.Greeter.Compute:input_type -> helloworld.InferRequest
	9,  // 4: helloworld.Greeter.get_feature:input_type -> google.protobuf.BytesValue
	7,  // 5: helloworld.Greeter.object_detection:input_type -> helloworld.ObjectDetectionRequest
	10, // 6: helloworld.Greeter.getStatus:input_type -> google.protobuf.Empty
	1,  // 7: helloworld.Greeter.SayHello:output_type -> helloworld.HelloReply
	2,  // 8: helloworld.Greeter.Compute:output_type -> helloworld.Response
	6,  // 9: helloworld.Greeter.get_feature:output_type -> helloworld.FeatureResponse
	8,  // 10: helloworld.Greeter.object_detection:output_type -> helloworld.ObjectDetectionResponse
	11, // 11: helloworld.Greeter.getStatus:output_type -> google.protobuf.BoolValue
	7,  // [7:12] is the sub-list for method output_type
	2,  // [2:7] is the sub-list for method input_type
	2,  // [2:2] is the sub-list for extension type_name
	2,  // [2:2] is the sub-list for extension extendee
	0,  // [0:2] is the sub-list for field type_name
}

func init() { file_hw_proto_init() }
func file_hw_proto_init() {
	if File_hw_proto != nil {
		return
	}
	if !protoimpl.UnsafeEnabled {
		file_hw_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
			switch v := v.(*HelloRequest); i {
			case 0:
				return &v.state
			case 1:
				return &v.sizeCache
			case 2:
				return &v.unknownFields
			default:
				return nil
			}
		}
		file_hw_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
			switch v := v.(*HelloReply); i {
			case 0:
				return &v.state
			case 1:
				return &v.sizeCache
			case 2:
				return &v.unknownFields
			default:
				return nil
			}
		}
		file_hw_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
			switch v := v.(*Response); i {
			case 0:
				return &v.state
			case 1:
				return &v.sizeCache
			case 2:
				return &v.unknownFields
			default:
				return nil
			}
		}
		file_hw_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
			switch v := v.(*InferRequest); i {
			case 0:
				return &v.state
			case 1:
				return &v.sizeCache
			case 2:
				return &v.unknownFields
			default:
				return nil
			}
		}
		file_hw_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
			switch v := v.(*Box); i {
			case 0:
				return &v.state
			case 1:
				return &v.sizeCache
			case 2:
				return &v.unknownFields
			default:
				return nil
			}
		}
		file_hw_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
			switch v := v.(*FeatureResult); i {
			case 0:
				return &v.state
			case 1:
				return &v.sizeCache
			case 2:
				return &v.unknownFields
			default:
				return nil
			}
		}
		file_hw_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
			switch v := v.(*FeatureResponse); i {
			case 0:
				return &v.state
			case 1:
				return &v.sizeCache
			case 2:
				return &v.unknownFields
			default:
				return nil
			}
		}
		file_hw_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
			switch v := v.(*ObjectDetectionRequest); i {
			case 0:
				return &v.state
			case 1:
				return &v.sizeCache
			case 2:
				return &v.unknownFields
			default:
				return nil
			}
		}
		file_hw_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
			switch v := v.(*ObjectDetectionResponse); i {
			case 0:
				return &v.state
			case 1:
				return &v.sizeCache
			case 2:
				return &v.unknownFields
			default:
				return nil
			}
		}
	}
	type x struct{}
	out := protoimpl.TypeBuilder{
		File: protoimpl.DescBuilder{
			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
			RawDescriptor: file_hw_proto_rawDesc,
			NumEnums:      0,
			NumMessages:   9,
			NumExtensions: 0,
			NumServices:   1,
		},
		GoTypes:           file_hw_proto_goTypes,
		DependencyIndexes: file_hw_proto_depIdxs,
		MessageInfos:      file_hw_proto_msgTypes,
	}.Build()
	File_hw_proto = out.File
	file_hw_proto_rawDesc = nil
	file_hw_proto_goTypes = nil
	file_hw_proto_depIdxs = nil
}


<|start_filename|>internal/meta/sinkMeta_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package meta

import (
	"testing"
)

func TestHintWhenModifySink(t *testing.T) {
	taosMeta := &uiSink{
		Fields: []field{
			{
				Name:    "ip",
				Default: "911.911.911.911",
			},
		},
	}
	logMeta := &uiSink{
		Fields: []field{
			{
				Name:    "ip",
				Default: "911.911.911.911",
			},
		},
	}

	gSinkmetadata = make(map[string]*uiSink)
	gSinkmetadata["taos.json"] = taosMeta
	gSinkmetadata["log.json"] = logMeta

	oldSink, err := GetSinkMeta("taos", "en_US")
	if err != nil {
		t.Errorf("%v", err)
	} else {
		if "911.911.911.911" != oldSink.Fields[0].Default {
			t.Errorf("fail")
		}
	}
}


<|start_filename|>etc/sinks/log.json<|end_filename|>
{
  "about": {
    "trial": false,
    "author": {
      "name": "EMQ",
      "email": "<EMAIL>",
      "company": "EMQ Technologies Co., Ltd",
      "website": "https://www.emqx.io"
    },
    "helpUrl": {
      "en_US": "https://github.com/lf-edge/ekuiper/blob/master/docs/en_US/rules/sinks/log.md",
      "zh_CN": "https://github.com/lf-edge/ekuiper/blob/master/docs/zh_CN/rules/sinks/log.md"
    },
    "description": {
      "en_US": "The action is used for print output message into log file, the log file is at $ekuiper_install/log/stream.log by default.",
      "zh_CN": "该操作用于将输出消息打印到日志文件中，默认情况下，该日志文件位于 $ekuiper_install/log/stream.log 中。"
    }
  },
  "properties": [
  ]
}

<|start_filename|>internal/binder/factory_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package binder

import (
	"github.com/lf-edge/ekuiper/internal/binder/mock"
	"testing"
)

func TestEntry(t *testing.T) {
	m := mock.NewMockFactory()
	e := &FactoryEntry{
		Name:    "mock",
		Factory: m,
	}
	if e == nil {
		t.Errorf("cannot instantiate FactoryEntry")
	}
}


<|start_filename|>internal/conf/jsonpath_eval.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conf

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/PaesslerAG/gval"
	"github.com/PaesslerAG/jsonpath"
	"github.com/lf-edge/ekuiper/pkg/cast"
	"reflect"
)

var builder = gval.Full(jsonpath.PlaceholderExtension())

type JsonPathEval interface {
	Eval(data interface{}) (interface{}, error)
}

type gvalPathEval struct {
	valuer gval.Evaluable
}

func (e *gvalPathEval) Eval(data interface{}) (interface{}, error) {
	var input interface{}
	at := reflect.TypeOf(data)
	if at != nil {
		switch at.Kind() {
		case reflect.Map:
			input = cast.ConvertToInterfaceArr(data.(map[string]interface{}))
		case reflect.Slice:
			input = cast.ConvertSlice(data)
		case reflect.String:
			v, _ := data.(string)
			err := json.Unmarshal([]byte(v), &input)
			if err != nil {
				return nil, fmt.Errorf("data '%v' is not a valid json string", data)
			}
		default:
			return nil, fmt.Errorf("invalid data %v for jsonpath", data)
		}
	} else {
		return nil, fmt.Errorf("invalid data nil for jsonpath")
	}
	return e.valuer(context.Background(), input)
}

func GetJsonPathEval(jsonpath string) (JsonPathEval, error) {
	e, err := builder.NewEvaluable(jsonpath)
	if err != nil {
		return nil, err
	}
	return &gvalPathEval{valuer: e}, nil
}


<|start_filename|>internal/topo/operator/order_operator.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package operator

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/xsql"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/ast"
)

type OrderOp struct {
	SortFields ast.SortFields
}

/**
 *  input: *xsql.Tuple from preprocessor | xsql.WindowTuplesSet from windowOp | xsql.JoinTupleSets from joinOp
 *  output: *xsql.Tuple | xsql.WindowTuplesSet | xsql.JoinTupleSets
 */
func (p *OrderOp) Apply(ctx api.StreamContext, data interface{}, fv *xsql.FunctionValuer, _ *xsql.AggregateFunctionValuer) interface{} {
	log := ctx.GetLogger()
	log.Debugf("order plan receive %s", data)
	sorter := xsql.OrderedBy(p.SortFields, fv)
	switch input := data.(type) {
	case error:
		return input
	case xsql.Valuer:
		return input
	case xsql.SortingData:
		if err := sorter.Sort(input); err != nil {
			return fmt.Errorf("run Order By error: %s", err)
		}
		return input
	default:
		return fmt.Errorf("run Order By error: expect xsql.Valuer or its array type")
	}
}


<|start_filename|>internal/topo/operator/join_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package operator

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/lf-edge/ekuiper/internal/conf"
	"github.com/lf-edge/ekuiper/internal/topo/context"
	"github.com/lf-edge/ekuiper/internal/xsql"
	"github.com/lf-edge/ekuiper/pkg/ast"
	"github.com/lf-edge/ekuiper/pkg/cast"
	"reflect"
	"strings"
	"testing"
)

func TestLeftJoinPlan_Apply(t *testing.T) {
	var tests = []struct {
		sql    string
		data   xsql.WindowTuplesSet
		result interface{}
	}{
		{ //0
			sql: "SELECT id1 FROM src1 left join src2 on id1 = id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
				WindowRange: &xsql.WindowRange{
					WindowStart: 1541152486013,
					WindowEnd:   1541152487013,
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
				},
				WindowRange: &xsql.WindowRange{
					WindowStart: 1541152486013,
					WindowEnd:   1541152487013,
				},
			},
		},
		{ // 1
			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
				},
			},
		},
		{ // 2
			sql: "SELECT id1 FROM src1 left join src2 on src1.ts = src2.ts",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854515000)},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2", "ts": cast.TimeFromUnixMilli(1568854525000)},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3", "ts": cast.TimeFromUnixMilli(1568854535000)},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1", "ts": cast.TimeFromUnixMilli(1568854515000)},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2", "ts": cast.TimeFromUnixMilli(1568854525000)},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3", "ts": cast.TimeFromUnixMilli(1568854545000)},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1", "ts": cast.TimeFromUnixMilli(1568854515000)}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1", "ts": cast.TimeFromUnixMilli(1568854515000)}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2", "ts": cast.TimeFromUnixMilli(1568854525000)}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2", "ts": cast.TimeFromUnixMilli(1568854525000)}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3", "ts": cast.TimeFromUnixMilli(1568854535000)}},
						},
					},
				},
			},
		},
		{ // 3
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 5, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 6, "f2": "w3"},
							},
						},
					},
				},
			},
			result: nil,
		},

		{ // 4
			sql: "SELECT id1 FROM src1 As s1 left join src2 as s2 on s1.id1 = s2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "s1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "s1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "s1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "s1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "s2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "s2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "s2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "s2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "s1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "s2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "s1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "s2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "s1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
				},
			},
		},

		{ // 5
			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, {Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}},
						},
					},
				},
			},
		},

		{ // 6
			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples:  []xsql.Tuple{},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples:  nil,
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples:  []xsql.Tuple{},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: nil,
		},
		{
			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples:  nil,
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: nil,
		},

		{ // 10     select id1 FROM src1 left join src2 on null = null
			sql: "SELECT id1 FROM src1 left join src2 on src1.id2 = src2.id1",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f2": "w1"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f2": "w1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f2": "w1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 left join src2 on src1.id1*2 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2*2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 left join src2 on src1.f1->cid = src2.f2->cid",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": str2Map(`{"cid" : 1, "name" : "tom1"}`)},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": str2Map(`{"cid" : 2, "name" : "mike1"}`)},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": str2Map(`{"cid" : 3, "name" : "alice1"}`)},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": str2Map(`{"cid" : 1, "name" : "tom2"}`)},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": str2Map(`{"cid" : 2, "name" : "mike2"}`)},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": str2Map(`{"cid" : 4, "name" : "alice2"}`)},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": str2Map(`{"cid" : 1, "name" : "tom1"}`)}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": str2Map(`{"cid" : 1, "name" : "tom2"}`)}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": str2Map(`{"cid" : 2, "name" : "mike1"}`)}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": str2Map(`{"cid" : 2, "name" : "mike2"}`)}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": str2Map(`{"cid" : 3, "name" : "alice1"}`)}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1, mqtt(src1.topic) AS a, mqtt(src2.topic) as b FROM src1 left join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter:  "src1",
								Message:  xsql.Message{"id1": 1, "f1": "v1"},
								Metadata: xsql.Metadata{"topic": "devices/type1/device001"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter:  "src2",
								Message:  xsql.Message{"id2": 1, "f2": "w1"},
								Metadata: xsql.Metadata{"topic": "devices/type2/device001"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}, Metadata: xsql.Metadata{"topic": "devices/type1/device001"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}, Metadata: xsql.Metadata{"topic": "devices/type2/device001"}},
						},
					},
				},
			},
		},
		{
			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v4"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 4, "f1": "v5"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 3, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 3, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 3, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 3, "f2": "w3"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v4"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 3, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v4"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 3, "f2": "w3"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 4, "f1": "v5"}},
						},
					},
				},
			},
		},
		{
			sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w3"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
				},
			},
		},
	}

	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	contextLogger := conf.Log.WithField("rule", "TestLeftJoinPlan_Apply")
	ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger)
	for i, tt := range tests {
		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
		if err != nil {
			t.Errorf("statement parse error %s", err)
			break
		}

		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
			t.Errorf("statement source is not a table")
		} else {
			fv, afv := xsql.NewFunctionValuersForOp(nil)
			pp := &JoinOp{Joins: stmt.Joins, From: table}
			result := pp.Apply(ctx, tt.data, fv, afv)
			if !reflect.DeepEqual(tt.result, result) {
				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
			}
		}
	}
}

func TestInnerJoinPlan_Apply(t *testing.T) {
	var tests = []struct {
		sql    string
		data   xsql.WindowTuplesSet
		result interface{}
	}{
		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
				},
			},
		},
		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
				},
			},
		},
		{
			sql: "SELECT id1 FROM src1 As s1 inner join src2 as s2 on s1.id1 = s2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "s1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "s1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "s1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "s1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "s2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "s2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "s2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "s2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "s1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "s2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "s1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "s2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							},
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v2"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples:  []xsql.Tuple{},
					},
				},
			},
			result: nil,
		},

		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples:  nil,
					},
				},
			},
			result: nil,
		},

		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples:  []xsql.Tuple{},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: nil,
		},

		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples:  nil,
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: nil,
		},

		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1*2 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2*2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 inner join src2 on src1.f1->cid = src2.f2->cid",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": str2Map(`{"cid" : 1, "name" : "tom1"}`)},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": str2Map(`{"cid" : 2, "name" : "mike1"}`)},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": str2Map(`{"cid" : 3, "name" : "alice1"}`)},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": str2Map(`{"cid" : 1, "name" : "tom2"}`)},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": str2Map(`{"cid" : 2, "name" : "mike2"}`)},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": str2Map(`{"cid" : 4, "name" : "alice2"}`)},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": str2Map(`{"cid" : 1, "name" : "tom1"}`)}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": str2Map(`{"cid" : 1, "name" : "tom2"}`)}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": str2Map(`{"cid" : 2, "name" : "mike1"}`)}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": str2Map(`{"cid" : 2, "name" : "mike2"}`)}},
						},
					},
				},
			},
		},
		{
			sql: "SELECT id1 FROM src1 As s1 inner join src2 as s2 on s1.id1 = s2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "s1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "s1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "s1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "s1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "s2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "s2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "s2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "s2",
								Message: xsql.Message{"id2": 2, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "s1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "s2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "s1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "s2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "s1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "s2", Message: xsql.Message{"id2": 2, "f2": "w3"}},
						},
					},
				},
			},
		},
	}

	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	contextLogger := conf.Log.WithField("rule", "TestInnerJoinPlan_Apply")
	ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger)
	for i, tt := range tests {
		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
		if err != nil {
			t.Errorf("statement parse error %s", err)
			break
		}

		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
			t.Errorf("statement source is not a table")
		} else {
			fv, afv := xsql.NewFunctionValuersForOp(nil)
			pp := &JoinOp{Joins: stmt.Joins, From: table}
			result := pp.Apply(ctx, tt.data, fv, afv)
			if !reflect.DeepEqual(tt.result, result) {
				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
			}
		}
	}
}

func TestRightJoinPlan_Apply(t *testing.T) {
	var tests = []struct {
		sql    string
		data   xsql.WindowTuplesSet
		result interface{}
	}{
		{
			sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},
				},
			},
		},
		{
			sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v3"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"f2": "w2"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},
				},
			},
		},
		{
			sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}},
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
						},
					},
				},
			},
		},
		{
			sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples:  []xsql.Tuple{},
					},
				},
			},
			result: nil,
		},
		{
			sql: "SELECT id1 FROM src1 right join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples:  []xsql.Tuple{},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"f2": "w2"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},
				},
			},
		},
	}
	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	contextLogger := conf.Log.WithField("rule", "TestRightJoinPlan_Apply")
	ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger)
	for i, tt := range tests {
		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
		if err != nil {
			t.Errorf("statement parse error %s", err)
			break
		}

		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
			t.Errorf("statement source is not a table")
		} else {
			fv, afv := xsql.NewFunctionValuersForOp(nil)
			pp := &JoinOp{Joins: stmt.Joins, From: table}
			result := pp.Apply(ctx, tt.data, fv, afv)
			if !reflect.DeepEqual(tt.result, result) {
				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
			}
		}
	}
}

func TestFullJoinPlan_Apply(t *testing.T) {
	var tests = []struct {
		sql    string
		data   xsql.WindowTuplesSet
		result interface{}
	}{
		{
			sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w4"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}}, {Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w4"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples:  []xsql.Tuple{},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 5, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 6, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 5, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 6, "f2": "w3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples:  []xsql.Tuple{},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},
				},
			},
		},
	}
	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	contextLogger := conf.Log.WithField("rule", "TestFullJoinPlan_Apply")
	ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger)
	for i, tt := range tests {
		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
		if err != nil {
			t.Errorf("statement parse error %s", err)
			break
		}

		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
			t.Errorf("statement source is not a table")
		} else {
			fv, afv := xsql.NewFunctionValuersForOp(nil)
			pp := &JoinOp{Joins: stmt.Joins, From: table}
			result := pp.Apply(ctx, tt.data, fv, afv)
			if !reflect.DeepEqual(tt.result, result) {
				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
			}
		}
	}
}

func TestCrossJoinPlan_Apply(t *testing.T) {
	var tests = []struct {
		sql    string
		data   xsql.WindowTuplesSet
		result interface{}
	}{
		{
			sql: "SELECT id1 FROM src1 cross join src2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 2, "f1": "v2"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},

					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 2, "f2": "w2"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 3, "f1": "v3"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 4, "f2": "w3"}},
						},
					},
				},
			},
		},

		{
			sql: "SELECT id1 FROM src1 cross join src2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: &xsql.JoinTupleSets{
				Content: []xsql.JoinTuple{
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}},
							{Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w1"}},
						},
					},
					{
						Tuples: []xsql.Tuple{
							{Emitter: "src1", Message: xsql.Message{"id1": 1, "f1": "v1"}}, {Emitter: "src2", Message: xsql.Message{"id2": 1, "f2": "w2"}},
						},
					},
				},
			},
		},
		{
			sql: "SELECT id1 FROM src1 cross join src2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples:  []xsql.Tuple{},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: nil,
		},
		{
			sql: "SELECT id1 FROM src2 cross join src1",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples:  []xsql.Tuple{},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w2"},
							},
						},
					},
				},
			},
			result: nil,
		},
	}
	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	contextLogger := conf.Log.WithField("rule", "TestCrossJoinPlan_Apply")
	ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger)
	for i, tt := range tests {
		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
		if err != nil {
			t.Errorf("statement parse error %s", err)
			break
		}

		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
			t.Errorf("statement source is not a table")
		} else {
			fv, afv := xsql.NewFunctionValuersForOp(nil)
			pp := &JoinOp{Joins: stmt.Joins, From: table}
			result := pp.Apply(ctx, tt.data, fv, afv)
			if !reflect.DeepEqual(tt.result, result) {
				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
			}
		}
	}
}

func TestCrossJoinPlanError(t *testing.T) {
	var tests = []struct {
		sql    string
		data   interface{}
		result interface{}
	}{
		{
			sql:    "SELECT id1 FROM src1 cross join src2",
			data:   errors.New("an error from upstream"),
			result: errors.New("an error from upstream"),
		}, {
			sql: "SELECT id1 FROM src1 full join src2 on src1.id1 = src2.id2",
			data: xsql.WindowTuplesSet{
				Content: []xsql.WindowTuples{
					{
						Emitter: "src1",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src1",
								Message: xsql.Message{"id1": 1, "f1": "v1"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 2, "f1": "v2"},
							}, {
								Emitter: "src1",
								Message: xsql.Message{"id1": 3, "f1": "v3"},
							},
						},
					},

					{
						Emitter: "src2",
						Tuples: []xsql.Tuple{
							{
								Emitter: "src2",
								Message: xsql.Message{"id2": 1, "f2": "w1"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": "3", "f2": "w2"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 4, "f2": "w3"},
							}, {
								Emitter: "src2",
								Message: xsql.Message{"id2": 2, "f2": "w4"},
							},
						},
					},
				},
			},
			result: errors.New("run Join error: invalid operation int64(1) = string(3)"),
		},
	}
	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	contextLogger := conf.Log.WithField("rule", "TestCrossJoinPlan_Apply")
	ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger)
	for i, tt := range tests {
		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
		if err != nil {
			t.Errorf("statement parse error %s", err)
			break
		}

		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
			t.Errorf("statement source is not a table")
		} else {
			fv, afv := xsql.NewFunctionValuersForOp(nil)
			pp := &JoinOp{Joins: stmt.Joins, From: table}
			result := pp.Apply(ctx, tt.data, fv, afv)
			if !reflect.DeepEqual(tt.result, result) {
				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
			}
		}
	}
}

func str2Map(s string) map[string]interface{} {
	var input map[string]interface{}
	if err := json.Unmarshal([]byte(s), &input); err != nil {
		fmt.Printf("Failed to parse the JSON data.\n")
		return nil
	}
	return input
}


<|start_filename|>internal/topo/context/func_context.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package context

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/topo/connection"
	"github.com/lf-edge/ekuiper/pkg/api"
)

type DefaultFuncContext struct {
	api.StreamContext
	funcId int
}

func NewDefaultFuncContext(ctx api.StreamContext, id int) *DefaultFuncContext {
	return &DefaultFuncContext{
		StreamContext: ctx,
		funcId:        id,
	}
}

func (c *DefaultFuncContext) IncrCounter(key string, amount int) error {
	return c.StreamContext.IncrCounter(c.convertKey(key), amount)
}

func (c *DefaultFuncContext) GetCounter(key string) (int, error) {
	return c.StreamContext.GetCounter(c.convertKey(key))
}

func (c *DefaultFuncContext) PutState(key string, value interface{}) error {
	return c.StreamContext.PutState(c.convertKey(key), value)
}

func (c *DefaultFuncContext) GetState(key string) (interface{}, error) {
	return c.StreamContext.GetState(c.convertKey(key))
}

func (c *DefaultFuncContext) DeleteState(key string) error {
	return c.StreamContext.DeleteState(c.convertKey(key))
}

func (c *DefaultFuncContext) GetFuncId() int {
	return c.funcId
}

func (c *DefaultFuncContext) GetConnection(connectSelector string) (interface{}, error) {
	return connection.GetConnection(connectSelector)
}

func (c *DefaultFuncContext) ReleaseConnection(connectSelector string) {
	connection.ReleaseConnection(connectSelector)
}

func (c *DefaultFuncContext) convertKey(key string) string {
	return fmt.Sprintf("$$func%d_%s", c.funcId, key)
}


<|start_filename|>internal/plugin/portable/model_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package portable

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
	"github.com/lf-edge/ekuiper/internal/testx"
	"reflect"
	"testing"
)

func TestValidate(t *testing.T) {
	var tests = []struct {
		p   *PluginInfo
		err string
	}{
		{
			p: &PluginInfo{
				PluginMeta: runtime.PluginMeta{
					Name:       "mirror",
					Version:    "1.0.0",
					Language:   "go",
					Executable: "mirror.exe",
				},
			},
			err: "invalid plugin, must define at lease one source, sink or function",
		}, {
			p: &PluginInfo{
				PluginMeta: runtime.PluginMeta{
					Name:       "wrr",
					Version:    "1.0.0",
					Language:   "go",
					Executable: "mirror.exe",
				},
				Sources: []string{"a", "b"},
			},
			err: "invalid plugin, expect name 'mirror' but got 'wrr'",
		}, {
			p: &PluginInfo{
				PluginMeta: runtime.PluginMeta{
					Name:       "mirror",
					Language:   "go",
					Executable: "mirror.exe",
				},
				Sinks: []string{"a", "b"},
			},
			err: "",
		}, {
			p: &PluginInfo{
				PluginMeta: runtime.PluginMeta{
					Name:     "mirror",
					Version:  "1.0.0",
					Language: "go",
				},
				Sources:   []string{"a", "b"},
				Sinks:     []string{"a", "b"},
				Functions: []string{"aa"},
			},
			err: "invalid plugin, missing executable",
		}, {
			p: &PluginInfo{
				PluginMeta: runtime.PluginMeta{
					Name:       "mirror",
					Version:    "1.0.0",
					Executable: "tt",
				},
				Sources:   []string{"a", "b"},
				Sinks:     []string{"a", "b"},
				Functions: []string{"aa"},
			},
			err: "invalid plugin, missing language",
		}, {
			p: &PluginInfo{
				PluginMeta: runtime.PluginMeta{
					Name:       "mirror",
					Version:    "1.0.0",
					Language:   "c",
					Executable: "tt",
				},
				Sources:   []string{"a", "b"},
				Sinks:     []string{"a", "b"},
				Functions: []string{"aa"},
			},
			err: "invalid plugin, language 'c' is not supported",
		},
	}
	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	for i, tt := range tests {
		err := tt.p.Validate("mirror")
		if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
			t.Errorf("%d error mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.err, err.Error())
		}
	}
}


<|start_filename|>internal/pkg/store/builder.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/pkg/db"
	"github.com/lf-edge/ekuiper/internal/pkg/db/redis"
	"github.com/lf-edge/ekuiper/internal/pkg/db/sql"
	rb "github.com/lf-edge/ekuiper/internal/pkg/store/redis"
	sb "github.com/lf-edge/ekuiper/internal/pkg/store/sql"
	"github.com/lf-edge/ekuiper/pkg/kv"
)

type Builder interface {
	CreateStore(table string) (kv.KeyValue, error)
}

func CreateStoreBuilder(database db.Database) (Builder, error) {
	switch database.(type) {
	case *redis.Instance:
		d := *database.(*redis.Instance)
		return rb.NewStoreBuilder(d), nil
	case sql.Database:
		d := database.(sql.Database)
		return sb.NewStoreBuilder(d), nil
	default:
		return nil, fmt.Errorf("unrecognized database type")
	}
}


<|start_filename|>internal/meta/connectionMeta.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package meta

import (
	"fmt"
)

func GetConnectionMeta(connectionName, language string) (ptrSourceProperty *uiSource, err error) {

	gSourcemetaLock.RLock()
	defer gSourcemetaLock.RUnlock()

	v, found := gSourcemetadata[connectionName+`.json`]
	if !found {
		return nil, fmt.Errorf(`%s%s`, getMsg(language, source, "not_found_plugin"), connectionName)
	}
	ret := make(map[string][]field)
	for kcfg, cfg := range v.ConfKeys {
		var sli []field
		for _, kvs := range cfg {
			if kvs.ConnectionRelated {
				sli = append(sli, kvs)
			}
		}
		ret[kcfg] = sli
	}
	ui := new(uiSource)
	*ui = *v
	ui.ConfKeys = ret
	return ui, nil
}

func GetConnectionPlugins() (sources []*pluginfo) {
	ConfigManager.lock.RLock()
	defer ConfigManager.lock.RUnlock()

	for _, conf := range ConfigManager.cfgOperators {
		if conf.IsSource() {
			continue
		}

		plugName := conf.GetPluginName()

		uiSourceRepKey := plugName + `.json`
		gSourcemetaLock.RLock()
		v, found := gSourcemetadata[uiSourceRepKey]
		if !found {
			gSourcemetaLock.RUnlock()
			continue
		}
		gSourcemetaLock.RUnlock()

		node := new(pluginfo)
		node.Name = plugName

		if nil == v.About {
			continue
		}
		node.About = v.About
		i := 0
		for ; i < len(sources); i++ {
			if node.Name <= sources[i].Name {
				sources = append(sources, node)
				copy(sources[i+1:], sources[i:])
				sources[i] = node
				break
			}
		}
		if len(sources) == i {
			sources = append(sources, node)
		}
	}
	return sources
}


<|start_filename|>internal/pkg/cert/cert.go<|end_filename|>
package cert

import (
	"crypto/tls"
	"crypto/x509"
	"github.com/lf-edge/ekuiper/internal/conf"
	"io/ioutil"
)

type TlsConfigurationOptions struct {
	SkipCertVerify bool
	CertFile       string
	KeyFile        string
	CaFile         string
}

func GenerateTLSForClient(
	Opts TlsConfigurationOptions) (*tls.Config, error) {

	tlsConfig := &tls.Config{
		InsecureSkipVerify: Opts.SkipCertVerify,
	}

	if len(Opts.CertFile) <= 0 && len(Opts.KeyFile) <= 0 {
		tlsConfig.Certificates = nil
	} else {
		if cert, err := certLoader(Opts.CertFile, Opts.KeyFile); err != nil {
			return nil, err
		} else {
			tlsConfig.Certificates = []tls.Certificate{cert}
		}
	}

	if len(Opts.CaFile) > 0 {
		root, err := caLoader(Opts.CaFile)
		if err != nil {
			return nil, err
		}
		tlsConfig.RootCAs = root
	}

	return tlsConfig, nil
}

func certLoader(certFilePath, keyFilePath string) (tls.Certificate, error) {
	if cp, err := conf.ProcessPath(certFilePath); err == nil {
		if kp, err1 := conf.ProcessPath(keyFilePath); err1 == nil {
			if cer, err2 := tls.LoadX509KeyPair(cp, kp); err2 != nil {
				return tls.Certificate{}, err2
			} else {
				return cer, nil
			}
		} else {
			return tls.Certificate{}, err1
		}
	} else {
		return tls.Certificate{}, err
	}
}

func caLoader(caFilePath string) (*x509.CertPool, error) {
	if cp, err := conf.ProcessPath(caFilePath); err == nil {
		pool := x509.NewCertPool()
		caCrt, err1 := ioutil.ReadFile(cp)
		if err1 != nil {
			return nil, err1
		}
		pool.AppendCertsFromPEM(caCrt)
		return pool, err1
	} else {
		return nil, err
	}
}


<|start_filename|>internal/pkg/filex/parser.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filex

import (
	"encoding/json"
	"gopkg.in/yaml.v3"
	"io/ioutil"
)

func ReadJsonUnmarshal(path string, ret interface{}) error {
	sliByte, err := ioutil.ReadFile(path)
	if nil != err {
		return err
	}
	err = json.Unmarshal(sliByte, ret)
	if nil != err {
		return err
	}
	return nil
}
func WriteYamlMarshal(path string, data interface{}) error {
	y, err := yaml.Marshal(data)
	if nil != err {
		return err
	}
	return ioutil.WriteFile(path, y, 0666)
}

func ReadYamlUnmarshal(path string, ret interface{}) error {
	sliByte, err := ioutil.ReadFile(path)
	if nil != err {
		return err
	}
	err = yaml.Unmarshal(sliByte, ret)
	if nil != err {
		return err
	}
	return nil
}


<|start_filename|>pkg/ast/funcArgsValidator.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ast

import "fmt"

// ProduceErrInfo Index is starting from 0
func ProduceErrInfo(name string, index int, expect string) (err error) {
	index++
	err = fmt.Errorf("Expect %s type for %d parameter of function %s.", expect, index, name)
	return
}

func ValidateLen(funcName string, exp, actual int) error {
	if actual != exp {
		return fmt.Errorf("The arguments for %s should be %d.", funcName, exp)
	}
	return nil
}

func IsNumericArg(arg Expr) bool {
	if _, ok := arg.(*NumberLiteral); ok {
		return true
	} else if _, ok := arg.(*IntegerLiteral); ok {
		return true
	}
	return false
}

func IsIntegerArg(arg Expr) bool {
	if _, ok := arg.(*IntegerLiteral); ok {
		return true
	}
	return false
}

func IsFloatArg(arg Expr) bool {
	if _, ok := arg.(*NumberLiteral); ok {
		return true
	}
	return false
}

func IsBooleanArg(arg Expr) bool {
	switch t := arg.(type) {
	case *BooleanLiteral:
		return true
	case *BinaryExpr:
		switch t.OP {
		case AND, OR, EQ, NEQ, LT, LTE, GT, GTE:
			return true
		default:
			return false
		}
	default:
		return false
	}
}

func IsStringArg(arg Expr) bool {
	if _, ok := arg.(*StringLiteral); ok {
		return true
	}
	return false
}

func IsTimeArg(arg Expr) bool {
	if _, ok := arg.(*TimeLiteral); ok {
		return true
	}
	return false
}


<|start_filename|>extensions/sources/zmq/zmq.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/message"
	zmq "github.com/pebbe/zmq4"
)

type zmqSource struct {
	subscriber    *zmq.Socket
	srv           string
	topic         string
	messageFormat string
	cancel        context.CancelFunc
}

func (s *zmqSource) Configure(topic string, props map[string]interface{}) error {
	s.topic = topic
	srv, ok := props["server"]
	if !ok {
		return fmt.Errorf("zmq source is missing property server")
	}
	s.srv = srv.(string)
	f, ok := props["format"]
	if !ok {
		s.messageFormat = message.FormatJson
	} else {
		s.messageFormat = f.(string)
	}
	return nil
}

func (s *zmqSource) Open(ctx api.StreamContext, consumer chan<- api.SourceTuple, errCh chan<- error) {
	logger := ctx.GetLogger()
	var err error
	s.subscriber, err = zmq.NewSocket(zmq.SUB)
	if err != nil {
		errCh <- fmt.Errorf("zmq source fails to create socket: %v", err)
	}
	err = s.subscriber.Connect(s.srv)
	if err != nil {
		errCh <- fmt.Errorf("zmq source fails to connect to %s: %v", s.srv, err)
	}
	s.subscriber.SetSubscribe(s.topic)
	logger.Debugf("zmq source subscribe to topic %s", s.topic)
	exeCtx, cancel := ctx.WithCancel()
	s.cancel = cancel
	logger.Debugf("start to listen")
	for {
		msgs, err := s.subscriber.RecvMessageBytes(0)
		if err != nil {
			id, err := s.subscriber.GetIdentity()
			errCh <- fmt.Errorf("zmq source getting message %s error: %v", id, err)
		} else {
			logger.Debugf("zmq source receive %v", msgs)
			var m []byte
			for i, msg := range msgs {
				if i == 0 && s.topic != "" {
					continue
				}
				m = append(m, msg...)
			}
			meta := make(map[string]interface{})
			if s.topic != "" {
				meta["topic"] = string(msgs[0])
			}
			result, e := message.Decode(m, s.messageFormat)
			if e != nil {
				logger.Errorf("Invalid data format, cannot decode %v to %s format with error %s", m, s.messageFormat, e)
			} else {
				consumer <- api.NewDefaultSourceTuple(result, meta)
			}
		}
		select {
		case <-exeCtx.Done():
			logger.Infof("zmq source done")
			if s.subscriber != nil {
				s.subscriber.Close()
			}
			return
		default:
			//do nothing
		}
	}
}

func (s *zmqSource) Close(ctx api.StreamContext) error {
	if s.cancel != nil {
		s.cancel()
	}
	return nil
}

func Zmq() api.Source {
	return &zmqSource{}
}


<|start_filename|>internal/service/external_service_rule_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/golang/protobuf/ptypes/empty"
	"github.com/golang/protobuf/ptypes/wrappers"
	"github.com/gorilla/mux"
	kconf "github.com/lf-edge/ekuiper/internal/conf"
	pb "github.com/lf-edge/ekuiper/internal/service/test/schemas/helloworld"
	"github.com/lf-edge/ekuiper/internal/topo/topotest"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/msgpack-rpc/msgpack-rpc-go/rpc"
	"google.golang.org/grpc"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strconv"
	"testing"
)

type HelloRequest struct {
	Name string `json:"name,omitempty"`
}

type HelloReply struct {
	Message string `json:"message,omitempty"`
}

type ObjectDetectRequest struct {
	Command string `json:"cmd,omitempty"`
	Image   string `json:"base64_img,omitempty"`
}

type ObjectDetectResponse struct {
	Info   string `json:"cmd,omitempty"`
	Code   int    `json:"base64_img,omitempty"`
	Image  string `json:"image,omitempty"`
	Result string `json:"result,omitempty"`
	Type   string `json:"type,omitempty"`
}

//type Box struct {
//	X int32 `json:"x,omitempty"`
//	Y int32 `json:"y,omitempty"`
//	W int32 `json:"w,omitempty"`
//	H int32 `json:"h,omitempty"`
//}

//type FeatureResult struct {
//	Features []float64 `json:"features,omitempty"`
//	Box      Box       `json:"box,omitempty"`
//}

type EncodedRequest struct {
	Name string `json:"name,omitempty"`
	Size int    `json:"size,omitempty"`
}

type ShelfMessage struct {
	Id    string `json:"id,omitempty"`
	Theme string `json:"theme,omitempty"`
}

type ShelfMessageOut struct {
	Id    int64  `json:"id,omitempty"`
	Theme string `json:"theme,omitempty"`
}

type BookMessage struct {
	Id     int64  `json:"id,omitempty"`
	Author string `json:"author,omitempty"`
	Title  string `json:"title,omitempty"`
}

type MessageMessage struct {
	Text string `json:"text,omitempty"`
}

func TestRestService(t *testing.T) {
	// mock server, the port is set in the sample.json
	l, err := net.Listen("tcp", "127.0.0.1:51234")
	if err != nil {
		t.Error(err)
		t.FailNow()
	}
	count := 0
	router := mux.NewRouter()
	router.HandleFunc("/SayHello", func(w http.ResponseWriter, r *http.Request) {
		body := &HelloRequest{}
		err := json.NewDecoder(r.Body).Decode(body)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		out := &HelloReply{Message: body.Name}
		jsonOut(w, err, out)
	}).Methods(http.MethodPost)
	router.HandleFunc("/object_detection", func(w http.ResponseWriter, r *http.Request) {
		req := &ObjectDetectRequest{}
		err := json.NewDecoder(r.Body).Decode(req)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		if req.Image == "" {
			http.Error(w, "image is not found", http.StatusBadRequest)
		}
		out := &ObjectDetectResponse{
			Info:   req.Command,
			Code:   200,
			Image:  req.Image,
			Result: req.Command + " success",
			Type:   "S",
		}
		jsonOut(w, err, out)
	}).Methods(http.MethodPost)
	router.HandleFunc("/getStatus", func(w http.ResponseWriter, r *http.Request) {
		result := count%2 == 0
		count++
		io.WriteString(w, fmt.Sprintf("%v", result))
	}).Methods(http.MethodPost)
	router.HandleFunc("/RestEncodedJson", func(w http.ResponseWriter, r *http.Request) {
		req := &EncodedRequest{}
		err := json.NewDecoder(r.Body).Decode(req)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		io.WriteString(w, req.Name)
	}).Methods(http.MethodPost)
	router.HandleFunc("/bookshelf/v1/shelves", func(w http.ResponseWriter, r *http.Request) {
		req := &ShelfMessage{}
		err := json.NewDecoder(r.Body).Decode(req)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		if req.Id == "" || req.Theme == "" {
			http.Error(w, "empty request", http.StatusBadRequest)
		}
		idint, _ := strconv.Atoi(req.Id)
		out := ShelfMessageOut{Id: int64(idint), Theme: req.Theme}
		jsonOut(w, err, out)
	}).Methods(http.MethodPost)
	router.HandleFunc("/bookshelf/v1/shelves/{shelf}/books/{book}", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		vars := mux.Vars(r)
		shelf, book := vars["shelf"], vars["book"]
		if shelf == "" || book == "" {
			http.Error(w, "empty request", http.StatusBadRequest)
		}
		idint, _ := strconv.Atoi(book)
		out := BookMessage{Id: int64(idint), Author: "NA", Title: "title_" + book}
		jsonOut(w, err, out)
	}).Methods(http.MethodGet)
	router.HandleFunc("/messaging/v1/messages/{name}", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		vars := mux.Vars(r)
		name := vars["name"]
		if name == "" {
			http.Error(w, "empty request", http.StatusBadRequest)
		}
		out := MessageMessage{Text: name + " content"}
		jsonOut(w, err, out)
	}).Methods(http.MethodGet)
	router.HandleFunc("/messaging/v1/messages/filter/{name}", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		vars := mux.Vars(r)
		name := vars["name"]
		q := r.URL.Query()
		rev, sub := q.Get("revision"), q.Get("sub.subfield")
		if name == "" || rev == "" || sub == "" {
			http.Error(w, "empty request", http.StatusBadRequest)
		}
		out := MessageMessage{Text: name + rev + sub}
		jsonOut(w, err, out)
	}).Methods(http.MethodGet)
	router.HandleFunc("/messaging/v1/messages/{name}", func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		vars := mux.Vars(r)
		name := vars["name"]
		if name == "" {
			http.Error(w, "empty request", http.StatusBadRequest)
		}
		body := &MessageMessage{}
		err := json.NewDecoder(r.Body).Decode(body)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		out := MessageMessage{Text: body.Text}
		jsonOut(w, err, out)
	}).Methods(http.MethodPut, http.MethodPatch)
	server := httptest.NewUnstartedServer(router)
	server.Listener.Close()
	server.Listener = l

	// Start the server.
	server.Start()

	defer server.Close()
	//Reset
	streamList := []string{"helloStr", "commands", "fakeBin", "shelves", "demo", "mes"}
	topotest.HandleStream(false, streamList, t)
	//Data setup
	var tests = []topotest.RuleTest{
		{
			Name: `TestRestRule1`,
			Sql:  `SELECT helloFromRest(name) as wc FROM helloStr`,
			R: [][]map[string]interface{}{
				{{
					"wc": map[string]interface{}{
						"message": "world",
					},
				}},
				{{
					"wc": map[string]interface{}{
						"message": "golang",
					},
				}},
				{{
					"wc": map[string]interface{}{
						"message": "peacock",
					},
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule2`,
			Sql:  `SELECT objectDetectFromRest(cmd, base64_img)->result FROM commands`,
			R: [][]map[string]interface{}{
				{{
					"kuiper_field_0": "get success",
				}},
				{{
					"kuiper_field_0": "detect success",
				}},
				{{
					"kuiper_field_0": "delete success",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule3`,
			Sql:  `SELECT objectDetectFromRest(*)->result FROM commands`,
			R: [][]map[string]interface{}{
				{{
					"kuiper_field_0": "get success",
				}},
				{{
					"kuiper_field_0": "detect success",
				}},
				{{
					"kuiper_field_0": "delete success",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
			//}, {
			//	Name: `TestRestRule3`,
			//	Sql:  `SELECT getFeatureFromRest(self)->feature[0]->box->h FROM fakeBin`,
			//	R: [][]map[string]interface{}{
			//		{{
			//			"kuiper_field_0": 106,
			//		}},
			//		{{
			//			"kuiper_field_0": 107,
			//		}},
			//		{{
			//			"kuiper_field_0": 108,
			//		}},
			//	},
			//	M: map[string]interface{}{
			//		"op_2_project_0_exceptions_total":   int64(0),
			//		"op_2_project_0_process_latency_us": int64(0),
			//		"op_2_project_0_records_in_total":   int64(3),
			//		"op_2_project_0_records_out_total":  int64(3),
			//
			//		"sink_mockSink_0_exceptions_total":  int64(0),
			//		"sink_mockSink_0_records_in_total":  int64(3),
			//		"sink_mockSink_0_records_out_total": int64(3),
			//	},
		}, {
			Name: `TestRestRule4`,
			Sql:  `SELECT getStatusFromRest(), cmd FROM commands`,
			R: [][]map[string]interface{}{
				{{
					"getStatusFromRest": true,
					"cmd":               "get",
				}},
				{{
					"getStatusFromRest": false,
					"cmd":               "detect",
				}},
				{{
					"getStatusFromRest": true,
					"cmd":               "delete",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule5`,
			Sql:  `SELECT restEncodedJson(encoded_json) as name FROM commands`,
			R: [][]map[string]interface{}{
				{{
					"name": "name1",
				}},
				{{
					"name": "name2",
				}},
				{{
					"name": "name3",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule6`,
			Sql:  `SELECT CreateShelf(shelf)->theme as theme FROM shelves`,
			R: [][]map[string]interface{}{
				{{
					"theme": "tandra",
				}},
				{{
					"theme": "claro",
				}},
				{{
					"theme": "dark",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule7`,
			Sql:  `SELECT GetBook(size, ts)->title as title FROM demo WHERE size > 3 `,
			R: [][]map[string]interface{}{
				{{
					"title": "title_1541152486822",
				}},
				{{
					"title": "title_1541152488442",
				}},
			},
			M: map[string]interface{}{
				"op_2_filter_0_exceptions_total":   int64(0),
				"op_2_filter_0_process_latency_us": int64(0),
				"op_2_filter_0_records_in_total":   int64(5),
				"op_2_filter_0_records_out_total":  int64(2),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(2),
				"sink_mockSink_0_records_out_total": int64(2),
			},
		}, {
			Name: `TestRestRule8`,
			Sql:  `SELECT GetMessage(concat("messages/",ts))->text as message FROM demo WHERE size > 3`,
			R: [][]map[string]interface{}{
				{{
					"message": "1541152486822 content",
				}},
				{{
					"message": "1541152488442 content",
				}},
			},
			M: map[string]interface{}{
				"op_2_filter_0_exceptions_total":   int64(0),
				"op_2_filter_0_process_latency_us": int64(0),
				"op_2_filter_0_records_in_total":   int64(5),
				"op_2_filter_0_records_out_total":  int64(2),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(2),
				"sink_mockSink_0_records_out_total": int64(2),
			},
		}, {
			Name: `TestRestRule9`,
			Sql:  `SELECT SearchMessage(name, size, shelf)->text as message FROM shelves`,
			R: [][]map[string]interface{}{
				{{
					"message": "name12sub1",
				}},
				{{
					"message": "name23sub2",
				}},
				{{
					"message": "name34sub3",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
			// TODO support * as one of the parameters
			//},{
			//	Name: `TestRestRule10`,
			//	Sql:  `SELECT UpdateMessage(message_id, *)->text as message FROM mes`,
			//	R: [][]map[string]interface{}{
			//		{{
			//			"message": "message1",
			//		}},
			//		{{
			//			"message": "message2",
			//		}},
			//		{{
			//			"message": "message3",
			//		}},
			//	},
			//	M: map[string]interface{}{
			//		"op_2_project_0_exceptions_total":   int64(0),
			//		"op_2_project_0_process_latency_us": int64(0),
			//		"op_2_project_0_records_in_total":   int64(3),
			//		"op_2_project_0_records_out_total":  int64(3),
			//
			//		"sink_mockSink_0_exceptions_total":  int64(0),
			//		"sink_mockSink_0_records_in_total":  int64(3),
			//		"sink_mockSink_0_records_out_total": int64(3),
			//	},
		}, {
			Name: `TestRestRule11`,
			Sql:  `SELECT PatchMessage(message_id, text)->text as message FROM mes`,
			R: [][]map[string]interface{}{
				{{
					"message": "message1",
				}},
				{{
					"message": "message2",
				}},
				{{
					"message": "message3",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		},
	}
	topotest.HandleStream(true, streamList, t)
	topotest.DoRuleTest(t, tests, 0, &api.RuleOption{
		BufferLength: 100,
		SendError:    true,
	}, 0)
}

func jsonOut(w http.ResponseWriter, err error, out interface{}) {
	w.Header().Add("Content-Type", "application/json")
	enc := json.NewEncoder(w)
	err = enc.Encode(out)
	// Problems encoding
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
	}
}

type Resolver map[string]reflect.Value

func (self Resolver) Resolve(name string, _ []reflect.Value) (reflect.Value, error) {
	return self[name], nil
}

func SayHello(name string) map[string]interface{} {
	return map[string]interface{}{
		"message": name,
	}
}

func get_feature(img []byte) map[string]interface{} {
	l := len(string(img))
	return map[string]interface{}{
		"feature": []map[string]interface{}{
			{
				"features": []float64{-1.444, 2.55452, 5.121},
				"box": map[string]interface{}{
					"x": 153,
					"y": 107,
					"w": 174,
					"h": 100 + l,
				},
			}, {
				"features": []float64{1.444, -2.55452, -5.121},
				"box": map[string]interface{}{
					"x": 257,
					"y": 92,
					"w": 169,
					"h": 208,
				},
			},
		},
	}
}

func object_detection(command string, image string) map[string]interface{} {
	out := map[string]interface{}{
		"info":   command,
		"code":   200,
		"image":  image,
		"result": command + " success",
		"type":   "S",
	}
	return out
}

func getStatus() bool {
	return true
}

func TestMsgpackService(t *testing.T) {
	// mock server
	res := Resolver{"SayHello": reflect.ValueOf(SayHello), "object_detection": reflect.ValueOf(object_detection), "get_feature": reflect.ValueOf(get_feature), "getStatus": reflect.ValueOf(getStatus)}
	serv := rpc.NewServer(res, true, nil)
	l, _ := net.Listen("tcp", ":50000")
	serv.Listen(l)
	go serv.Run()
	// Comment out because the bug in the msgpack rpc
	// defer serv.Stop()

	//Reset
	streamList := []string{"helloStr", "commands", "fakeBin"}
	topotest.HandleStream(false, streamList, t)
	//Data setup
	var tests = []topotest.RuleTest{
		{
			Name: `TestRestRule1`,
			Sql:  `SELECT helloFromMsgpack(name) as wc FROM helloStr`,
			R: [][]map[string]interface{}{
				{{
					"wc": map[string]interface{}{
						"message": "world",
					},
				}},
				{{
					"wc": map[string]interface{}{
						"message": "golang",
					},
				}},
				{{
					"wc": map[string]interface{}{
						"message": "peacock",
					},
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule2`,
			Sql:  `SELECT objectDetectFromMsgpack(*)->result FROM commands`,
			R: [][]map[string]interface{}{
				{{
					"kuiper_field_0": "get success",
				}},
				{{
					"kuiper_field_0": "detect success",
				}},
				{{
					"kuiper_field_0": "delete success",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule3`,
			Sql:  `SELECT getFeatureFromMsgpack(self)->feature[0]->box->h FROM fakeBin`,
			R: [][]map[string]interface{}{
				{{
					"kuiper_field_0": float64(106), //Convert by the testing tool
				}},
				{{
					"kuiper_field_0": float64(107),
				}},
				{{
					"kuiper_field_0": float64(108),
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
			//}, {
			//	Name: `TestRestRule4`,
			//	Sql:  `SELECT getStatusFromMsgpack(), command FROM commands`,
			//	R: [][]map[string]interface{}{
			//		{{
			//			"getStatusFromRest": true,
			//			"command": "get",
			//		}},
			//		{{
			//			"getStatusFromRest": true,
			//			"command": "detect",
			//		}},
			//		{{
			//			"getStatusFromRest": true,
			//			"command": "delete",
			//		}},
			//	},
			//	M: map[string]interface{}{
			//		"op_2_project_0_exceptions_total":   int64(0),
			//		"op_2_project_0_process_latency_us": int64(0),
			//		"op_2_project_0_records_in_total":   int64(3),
			//		"op_2_project_0_records_out_total":  int64(3),
			//
			//		"sink_mockSink_0_exceptions_total":  int64(0),
			//		"sink_mockSink_0_records_in_total":  int64(3),
			//		"sink_mockSink_0_records_out_total": int64(3),
			//	},
		},
	}
	topotest.HandleStream(true, streamList, t)
	topotest.DoRuleTest(t, tests, 0, &api.RuleOption{
		BufferLength: 100,
		SendError:    true,
	}, 0)
}

type server struct {
	pb.UnimplementedGreeterServer
}

func (s *server) SayHello(_ context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
	return &pb.HelloReply{Message: in.GetName()}, nil
}

func (s *server) ObjectDetection(_ context.Context, in *pb.ObjectDetectionRequest) (*pb.ObjectDetectionResponse, error) {
	return &pb.ObjectDetectionResponse{
		Info:   in.Cmd,
		Code:   200,
		Image:  in.Base64Img,
		Result: in.Cmd + " success",
		Type:   "S",
	}, nil
}

func (s *server) GetFeature(_ context.Context, v *wrappers.BytesValue) (*pb.FeatureResponse, error) {
	l := len(string(v.Value))
	return &pb.FeatureResponse{
		Feature: []*pb.FeatureResult{
			{
				Features: []float32{-1.444, 2.55452, 5.121},
				Box: &pb.Box{
					X: 153,
					Y: 107,
					W: 174,
					H: int32(100 + l),
				},
			},
			{
				Features: []float32{1.444, -2.55452, -5.121},
				Box: &pb.Box{
					X: 257,
					Y: 92,
					W: 169,
					H: 208,
				},
			},
		},
	}, nil
}

func (s *server) GetStatus(context.Context, *empty.Empty) (*wrappers.BoolValue, error) {
	return &wrappers.BoolValue{Value: true}, nil
}

func TestGrpcService(t *testing.T) {
	lis, err := net.Listen("tcp", ":50051")
	if err != nil {
		kconf.Log.Fatalf("failed to listen: %v", err)
	}
	s := grpc.NewServer()
	pb.RegisterGreeterServer(s, &server{})
	go func() {
		if err := s.Serve(lis); err != nil {
			kconf.Log.Fatalf("failed to serve: %v", err)
		}
	}()
	defer s.Stop()

	//Reset
	streamList := []string{"helloStr", "commands", "fakeBin"}
	topotest.HandleStream(false, streamList, t)
	//Data setup
	var tests = []topotest.RuleTest{
		{
			Name: `TestRestRule1`,
			Sql:  `SELECT helloFromGrpc(name) as wc FROM helloStr`,
			R: [][]map[string]interface{}{
				{{
					"wc": map[string]interface{}{
						"message": "world",
					},
				}},
				{{
					"wc": map[string]interface{}{
						"message": "golang",
					},
				}},
				{{
					"wc": map[string]interface{}{
						"message": "peacock",
					},
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule2`,
			Sql:  `SELECT objectDetectFromGrpc(cmd, base64_img)->result FROM commands`,
			R: [][]map[string]interface{}{
				{{
					"kuiper_field_0": "get success",
				}},
				{{
					"kuiper_field_0": "detect success",
				}},
				{{
					"kuiper_field_0": "delete success",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule3`,
			Sql:  `SELECT getFeatureFromGrpc(self)->feature[0]->box->h FROM fakeBin`,
			R: [][]map[string]interface{}{
				{{
					"kuiper_field_0": float64(106), //Convert by the testing tool
				}},
				{{
					"kuiper_field_0": float64(107),
				}},
				{{
					"kuiper_field_0": float64(108),
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestRestRule4`,
			Sql:  `SELECT getStatusFromGrpc(), cmd FROM commands`,
			R: [][]map[string]interface{}{
				{{
					"getStatusFromGrpc": true,
					"cmd":               "get",
				}},
				{{
					"getStatusFromGrpc": true,
					"cmd":               "detect",
				}},
				{{
					"getStatusFromGrpc": true,
					"cmd":               "delete",
				}},
			},
			M: map[string]interface{}{
				"op_2_project_0_exceptions_total":   int64(0),
				"op_2_project_0_process_latency_us": int64(0),
				"op_2_project_0_records_in_total":   int64(3),
				"op_2_project_0_records_out_total":  int64(3),

				"sink_mockSink_0_exceptions_total":  int64(0),
				"sink_mockSink_0_records_in_total":  int64(3),
				"sink_mockSink_0_records_out_total": int64(3),
			},
		},
	}
	topotest.HandleStream(true, streamList, t)
	topotest.DoRuleTest(t, tests, 0, &api.RuleOption{
		BufferLength: 100,
		SendError:    true,
	}, 0)
}


<|start_filename|>internal/topo/checkpoint/defs.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package checkpoint

import (
	"github.com/lf-edge/ekuiper/pkg/api"
)

type StreamTask interface {
	Broadcast(data interface{}) error
	GetName() string
	GetStreamContext() api.StreamContext
	SetQos(api.Qos)
}

type NonSourceTask interface {
	StreamTask
	GetInputCount() int
	AddInputCount()

	SetBarrierHandler(BarrierHandler)
}

type SinkTask interface {
	NonSourceTask

	SaveCache()
}

type BufferOrEvent struct {
	Data    interface{}
	Channel string
}

type StreamCheckpointContext interface {
	Snapshot() error
	SaveState(checkpointId int64) error
}

type Message int

const (
	STOP Message = iota
	ACK
	DEC
)

type Signal struct {
	Message Message
	Barrier
}

type Barrier struct {
	CheckpointId int64
	OpId         string
}


<|start_filename|>internal/xsql/checkAgg.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package xsql

import (
	"github.com/lf-edge/ekuiper/internal/binder/function"
	"github.com/lf-edge/ekuiper/pkg/ast"
)

// IsAggregate check if an expression is aggregate with the binding alias info
func IsAggregate(expr ast.Expr) (r bool) {
	ast.WalkFunc(expr, func(n ast.Node) bool {
		switch f := n.(type) {
		case *ast.Call:
			if ok := function.IsAggFunc(f.Name); ok {
				r = true
				return false
			}
		case *ast.FieldRef:
			// lazy calculate
			if getOrCalculateAgg(f) {
				r = true
				return false
			}
		}
		return true
	})
	return
}

func getOrCalculateAgg(f *ast.FieldRef) bool {
	if f.IsAlias() {
		p := f.IsAggregate
		if p == nil {
			tr := IsAggregate(f.Expression)
			p = &tr
			f.IsAggregate = p
		}
		return *p
	}
	return false
}

func IsAggStatement(stmt *ast.SelectStatement) bool {
	if stmt.Dimensions != nil {
		ds := stmt.Dimensions.GetGroups()
		if ds != nil && len(ds) > 0 {
			return true
		}
	}
	r := false
	ast.WalkFunc(stmt.Fields, func(n ast.Node) bool {
		switch f := n.(type) {
		case *ast.Call:
			if ok := function.IsAggFunc(f.Name); ok {
				r = true
				return false
			}
		}
		return true
	})
	return r
}

func HasAggFuncs(node ast.Node) bool {
	if node == nil {
		return false
	}
	var r = false
	ast.WalkFunc(node, func(n ast.Node) bool {
		if f, ok := n.(*ast.Call); ok {
			if ok := function.IsAggFunc(f.Name); ok {
				r = true
				return false
			}
		}
		return true
	})
	return r
}


<|start_filename|>sdk/go/runtime/shared.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package runtime

const (
	TYPE_SOURCE = "source"
	TYPE_SINK   = "sink"
	TYPE_FUNC   = "func"
)

type Meta struct {
	RuleId     string `json:"ruleId"`
	OpId       string `json:"opId"`
	InstanceId int    `json:"instanceId"`
}

type FuncMeta struct {
	Meta
	FuncId int `json:"funcId"`
}

type Control struct {
	SymbolName string                 `json:"symbolName"`
	Meta       *Meta                  `json:"meta,omitempty"`
	PluginType string                 `json:"pluginType"`
	DataSource string                 `json:"dataSource,omitempty"`
	Config     map[string]interface{} `json:"config,omitempty"`
}

type Command struct {
	Cmd string `json:"cmd"`
	Arg string `json:"arg"`
}

const (
	CMD_START = "start"
	CMD_STOP  = "stop"
)

const (
	REPLY_OK = "ok"
)

type PortableConfig struct {
	SendTimeout int64 `json:"sendTimeout"`
}

type FuncData struct {
	Func string      `json:"func"`
	Arg  interface{} `json:"arg"`
}

type FuncReply struct {
	State  bool        `json:"state"`
	Result interface{} `json:"result"`
}


<|start_filename|>internal/service/model.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

type (
	protocol string
	schema   string
)

const (
	REST    protocol = "rest"
	GRPC             = "grpc"
	MSGPACK          = "msgpack-rpc"
)

const (
	PROTOBUFF schema = "protobuf"
)

type (
	author struct {
		Name    string `json:"name"`
		Email   string `json:"email"`
		Company string `json:"company"`
		Website string `json:"website"`
	}
	fileLanguage struct {
		English string `json:"en_US"`
		Chinese string `json:"zh_CN"`
	}
	about struct {
		Author      *author       `json:"author"`
		HelpUrl     *fileLanguage `json:"helpUrl"`
		Description *fileLanguage `json:"description"`
	}
	mapping struct {
		Name        string        `json:"name"`
		ServiceName string        `json:"serviceName"`
		Description *fileLanguage `json:"description"`
	}
	binding struct {
		Name        string                 `json:"name"`
		Description *fileLanguage          `json:"description"`
		Address     string                 `json:"address"`
		Protocol    protocol               `json:"protocol"`
		SchemaType  schema                 `json:"schemaType"`
		SchemaFile  string                 `json:"schemaFile"`
		Functions   []*mapping             `json:"functions"`
		Options     map[string]interface{} `json:"options"`
	}

	conf struct {
		About      *about              `json:"about"`
		Interfaces map[string]*binding `json:"interfaces"`
	}
)

// The external function's location, currently service.interface.
type serviceInfo struct {
	About      *about
	Interfaces map[string]*interfaceInfo
}

type schemaInfo struct {
	SchemaType schema
	SchemaFile string
}

type interfaceInfo struct {
	Desc      *fileLanguage
	Addr      string
	Protocol  protocol
	Schema    *schemaInfo
	Functions []string
	Options   map[string]interface{}
}

type restOption struct {
	InsecureSkipVerify bool              `json:"insecureSkipVerify"`
	Headers            map[string]string `json:"headers"`
}

type functionContainer struct {
	ServiceName   string
	InterfaceName string
	MethodName    string
}

type FunctionExec struct {
	Protocol protocol
	Addr     string
}


<|start_filename|>sdk/go/connection/connection.go<|end_filename|>
// Copyright 2022 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package connection

import (
	"fmt"
	"github.com/lf-edge/ekuiper/sdk/go/api"
	"github.com/lf-edge/ekuiper/sdk/go/context"
	"go.nanomsg.org/mangos/v3"
	"go.nanomsg.org/mangos/v3/protocol/pull"
	"go.nanomsg.org/mangos/v3/protocol/push"
	"go.nanomsg.org/mangos/v3/protocol/req"
	_ "go.nanomsg.org/mangos/v3/transport/ipc"
	"time"
)

// Options Initialized in plugin.go Start according to the config
var Options map[string]interface{}

type Closable interface {
	Close() error
}

type ReplyFunc func([]byte) []byte

type ControlChannel interface {
	// reply with string message
	Run(ReplyFunc) error
	Closable
}

type DataInChannel interface {
	Recv() ([]byte, error)
	Closable
}

type DataOutChannel interface {
	Send([]byte) error
	Closable
}

type DataInOutChannel interface {
	Run(ReplyFunc) error
	Closable
}

type NanomsgRepChannel struct {
	sock mangos.Socket
}

// Run until process end
func (r *NanomsgRepChannel) Run(f ReplyFunc) error {
	err := r.sock.Send([]byte("handshake"))
	if err != nil {
		return fmt.Errorf("can't send handshake: %s", err.Error())
	}
	for {
		msg, err := r.sock.Recv()
		if err != nil {
			return fmt.Errorf("cannot receive on rep socket: %s", err.Error())
		}
		reply := f(msg)
		err = r.sock.Send(reply)
		if err != nil {
			return fmt.Errorf("can't send reply: %s", err.Error())
		}
	}
	return nil
}

func (r *NanomsgRepChannel) Close() error {
	return r.sock.Close()
}

func CreateControlChannel(pluginName string) (ControlChannel, error) {
	var (
		sock mangos.Socket
		err  error
	)
	if sock, err = req.NewSocket(); err != nil {
		return nil, fmt.Errorf("can't get new req socket: %s", err)
	}
	setSockOptions(sock)
	sock.SetOption(mangos.OptionRetryTime, 0)
	url := fmt.Sprintf("ipc:///tmp/plugin_%s.ipc", pluginName)
	if err = sock.Dial(url); err != nil {
		return nil, fmt.Errorf("can't dial on req socket: %s", err.Error())
	}
	return &NanomsgRepChannel{sock: sock}, nil
}

func CreateSourceChannel(ctx api.StreamContext) (DataOutChannel, error) {
	var (
		sock mangos.Socket
		err  error
	)
	if sock, err = push.NewSocket(); err != nil {
		return nil, fmt.Errorf("can't get new push socket: %s", err)
	}
	setSockOptions(sock)
	url := fmt.Sprintf("ipc:///tmp/%s_%s_%d.ipc", ctx.GetRuleId(), ctx.GetOpId(), ctx.GetInstanceId())
	if err = sock.Dial(url); err != nil {
		return nil, fmt.Errorf("can't dial on push socket: %s", err.Error())
	}
	return sock, nil
}

func CreateFuncChannel(symbolName string) (DataInOutChannel, error) {
	var (
		sock mangos.Socket
		err  error
	)
	if sock, err = req.NewSocket(); err != nil {
		return nil, fmt.Errorf("can't get new req socket: %s", err)
	}
	setSockOptions(sock)
	sock.SetOption(mangos.OptionRetryTime, 0)
	url := fmt.Sprintf("ipc:///tmp/func_%s.ipc", symbolName)
	if err = sock.Dial(url); err != nil {
		return nil, fmt.Errorf("can't dial on req socket: %s", err.Error())
	}
	return &NanomsgRepChannel{sock: sock}, nil
}

func CreateSinkChannel(ctx api.StreamContext) (DataInChannel, error) {
	var (
		sock mangos.Socket
		err  error
	)
	if sock, err = pull.NewSocket(); err != nil {
		return nil, fmt.Errorf("can't get new pull socket: %s", err)
	}
	setSockOptions(sock)
	url := fmt.Sprintf("ipc:///tmp/%s_%s_%d.ipc", ctx.GetRuleId(), ctx.GetOpId(), ctx.GetInstanceId())
	if err = listenWithRetry(sock, url); err != nil {
		return nil, fmt.Errorf("can't listen on pull socket for %s: %s", url, err.Error())
	}
	return sock, nil
}

func setSockOptions(sock mangos.Socket) {
	for k, v := range Options {
		sock.SetOption(k, v)
	}
}

func listenWithRetry(sock mangos.Socket, url string) error {
	var (
		retryCount    = 300
		retryInterval = 10
	)
	for {
		err := sock.Listen(url)
		if err == nil {
			context.Log.Infof("plugin start to listen after %d tries", retryCount)
			return err
		}
		retryCount--
		if retryCount < 0 {
			return err
		}
		time.Sleep(time.Duration(retryInterval) * time.Millisecond)
	}
}


<|start_filename|>sdk/go/runtime/source.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package runtime

import (
	context2 "context"
	"fmt"
	"github.com/lf-edge/ekuiper/sdk/go/api"
	"github.com/lf-edge/ekuiper/sdk/go/connection"
)

// lifecycle controlled by plugin
// if stop by error, inform plugin

type sourceRuntime struct {
	s      api.Source
	ch     connection.DataOutChannel
	ctx    api.StreamContext
	cancel context2.CancelFunc
	key    string
}

func setupSourceRuntime(con *Control, s api.Source) (*sourceRuntime, error) {
	// init context with args
	ctx, err := parseContext(con)
	// TODO check cmd error handling or using health check
	if err != nil {
		return nil, err
	}
	// init config with args and call source config
	err = s.Configure(con.DataSource, con.Config)
	if err != nil {
		return nil, err
	}
	// connect to mq server
	ch, err := connection.CreateSourceChannel(ctx)
	if err != nil {
		return nil, err
	}
	ctx.GetLogger().Info("Setup message pipeline, start sending")
	ctx, cancel := ctx.WithCancel()
	return &sourceRuntime{
		s:      s,
		ch:     ch,
		ctx:    ctx,
		cancel: cancel,
		key:    fmt.Sprintf("%s_%s_%d_%s", con.Meta.RuleId, con.Meta.OpId, con.Meta.InstanceId, con.SymbolName),
	}, nil
}

func (s *sourceRuntime) run() {
	errCh := make(chan error)
	consumer := make(chan api.SourceTuple)
	go s.s.Open(s.ctx, consumer, errCh)
	for {
		select {
		case err := <-errCh:
			s.ctx.GetLogger().Errorf("%v", err)
			broadcast(s.ctx, s.ch, err)
			s.stop()
		case data := <-consumer:
			s.ctx.GetLogger().Debugf("broadcast data %v", data)
			broadcast(s.ctx, s.ch, data)
		case <-s.ctx.Done():
			s.s.Close(s.ctx)
			return
		}
	}
}

func (s *sourceRuntime) stop() error {
	s.cancel()
	err := s.ch.Close()
	if err != nil {
		s.ctx.GetLogger().Info(err)
	}
	s.ctx.GetLogger().Info("closed source data channel")
	reg.Delete(s.key)
	return nil
}

func (s *sourceRuntime) isRunning() bool {
	return s.ctx.Err() == nil
}


<|start_filename|>internal/topo/node/sink_node_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/conf"
	"github.com/lf-edge/ekuiper/internal/topo/context"
	"github.com/lf-edge/ekuiper/internal/topo/topotest/mocknode"
	"github.com/lf-edge/ekuiper/internal/xsql"
	"reflect"
	"testing"
	"time"
)

func TestSinkTemplate_Apply(t *testing.T) {
	conf.InitConf()
	var tests = []struct {
		config map[string]interface{}
		data   []map[string]interface{}
		result [][]byte
	}{
		{
			config: map[string]interface{}{
				"sendSingle":   true,
				"dataTemplate": `{"wrapper":"w1","content":{{toJson .}},"ab":"{{.ab}}"}`,
			},
			data:   []map[string]interface{}{{"ab": "hello1"}, {"ab": "hello2"}},
			result: [][]byte{[]byte(`{"wrapper":"w1","content":{"ab":"hello1"},"ab":"hello1"}`), []byte(`{"wrapper":"w1","content":{"ab":"hello2"},"ab":"hello2"}`)},
		}, {
			config: map[string]interface{}{
				"dataTemplate": `{"wrapper":"arr","content":{{json .}},"content0":{{json (index . 0)}},ab0":"{{index . 0 "ab"}}"}`,
			},
			data:   []map[string]interface{}{{"ab": "hello1"}, {"ab": "hello2"}},
			result: [][]byte{[]byte(`{"wrapper":"arr","content":[{"ab":"hello1"},{"ab":"hello2"}],"content0":{"ab":"hello1"},ab0":"hello1"}`)},
		}, {
			config: map[string]interface{}{
				"dataTemplate": `<div>results</div><ul>{{range .}}<li>{{.ab}}</li>{{end}}</ul>`,
			},
			data:   []map[string]interface{}{{"ab": "hello1"}, {"ab": "hello2"}},
			result: [][]byte{[]byte(`<div>results</div><ul><li>hello1</li><li>hello2</li></ul>`)},
		}, {
			config: map[string]interface{}{
				"dataTemplate": `{"content":{{toJson .}}}`,
			},
			data:   []map[string]interface{}{{"ab": "hello1"}, {"ab": "hello2"}},
			result: [][]byte{[]byte(`{"content":[{"ab":"hello1"},{"ab":"hello2"}]}`)},
		}, {
			config: map[string]interface{}{
				"sendSingle":   true,
				"dataTemplate": `{"newab":"{{.ab}}"}`,
			},
			data:   []map[string]interface{}{{"ab": "hello1"}, {"ab": "hello2"}},
			result: [][]byte{[]byte(`{"newab":"hello1"}`), []byte(`{"newab":"hello2"}`)},
		}, {
			config: map[string]interface{}{
				"sendSingle":   true,
				"dataTemplate": `{"newab":"{{.ab}}"}`,
			},
			data:   []map[string]interface{}{{"ab": "hello1"}, {"ab": "hello2"}},
			result: [][]byte{[]byte(`{"newab":"hello1"}`), []byte(`{"newab":"hello2"}`)},
		}, {
			config: map[string]interface{}{
				"sendSingle":   true,
				"dataTemplate": `{"__meta":{{toJson .__meta}},"temp":{{.temperature}}}`,
			},
			data:   []map[string]interface{}{{"temperature": 33, "humidity": 70, "__meta": xsql.Metadata{"messageid": 45, "other": "mock"}}},
			result: [][]byte{[]byte(`{"__meta":{"messageid":45,"other":"mock"},"temp":33}`)},
		}, {
			config: map[string]interface{}{
				"dataTemplate": `[{"__meta":{{toJson (index . 0 "__meta")}},"temp":{{index . 0 "temperature"}}}]`,
			},
			data:   []map[string]interface{}{{"temperature": 33, "humidity": 70, "__meta": xsql.Metadata{"messageid": 45, "other": "mock"}}},
			result: [][]byte{[]byte(`[{"__meta":{"messageid":45,"other":"mock"},"temp":33}]`)},
		}, {
			config: map[string]interface{}{
				"dataTemplate": `[{{range $index, $ele := .}}{{if $index}},{{end}}{"result":{{add $ele.temperature $ele.humidity}}}{{end}}]`,
			},
			data:   []map[string]interface{}{{"temperature": 33, "humidity": 70}, {"temperature": 22.0, "humidity": 50}, {"temperature": 11, "humidity": 90}},
			result: [][]byte{[]byte(`[{"result":103},{"result":72},{"result":101}]`)},
		}, {
			config: map[string]interface{}{
				"dataTemplate": `{{$counter := 0}}{{range $index, $ele := .}}{{if ne 90 $ele.humidity}}{{$counter = add $counter 1}}{{end}}{{end}}{"result":{{$counter}}}`,
			},
			data:   []map[string]interface{}{{"temperature": 33, "humidity": 70}, {"temperature": 22.0, "humidity": 50}, {"temperature": 11, "humidity": 90}},
			result: [][]byte{[]byte(`{"result":2}`)},
		}, {
			config: map[string]interface{}{
				"dataTemplate": `{"a":"{{base64 .a}}","b":"{{base64 .b}}","c":"{{b64enc .c}}","d":"{{b64enc .d}}","e":"{{base64 .e}}"}`,
				"sendSingle":   true,
			},
			data:   []map[string]interface{}{{"a": 1, "b": 3.1415, "c": "hello", "d": "{\"hello\" : 3}", "e": map[string]interface{}{"humidity": 20, "temperature": 30}}},
			result: [][]byte{[]byte(`{"a":"MQ==","b":"My4xNDE1","c":"aGVsbG8=","d":"eyJoZWxsbyIgOiAzfQ==","e":"<KEY>="}`)},
		},
	}
	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	contextLogger := conf.Log.WithField("rule", "TestSinkTemplate_Apply")
	ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger)

	for i, tt := range tests {
		mockSink := mocknode.NewMockSink()
		s := NewSinkNodeWithSink("mockSink", mockSink, tt.config)
		s.Open(ctx, make(chan error))
		s.input <- tt.data
		time.Sleep(1 * time.Second)
		s.close(ctx, contextLogger)
		results := mockSink.GetResults()
		if !reflect.DeepEqual(tt.result, results) {
			t.Errorf("%d \tresult mismatch:\n\nexp=%s\n\ngot=%s\n\n", i, tt.result, results)
		}
	}
}


<|start_filename|>internal/binder/io/binder_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package io

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/binder"
	"github.com/lf-edge/ekuiper/internal/binder/mock"
	"testing"
)

func TestBindings(t *testing.T) {
	m := mock.NewMockFactory()
	e := binder.FactoryEntry{
		Name:    "mock",
		Factory: m,
	}
	err := Initialize([]binder.FactoryEntry{e})
	if err != nil {
		t.Error(err)
		return
	}
	var tests = []struct {
		name     string
		isSource bool
		isSink   bool
	}{
		{
			name:     "unknown",
			isSource: false,
			isSink:   false,
		}, {
			name:     "mqtt",
			isSource: true,
			isSink:   true,
		}, {
			name:     "mock1",
			isSource: true,
			isSink:   true,
		}, {
			name:     "rest",
			isSource: false,
			isSink:   true,
		},
	}
	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	for _, tt := range tests {
		_, err := Source(tt.name)
		isSource := err == nil
		if tt.isSource != isSource {
			t.Errorf("%s is source: expect %v but got %v", tt.name, tt.isSource, isSource)
		}
		_, err = Sink(tt.name)
		isSink := err == nil
		if tt.isSink != isSink {
			t.Errorf("%s is sink: expect %v but got %v", tt.name, tt.isSink, isSink)
		}
	}
}


<|start_filename|>internal/pkg/db/database.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package db

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/pkg/db/redis"
	"github.com/lf-edge/ekuiper/internal/pkg/db/sql/sqlite"
)

type Database interface {
	Connect() error
	Disconnect() error
}

func CreateDatabase(conf Config) (Database, error) {
	var db Database
	var err error
	databaseType := conf.Type
	switch databaseType {
	case "redis":
		r := redis.NewRedisFromConf(conf.Redis)
		db = &r
	case "sqlite":
		err, db = sqlite.NewSqliteDatabase(conf.Sqlite)
		if err != nil {
			return nil, err
		}
	default:
		return nil, fmt.Errorf("unrecognized database type - %s", databaseType)
	}
	err = db.Connect()
	if err != nil {
		return nil, err
	}
	return db, nil
}


<|start_filename|>sdk/go/example/mirror/sinks/file.json<|end_filename|>
{
	"about": {
		"trial": true,
		"author": {
			"name": "EMQ",
			"email": "<EMAIL>",
			"company": "EMQ Technologies Co., Ltd",
			"website": "https://www.emqx.io"
		},
		"helpUrl": {
			"en_US": "https://github.com/lf-edge/ekuiper/blob/master/docs/en_US/plugins/sinks/file.md",
			"zh_CN": "https://github.com/lf-edge/ekuiper/blob/master/docs/zh_CN/plugins/sinks/file.md"
		},
		"description": {
			"en_US": "This a sink plugin for file, it can be used for saving the analysis data into file system.",
			"zh_CN": "本插件为文件持久化插件，可以用于将分析数据存入指定的文件中。"
		}
	},
	"libs": [
	],
	"properties": [{
		"name": "path",
		"default": "",
		"optional": false,
		"control": "text",
		"type": "string",
		"hint": {
			"en_US": "The file path for saving the result",
			"zh_CN": "保存结果的文件路径"
		},
		"label": {
			"en_US": "Path of file",
			"zh_CN": "文件路径"
		}
	}, {
		"name": "interval",
		"default": 1000,
		"optional": true,
		"control": "text",
		"type": "int",
		"hint": {
			"en_US": "The time interval (ms) for writing the analysis result.",
			"zh_CN": "写入分析结果的时间间隔（毫秒）。"
		},
		"label": {
			"en_US": "Intervals",
			"zh_CN": "间隔时间"
		}
	}]
}


<|start_filename|>internal/topo/planner/dataSourcePlan.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package planner

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/conf"
	"github.com/lf-edge/ekuiper/pkg/ast"
	"github.com/lf-edge/ekuiper/pkg/message"
	"sort"
	"strings"
)

type DataSourcePlan struct {
	baseLogicalPlan
	name ast.StreamName
	// calculated properties
	// initialized with stream definition, pruned with rule
	streamFields []interface{}
	metaFields   []string
	// passon properties
	streamStmt      *ast.StreamStmt
	allMeta         bool
	isBinary        bool
	iet             bool
	timestampFormat string
	timestampField  string
	// intermediate status
	isWildCard bool
	fields     map[string]interface{}
	metaMap    map[string]string
}

func (p DataSourcePlan) Init() *DataSourcePlan {
	p.baseLogicalPlan.self = &p
	return &p
}

// Presume no children for data source
func (p *DataSourcePlan) PushDownPredicate(condition ast.Expr) (ast.Expr, LogicalPlan) {
	owned, other := p.extract(condition)
	if owned != nil {
		// Add a filter plan for children
		f := FilterPlan{
			condition: owned,
		}.Init()
		f.SetChildren([]LogicalPlan{p})
		return other, f
	}
	return other, p
}

func (p *DataSourcePlan) extract(expr ast.Expr) (ast.Expr, ast.Expr) {
	s, hasDefault := getRefSources(expr)
	l := len(s)
	if hasDefault {
		l += 1
	}
	switch len(s) {
	case 0:
		return expr, nil
	case 1:
		if s[0] == p.name || s[0] == ast.DefaultStream {
			return expr, nil
		} else {
			return nil, expr
		}
	default:
		if be, ok := expr.(*ast.BinaryExpr); ok && be.OP == ast.AND {
			ul, pl := p.extract(be.LHS)
			ur, pr := p.extract(be.RHS)
			owned := combine(ul, ur)
			other := combine(pl, pr)
			return owned, other
		}
		return nil, expr
	}
}

func (p *DataSourcePlan) PruneColumns(fields []ast.Expr) error {
	//init values
	err := p.getProps()
	if err != nil {
		return err
	}
	p.fields = make(map[string]interface{})
	if !p.allMeta {
		p.metaMap = make(map[string]string)
	}
	if p.timestampField != "" {
		p.fields[p.timestampField] = p.timestampField
	}
	for _, field := range fields {
		switch f := field.(type) {
		case *ast.Wildcard:
			p.isWildCard = true
		case *ast.FieldRef:
			if !p.isWildCard && (f.StreamName == ast.DefaultStream || f.StreamName == p.name) {
				if _, ok := p.fields[f.Name]; !ok {
					sf := p.getField(f.Name)
					if sf != nil {
						p.fields[f.Name] = sf
					}
				}
			}
		case *ast.MetaRef:
			if p.allMeta {
				break
			}
			if f.StreamName == ast.DefaultStream || f.StreamName == p.name {
				if f.Name == "*" {
					p.allMeta = true
					p.metaMap = nil
				} else if !p.allMeta {
					p.metaMap[strings.ToLower(f.Name)] = f.Name
				}
			}
		case *ast.SortField:
			if !p.isWildCard {
				sf := p.getField(f.Name)
				if sf != nil {
					p.fields[f.Name] = sf
				}
			}
		default:
			return fmt.Errorf("unsupported field %v", field)
		}
	}
	p.getAllFields()
	return nil
}

func (p *DataSourcePlan) getField(name string) interface{} {
	if p.streamStmt.StreamFields != nil {
		for _, f := range p.streamStmt.StreamFields { // The input can only be StreamFields
			if f.Name == name {
				return &f
			}
		}
	} else {
		return name
	}
	return nil
}

func (p *DataSourcePlan) getAllFields() {
	// convert fields
	p.streamFields = make([]interface{}, 0)
	if p.isWildCard {
		if p.streamStmt.StreamFields != nil {
			for k := range p.streamStmt.StreamFields { // The input can only be StreamFields
				p.streamFields = append(p.streamFields, &p.streamStmt.StreamFields[k])
			}
		} else {
			p.streamFields = nil
		}
	} else {
		sfs := make([]interface{}, 0, len(p.fields))
		if conf.IsTesting {
			var keys []string
			for k := range p.fields {
				keys = append(keys, k)
			}
			sort.Strings(keys)
			for _, k := range keys {
				sfs = append(sfs, p.fields[k])
			}
		} else {
			for _, v := range p.fields {
				sfs = append(sfs, v)
			}
		}
		p.streamFields = sfs
	}
	p.metaFields = make([]string, 0, len(p.metaMap))
	for _, v := range p.metaMap {
		p.metaFields = append(p.metaFields, v)
	}
	// for consistency of results for testing
	sort.Strings(p.metaFields)
	p.fields = nil
	p.metaMap = nil
}

func (p *DataSourcePlan) getProps() error {
	if p.iet {
		if p.streamStmt.Options.TIMESTAMP != "" {
			p.timestampField = p.streamStmt.Options.TIMESTAMP
		} else {
			return fmt.Errorf("preprocessor is set to be event time but stream option TIMESTAMP not found")
		}
	}
	if p.streamStmt.Options.TIMESTAMP_FORMAT != "" {
		p.timestampFormat = p.streamStmt.Options.TIMESTAMP_FORMAT
	}
	if strings.EqualFold(p.streamStmt.Options.FORMAT, message.FormatBinary) {
		p.isBinary = true
	}
	return nil
}


<|start_filename|>internal/plugin/portable/manager_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package portable

import (
	"errors"
	"fmt"
	"github.com/lf-edge/ekuiper/internal/plugin"
	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
	"net/http"
	"net/http/httptest"
	"os"
	"path"
	"path/filepath"
	"reflect"
	"testing"
)

// Test only install API. Install from file is tested in the integration test in test/portable_rule_test

func init() {
	InitManager()
}

func TestManager_Install(t *testing.T) {
	s := httptest.NewServer(
		http.FileServer(http.Dir("../testzips")),
	)
	defer s.Close()
	endpoint := s.URL

	data := []struct {
		n   string
		u   string
		v   string
		err error
	}{
		{ // 0
			n:   "",
			u:   "",
			err: errors.New("invalid name : should not be empty"),
		}, { // 1
			n:   "zipMissJson",
			u:   endpoint + "/functions/misc.zip",
			err: errors.New("fail to install plugin: missing or invalid json file zipMissJson.json"),
		}, { // 2
			n:   "urlerror",
			u:   endpoint + "/sinks/nozip",
			err: errors.New("invalid uri " + endpoint + "/sinks/nozip"),
		}, { // 3
			n:   "wrong",
			u:   endpoint + "/portables/wrong.zip",
			err: errors.New("fail to install plugin: missing mirror.exe"),
		}, { // 4
			n:   "wrongname",
			u:   endpoint + "/portables/mirror.zip",
			err: errors.New("fail to install plugin: missing or invalid json file wrongname.json"),
		}, { // 5
			n: "mirror2",
			u: endpoint + "/portables/mirror.zip",
		},
	}

	fmt.Printf("The test bucket size is %d.\n\n", len(data))
	for i, tt := range data {
		p := &plugin.IOPlugin{
			Name: tt.n,
			File: tt.u,
		}
		err := manager.Register(p)
		if !reflect.DeepEqual(tt.err, err) {
			t.Errorf("%d: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.err, err)
		} else if tt.err == nil {
			err := checkFileForMirror(manager.pluginDir, manager.etcDir, true)
			if err != nil {
				t.Errorf("%d: error : %s\n\n", i, err)
			}
		}
	}
}

func TestManager_Read(t *testing.T) {
	expPlugins := []*PluginInfo{
		{
			PluginMeta: runtime.PluginMeta{
				Name:       "mirror2",
				Version:    "v1.0.0",
				Language:   "go",
				Executable: filepath.Clean(path.Join(manager.pluginDir, "mirror2", "mirror2")),
			},
			Sources:   []string{"randomGo"},
			Sinks:     []string{"fileGo"},
			Functions: []string{"echoGo"},
		},
	}
	result := manager.List()
	if len(result) != 3 {
		t.Errorf("list result mismatch:\n  exp=%v\n  got=%v\n\n", expPlugins, result)
	}

	_, ok := manager.GetPluginInfo("mirror3")
	if ok {
		t.Error("find inexist plugin mirror3")
	}
	pi, ok := manager.GetPluginInfo("mirror2")
	if !ok {
		t.Error("can't find plugin mirror2")
	}
	if !reflect.DeepEqual(expPlugins[0], pi) {
		t.Errorf("Get plugin mirror2 mismatch:\n exp=%v\n got=%v", expPlugins[0], pi)
	}
	_, ok = manager.GetPluginMeta(plugin.SOURCE, "echoGo")
	if ok {
		t.Error("find inexist source symbol echo")
	}
	m, ok := manager.GetPluginMeta(plugin.SINK, "fileGo")
	if !ok {
		t.Error("can't find sink symbol fileGo")
	}
	if !reflect.DeepEqual(&(expPlugins[0].PluginMeta), m) {
		t.Errorf("Get sink symbol mismatch:\n exp=%v\n got=%v", expPlugins[0].PluginMeta, m)
	}
}

// This will start channel, so test it in integration tests.
//func TestFactory(t *testing.T){
//	_, err := manager.Source("alss")
//	expErr := fmt.Errorf("can't find random")
//	if !reflect.DeepEqual(expErr, err){
//		t.Errorf("error mismatch:\n  exp=%s\n  got=%s\n\n", expErr, err)
//	}
//	src, _ := manager.Source("randomGo")
//	if src  == nil {
//		t.Errorf("can't get source randomGo")
//	}
//	snk, _ := manager.Sink("fileGo")
//	if snk == nil {
//		t.Errorf("can't get sink fileGo")
//	}
//	fun, _ := manager.Function("echoGo")
//	if fun == nil {
//		t.Errorf("can't get function echoGo")
//	}
//	ok := manager.HasFunctionSet("echoGo")
//	if !ok {
//		t.Errorf("can't check function set")
//	}
//}

func TestDelete(t *testing.T) {
	err := manager.Delete("mirror2")
	if err != nil {
		t.Errorf("delete plugin error: %v", err)
	}
	err = checkFileForMirror(manager.pluginDir, manager.etcDir, false)
	if err != nil {
		t.Errorf("error : %s\n\n", err)
	}
}

func checkFileForMirror(pluginDir, etcDir string, exist bool) error {
	requiredFiles := []string{
		path.Join(pluginDir, "mirror2", "mirror2"),
		path.Join(pluginDir, "mirror2", "mirror2.json"),
		path.Join(etcDir, "sources", "randomGo.yaml"),
		path.Join(etcDir, "sources", "randomGo.json"),
		path.Join(etcDir, "functions", "echoGo.json"),
		path.Join(etcDir, "sinks", "fileGo.json"),
	}
	for _, file := range requiredFiles {
		_, err := os.Stat(file)
		if exist && err != nil {
			return err
		} else if !exist && err == nil {
			return fmt.Errorf("file still exists: %s", file)
		}
	}
	return nil
}


<|start_filename|>internal/binder/function/funcs_math.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package function

import (
	"fmt"
	"math"
	"math/rand"
)

func mathCall(name string, args []interface{}) (interface{}, bool) {
	switch name {
	case "abs":
		if v, ok := args[0].(int); ok {
			t := float64(v)
			var ret = int(math.Abs(t))
			return ret, true
		} else if v, ok := args[0].(float64); ok {
			return math.Abs(v), true
		} else {
			return fmt.Errorf("only float64 & int type are supported"), false
		}
	case "acos":
		if v, e := toF64(args[0]); e == nil {
			return math.Acos(v), true
		} else {
			return e, false
		}
	case "asin":
		if v, e := toF64(args[0]); e == nil {
			return math.Asin(v), true
		} else {
			return e, false
		}
	case "atan":
		if v, e := toF64(args[0]); e == nil {
			return math.Atan(v), true
		} else {
			return e, false
		}
	case "atan2":
		if v1, e := toF64(args[0]); e == nil {
			if v2, e1 := toF64(args[1]); e1 == nil {
				return math.Atan2(v1, v2), true
			} else {
				return e1, false
			}
		} else {
			return e, false
		}
	case "bitand":
		v1, ok1 := args[0].(int)
		v2, ok2 := args[1].(int)
		if ok1 && ok2 {
			return v1 & v2, true
		} else {
			return fmt.Errorf("Expect int type for both operands."), false
		}
	case "bitor":
		v1, ok1 := args[0].(int)
		v2, ok2 := args[1].(int)
		if ok1 && ok2 {
			return v1 | v2, true
		} else {
			return fmt.Errorf("Expect int type for both operands."), false
		}
	case "bitxor":
		v1, ok1 := args[0].(int)
		v2, ok2 := args[1].(int)
		if ok1 && ok2 {
			return v1 ^ v2, true
		} else {
			return fmt.Errorf("Expect int type for both operands."), false
		}
	case "bitnot":
		v1, ok1 := args[0].(int)
		if ok1 {
			return ^v1, true
		} else {
			return fmt.Errorf("Expect int type for operand."), false
		}
	case "ceil":
		if v, e := toF64(args[0]); e == nil {
			return math.Ceil(v), true
		} else {
			return e, false
		}
	case "cos":
		if v, e := toF64(args[0]); e == nil {
			return math.Cos(v), true
		} else {
			return e, false
		}
	case "cosh":
		if v, e := toF64(args[0]); e == nil {
			return math.Cosh(v), true
		} else {
			return e, false
		}
	case "exp":
		if v, e := toF64(args[0]); e == nil {
			return math.Exp(v), true
		} else {
			return e, false
		}
	case "ln":
		if v, e := toF64(args[0]); e == nil {
			return math.Log2(v), true
		} else {
			return e, false
		}
	case "log":
		if v, e := toF64(args[0]); e == nil {
			return math.Log10(v), true
		} else {
			return e, false
		}
	case "mod":
		if v, e := toF64(args[0]); e == nil {
			if v1, e1 := toF64(args[1]); e == nil {
				return math.Mod(v, v1), true
			} else {
				return e1, false
			}
		} else {
			return e, false
		}
	case "power":
		if v1, e := toF64(args[0]); e == nil {
			if v2, e2 := toF64(args[1]); e2 == nil {
				return math.Pow(v1, v2), true
			} else {
				return e2, false
			}
		} else {
			return e, false
		}
	case "rand":
		return rand.Float64(), true
	case "round":
		if v, e := toF64(args[0]); e == nil {
			return math.Round(v), true
		} else {
			return e, false
		}
	case "sign":
		if v, e := toF64(args[0]); e == nil {
			if v > 0 {
				return 1, true
			} else if v < 0 {
				return -1, true
			} else {
				return 0, true
			}
		} else {
			return e, false
		}
	case "sin":
		if v, e := toF64(args[0]); e == nil {
			return math.Sin(v), true
		} else {
			return e, false
		}
	case "sinh":
		if v, e := toF64(args[0]); e == nil {
			return math.Sinh(v), true
		} else {
			return e, false
		}
	case "sqrt":
		if v, e := toF64(args[0]); e == nil {
			return math.Sqrt(v), true
		} else {
			return e, false
		}

	case "tan":
		if v, e := toF64(args[0]); e == nil {
			return math.Tan(v), true
		} else {
			return e, false
		}

	case "tanh":
		if v, e := toF64(args[0]); e == nil {
			return math.Tanh(v), true
		} else {
			return e, false
		}
	}

	return fmt.Errorf("Unknown math function name."), false
}

func toF64(arg interface{}) (float64, error) {
	if v, ok := arg.(float64); ok {
		return v, nil
	} else if v, ok := arg.(int); ok {
		return float64(v), nil
	}
	return 0, fmt.Errorf("only float64 & int type are supported")
}


<|start_filename|>internal/plugin/portable/test/portable_rule_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"bufio"
	"context"
	"encoding/json"
	"fmt"
	"github.com/lf-edge/ekuiper/internal/binder"
	"github.com/lf-edge/ekuiper/internal/binder/function"
	"github.com/lf-edge/ekuiper/internal/binder/io"
	"github.com/lf-edge/ekuiper/internal/plugin/portable"
	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
	"github.com/lf-edge/ekuiper/internal/processor"
	"github.com/lf-edge/ekuiper/internal/topo"
	"github.com/lf-edge/ekuiper/internal/topo/planner"
	"github.com/lf-edge/ekuiper/internal/topo/topotest"
	"github.com/lf-edge/ekuiper/pkg/api"
	"log"
	"os"
	"reflect"
	"testing"
	"time"
)

func init() {
	m, err := portable.InitManager()
	if err != nil {
		panic(err)
	}
	entry := binder.FactoryEntry{Name: "portable plugin", Factory: m}
	err = function.Initialize([]binder.FactoryEntry{entry})
	if err != nil {
		panic(err)
	}
	err = io.Initialize([]binder.FactoryEntry{entry})
	if err != nil {
		panic(err)
	}
}

func TestSourceAndFunc(t *testing.T) {
	streamList := []string{"ext", "extpy"}
	topotest.HandleStream(false, streamList, t)
	var tests = []struct {
		Name string
		Rule string
		R    [][]map[string]interface{}
		M    map[string]interface{}
	}{
		{
			Name: `TestPortableRule1`,
			Rule: `{"sql":"SELECT echo(count) as ee FROM ext","actions":[{"file":{"path":"cache1"}}]}`,
			R: [][]map[string]interface{}{
				{{
					"ee": float64(50),
				}},
				{{
					"ee": float64(50),
				}},
				{{
					"ee": float64(50),
				}},
			},
			M: map[string]interface{}{
				"source_ext_0_exceptions_total":   int64(0),
				"source_ext_0_records_in_total":   int64(3),
				"source_ext_0_records_out_total":  int64(3),
				"sink_file_0_0_records_out_total": int64(3),
			},
		}, {
			Name: `TestPythonRule`,
			Rule: `{"sql":"SELECT revert(name) as ee FROM extpy","actions":[{"file":{"path":"cache2"}},{"print":{}}]}`,
			R: [][]map[string]interface{}{
				{{
					"ee": "nosjyp",
				}},
				{{
					"ee": "nosjyp",
				}},
				{{
					"ee": "nosjyp",
				}},
			},
			M: map[string]interface{}{
				"source_extpy_0_exceptions_total":  int64(0),
				"source_extpy_0_records_in_total":  int64(3),
				"source_extpy_0_records_out_total": int64(3),
				"sink_file_0_0_records_out_total":  int64(3),
				"sink_print_1_0_records_out_total": int64(3),
			},
		},
	}

	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	defer runtime.GetPluginInsManager().KillAll()
	for i, tt := range tests {
		_ = os.Remove(fmt.Sprintf("cache%d", i+1))
		topotest.HandleStream(true, streamList[i:i+1], t)
		rs, err := CreateRule(tt.Name, tt.Rule)
		if err != nil {
			t.Errorf("failed to create rule: %s.", err)
			continue
		}
		tp, err := planner.Plan(rs)
		if err != nil {
			t.Errorf("fail to init rule: %v", err)
			continue
		}
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		go func(ctx context.Context) {
			select {
			case err := <-tp.Open():
				log.Println(err)
				tp.Cancel()
			case <-ctx.Done():
				log.Printf("ctx done %v\n", ctx.Err())
				tp.Cancel()
			}
			fmt.Println("all exit")
		}(ctx)
		topotest.HandleStream(false, streamList[i:i+1], t)
		for {
			if ctx.Err() != nil {
				t.Errorf("Exiting with error %v", ctx.Err())
				break
			}
			time.Sleep(10 * time.Millisecond)
			if compareMetrics(tp, tt.M) {
				cancel()
				// need to wait for file results
				time.Sleep(100 * time.Millisecond)
				results := getResults(fmt.Sprintf("cache%d", i+1))
				fmt.Printf("get results %v\n", results)
				time.Sleep(10 * time.Millisecond)
				var mm [][]map[string]interface{}
				for i, v := range results {
					if i >= 3 {
						break
					}
					var mapRes []map[string]interface{}
					err := json.Unmarshal([]byte(v), &mapRes)
					if err != nil {
						t.Errorf("Failed to parse the input into map")
						continue
					}
					mm = append(mm, mapRes)
				}
				if !reflect.DeepEqual(tt.R, mm) {
					t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.Rule, tt.R, mm)
				}
				break
			}
		}
	}
	// wait for rule clean up
	time.Sleep(1 * time.Second)
}

func compareMetrics(tp *topo.Topo, m map[string]interface{}) bool {
	keys, values := tp.GetMetrics()
	for k, v := range m {
		var (
			index   int
			key     string
			matched bool
		)
		for index, key = range keys {
			if k == key {
				va, ok := values[index].(int64)
				if !ok {
					continue
				}
				ve := v.(int64)
				if va < ve {
					return false
				}
				matched = true
			}
		}
		if !matched {
			return false
		}
	}
	return true
}

func getResults(fileName string) []string {
	f, err := os.Open(fileName)
	if err != nil {
		panic(err)
	}
	result := make([]string, 0)
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		result = append(result, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		panic(err)
	}
	f.Close()
	return result
}

func CreateRule(name, sql string) (*api.Rule, error) {
	p := processor.NewRuleProcessor()
	p.ExecDrop(name)
	return p.ExecCreate(name, sql)
}


<|start_filename|>internal/topo/sink/mqtt_sink.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sink

import (
	"crypto/tls"
	"fmt"
	MQTT "github.com/eclipse/paho.mqtt.golang"
	"github.com/google/uuid"
	"github.com/lf-edge/ekuiper/internal/conf"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/cast"
	"github.com/lf-edge/ekuiper/pkg/errorx"
	"strings"
)

type MQTTSink struct {
	srv      string
	tpc      string
	clientid string
	pVersion uint
	qos      byte
	uName    string
	password string
	certPath string
	pkeyPath string
	conSel   string

	insecureSkipVerify bool
	retained           bool

	conn MQTT.Client
}

func (ms *MQTTSink) hasKeys(str []string, ps map[string]interface{}) bool {
	for _, v := range str {
		if _, ok := ps[v]; ok {
			return true
		}
	}
	return false
}

func (ms *MQTTSink) Configure(ps map[string]interface{}) error {
	conSelector := ""
	if pk, ok := ps["connectionSelector"]; ok {
		if v, ok := pk.(string); ok {
			conSelector = v
		}
		keys := []string{"server", "clientId", "protocolVersion", "username", "password", "certificationPath", "privateKeyPath", "insecureSkipVerify"}
		if ms.hasKeys(keys, ps) {
			return fmt.Errorf("already have connection selector: %s, remove connection related config", conSelector)
		}
		ms.conSel = conSelector
	} else {

		srv := ""
		if pk, ok := ps["server"]; ok {
			if v, ok := pk.(string); ok {
				srv = v
			}
		} else {
			return fmt.Errorf("mqtt sink is missing property server")
		}

		clientid, ok := ps["clientId"]
		if !ok {
			if uuid, err := uuid.NewUUID(); err != nil {
				return fmt.Errorf("mqtt sink fails to get uuid, the error is %s", err)
			} else {

				clientid = uuid.String()
			}
		}
		var pVersion uint = 3
		pVersionStr, ok := ps["protocolVersion"]
		if ok {
			v, _ := pVersionStr.(string)
			if v == "3.1" {
				pVersion = 3
			} else if v == "3.1.1" {
				pVersion = 4
			} else {
				return fmt.Errorf("unknown protocol version %s, the value could be only 3.1 or 3.1.1 (also refers to MQTT version 4)", pVersionStr)
			}
		}

		uName := ""
		un, ok := ps["username"]
		if ok {
			v, _ := un.(string)
			if strings.Trim(v, " ") != "" {
				uName = v
			}
		}

		password := ""
		pwd, ok := ps["password"]
		if ok {
			v, _ := pwd.(string)
			if strings.Trim(v, " ") != "" {
				password = v
			}
		}

		certPath := ""
		if cp, ok := ps["certificationPath"]; ok {
			if v, ok := cp.(string); ok {
				certPath = v
			}
		}

		pKeyPath := ""
		if pk, ok := ps["privateKeyPath"]; ok {
			if v, ok := pk.(string); ok {
				pKeyPath = v
			}
		}

		insecureSkipVerify := false
		if pk, ok := ps["insecureSkipVerify"]; ok {
			if v, ok := pk.(bool); ok {
				insecureSkipVerify = v
			}
		}

		ms.srv = srv
		ms.clientid = clientid.(string)
		ms.pVersion = pVersion

		ms.uName = uName
		ms.password = password
		ms.certPath = certPath
		ms.pkeyPath = pKeyPath
		ms.insecureSkipVerify = insecureSkipVerify
	}

	tpc, ok := ps["topic"]
	if !ok {
		return fmt.Errorf("mqtt sink is missing property topic")
	}

	var qos byte = 0
	if qosRec, ok := ps["qos"]; ok {
		if v, err := cast.ToInt(qosRec, cast.STRICT); err == nil {
			qos = byte(v)
		}
		if qos != 0 && qos != 1 && qos != 2 {
			return fmt.Errorf("not valid qos value %v, the value could be only int 0 or 1 or 2", qos)
		}
	}

	retained := false
	if pk, ok := ps["retained"]; ok {
		if v, ok := pk.(bool); ok {
			retained = v
		}
	}

	ms.qos = qos
	ms.tpc = tpc.(string)
	ms.retained = retained

	return nil
}

func (ms *MQTTSink) Open(ctx api.StreamContext) error {
	var client MQTT.Client
	log := ctx.GetLogger()
	if ms.conSel != "" {
		con, err := ctx.GetConnection(ms.conSel)
		if err != nil {
			log.Errorf("The mqtt client for connection selector %s get fail with error: %s", ms.conSel, err)
			return err
		}
		client = con.(MQTT.Client)
		log.Infof("The mqtt client for connection selector %s get successfully", ms.conSel)
	} else {
		log.Infof("Opening mqtt sink for rule %s.", ctx.GetRuleId())
		opts := MQTT.NewClientOptions().AddBroker(ms.srv).SetClientID(ms.clientid)

		if ms.certPath != "" || ms.pkeyPath != "" {
			log.Infof("Connect MQTT broker with certification and keys.")
			if cp, err := conf.ProcessPath(ms.certPath); err == nil {
				if kp, err1 := conf.ProcessPath(ms.pkeyPath); err1 == nil {
					if cer, err2 := tls.LoadX509KeyPair(cp, kp); err2 != nil {
						return err2
					} else {
						opts.SetTLSConfig(&tls.Config{Certificates: []tls.Certificate{cer}, InsecureSkipVerify: ms.insecureSkipVerify})
					}
				} else {
					return err1
				}
			} else {
				return err
			}
		} else {
			log.Infof("Connect MQTT broker with username and password.")
			if ms.uName != "" {
				opts = opts.SetUsername(ms.uName)
			}

			if ms.password != "" {
				opts = opts.SetPassword(<PASSWORD>)
			}
		}

		opts.SetAutoReconnect(true)
		var reconn = false
		opts.SetConnectionLostHandler(func(client MQTT.Client, e error) {
			log.Errorf("The connection %s is disconnected due to error %s, will try to re-connect later.", ms.srv+": "+ms.clientid, e)
			ms.conn = client
			reconn = true
		})

		opts.SetOnConnectHandler(func(client MQTT.Client) {
			if reconn {
				log.Infof("The connection is %s re-established successfully.", ms.srv+": "+ms.clientid)
			}
		})

		client = MQTT.NewClient(opts)
		if token := client.Connect(); token.Wait() && token.Error() != nil {
			return fmt.Errorf("Found error: %s", token.Error())
		}
		log.Infof("The connection to server %s:%d was established successfully", ms.srv, ms.clientid)
	}

	ms.conn = client
	return nil
}

func (ms *MQTTSink) Collect(ctx api.StreamContext, item interface{}) error {
	logger := ctx.GetLogger()
	jsonBytes, _, err := ctx.TransformOutput(item)
	if err != nil {
		return err
	}
	c := ms.conn
	logger.Debugf("%s publish %s", ctx.GetOpId(), jsonBytes)
	tpc, err := ctx.ParseDynamicProp(ms.tpc, item)
	if err != nil {
		return err
	}
	if tpc, ok := tpc.(string); !ok {
		return fmt.Errorf("the value %v of dynamic prop %s for topic is not a string", ms.tpc, tpc)
	}
	if token := c.Publish(tpc.(string), ms.qos, ms.retained, jsonBytes); token.Wait() && token.Error() != nil {
		return fmt.Errorf("%s: %s", errorx.IOErr, token.Error())
	}
	return nil
}

func (ms *MQTTSink) Close(ctx api.StreamContext) error {
	logger := ctx.GetLogger()
	logger.Infof("Closing mqtt sink")
	if ms.conn != nil && ms.conn.IsConnected() && ms.conSel == "" {
		ms.conn.Disconnect(5000)
	}
	if ms.conSel != "" {
		ctx.ReleaseConnection(ms.conSel)
	}
	return nil
}


<|start_filename|>extensions/functions/image/resize.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"fmt"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/nfnt/resize"
	"image"
	"image/jpeg"
	"image/png"
)

type imageResize struct {
}

func (f *imageResize) Validate(args []interface{}) error {
	if len(args) != 3 {
		return fmt.Errorf("The resize function supports 3 parameters, but got %d", len(args))
	}
	return nil
}

func (f *imageResize) Exec(args []interface{}, _ api.FunctionContext) (interface{}, bool) {
	arg, ok := args[0].([]byte)
	if !ok {
		return fmt.Errorf("arg[0] is not a bytea, got %v", args[0]), false
	}
	width, ok := args[1].(int)
	if !ok || 0 > width {
		return fmt.Errorf("arg[1] is not a bigint, got %v", args[1]), false
	}
	height, ok := args[2].(int)
	if !ok || 0 > height {
		return fmt.Errorf("arg[2] is not a bigint, got %v", args[2]), false
	}
	img, format, err := image.Decode(bytes.NewReader(arg))
	if nil != err {
		return fmt.Errorf("image decode error:%v", err), false
	}
	img = resize.Resize(uint(width), uint(height), img, resize.Bilinear)

	var b []byte
	buf := bytes.NewBuffer(b)
	switch format {
	case "png":
		err = png.Encode(buf, img)
	case "jpeg":
		err = jpeg.Encode(buf, img, nil)
	default:
		return fmt.Errorf("%s image type is not currently supported", format), false
	}
	if nil != err {
		return fmt.Errorf("image encode error:%v", err), false
	}
	return buf.Bytes(), true
}

func (f *imageResize) IsAggregate() bool {
	return false
}


<|start_filename|>pkg/ast/token.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ast

type Token int

const (
	// Special Tokens
	ILLEGAL Token = iota
	EOF
	WS
	COMMENT

	AS
	// Literals
	IDENT // main

	INTEGER   // 12345
	NUMBER    //12345.67
	STRING    // "abc"
	BADSTRING // "abc

	operatorBeg
	// ADD and the following are InfluxQL Operators
	ADD         // +
	SUB         // -
	MUL         // *
	DIV         // /
	MOD         // %
	BITWISE_AND // &
	BITWISE_OR  // |
	BITWISE_XOR // ^

	AND // AND
	OR  // OR

	EQ  // =
	NEQ // !=
	LT  // <
	LTE // <=
	GT  // >
	GTE // >=

	SUBSET //[
	ARROW  //->

	operatorEnd

	// Misc characters
	ASTERISK  // *
	COMMA     // ,
	LPAREN    // (
	RPAREN    // )
	LBRACKET  //[
	RBRACKET  //]
	HASH      // #
	DOT       // .
	COLON     //:
	SEMICOLON //;
	COLSEP    //\007

	// Keywords
	SELECT
	FROM
	JOIN
	INNER
	LEFT
	RIGHT
	FULL
	CROSS
	ON
	WHERE
	GROUP
	ORDER
	HAVING
	BY
	ASC
	DESC
	FILTER
	CASE
	WHEN
	THEN
	ELSE
	END

	TRUE
	FALSE

	CREATE
	DROP
	EXPLAIN
	DESCRIBE
	SHOW
	STREAM
	TABLE
	STREAMS
	TABLES
	WITH

	XBIGINT
	XFLOAT
	XSTRING
	XBYTEA
	XDATETIME
	XBOOLEAN
	XARRAY
	XSTRUCT

	DATASOURCE
	KEY
	FORMAT
	CONF_KEY
	TYPE
	STRICT_VALIDATION
	TIMESTAMP
	TIMESTAMP_FORMAT
	RETAIN_SIZE
	SHARED

	DD
	HH
	MI
	SS
	MS
)

var Tokens = []string{
	ILLEGAL: "ILLEGAL",
	EOF:     "EOF",
	AS:      "AS",
	WS:      "WS",
	IDENT:   "IDENT",
	INTEGER: "INTEGER",
	NUMBER:  "NUMBER",
	STRING:  "STRING",

	ADD:         "+",
	SUB:         "-",
	MUL:         "*",
	DIV:         "/",
	MOD:         "%",
	BITWISE_AND: "&",
	BITWISE_OR:  "|",
	BITWISE_XOR: "^",

	EQ:  "=",
	NEQ: "!=",
	LT:  "<",
	LTE: "<=",
	GT:  ">",
	GTE: ">=",

	SUBSET: "[]",
	ARROW:  "->",

	ASTERISK: "*",
	COMMA:    ",",

	LPAREN:    "(",
	RPAREN:    ")",
	LBRACKET:  "[",
	RBRACKET:  "]",
	HASH:      "#",
	DOT:       ".",
	SEMICOLON: ";",
	COLON:     ":",
	COLSEP:    "\007",

	SELECT: "SELECT",
	FROM:   "FROM",
	JOIN:   "JOIN",
	LEFT:   "LEFT",
	INNER:  "INNER",
	ON:     "ON",
	WHERE:  "WHERE",
	GROUP:  "GROUP",
	ORDER:  "ORDER",
	HAVING: "HAVING",
	BY:     "BY",
	ASC:    "ASC",
	DESC:   "DESC",

	CREATE:   "CREATE",
	DROP:     "RROP",
	EXPLAIN:  "EXPLAIN",
	DESCRIBE: "DESCRIBE",
	SHOW:     "SHOW",
	STREAM:   "STREAM",
	TABLE:    "TABLE",
	STREAMS:  "STREAMS",
	TABLES:   "TABLES",
	WITH:     "WITH",

	XBIGINT:   "BIGINT",
	XFLOAT:    "FLOAT",
	XSTRING:   "STRING",
	XBYTEA:    "BYTEA",
	XDATETIME: "DATETIME",
	XBOOLEAN:  "BOOLEAN",
	XARRAY:    "ARRAY",
	XSTRUCT:   "STRUCT",

	DATASOURCE:        "DATASOURCE",
	KEY:               "KEY",
	FORMAT:            "FORMAT",
	CONF_KEY:          "CONF_KEY",
	TYPE:              "TYPE",
	STRICT_VALIDATION: "STRICT_VALIDATION",
	TIMESTAMP:         "TIMESTAMP",
	TIMESTAMP_FORMAT:  "TIMESTAMP_FORMAT",
	RETAIN_SIZE:       "RETAIN_SIZE",
	SHARED:            "SHARED",

	AND:   "AND",
	OR:    "OR",
	TRUE:  "TRUE",
	FALSE: "FALSE",

	DD: "DD",
	HH: "HH",
	MI: "MI",
	SS: "SS",
	MS: "MS",
}

var COLUMN_SEPARATOR = Tokens[COLSEP]

func (tok Token) String() string {
	if tok >= 0 && tok < Token(len(Tokens)) {
		return Tokens[tok]
	}
	return ""
}

func (tok Token) IsOperator() bool {
	return (tok > operatorBeg && tok < operatorEnd) || tok == ASTERISK || tok == LBRACKET
}

func (tok Token) IsTimeLiteral() bool { return tok >= DD && tok <= MS }

func (tok Token) AllowedSourceToken() bool {
	return tok == IDENT || tok == DIV || tok == HASH || tok == ADD
}

//Allowed special field name token
func (tok Token) AllowedSFNToken() bool { return tok == DOT }

func (tok Token) Precedence() int {
	switch tok {
	case OR:
		return 1
	case AND:
		return 2
	case EQ, NEQ, LT, LTE, GT, GTE:
		return 3
	case ADD, SUB, BITWISE_OR, BITWISE_XOR:
		return 4
	case MUL, DIV, MOD, BITWISE_AND, SUBSET, ARROW:
		return 5
	}
	return 0
}

type DataType int

const (
	UNKNOWN DataType = iota
	BIGINT
	FLOAT
	STRINGS
	BYTEA
	DATETIME
	BOOLEAN
	ARRAY
	STRUCT
)

var dataTypes = []string{
	BIGINT:   "bigint",
	FLOAT:    "float",
	STRINGS:  "string",
	BYTEA:    "bytea",
	DATETIME: "datetime",
	BOOLEAN:  "boolean",
	ARRAY:    "array",
	STRUCT:   "struct",
}

func (d DataType) IsSimpleType() bool {
	return d >= BIGINT && d <= BOOLEAN
}

func (d DataType) String() string {
	if d >= 0 && d < DataType(len(dataTypes)) {
		return dataTypes[d]
	}
	return ""
}

func GetDataType(tok Token) DataType {
	switch tok {
	case XBIGINT:
		return BIGINT
	case XFLOAT:
		return FLOAT
	case XSTRING:
		return STRINGS
	case XBYTEA:
		return BYTEA
	case XDATETIME:
		return DATETIME
	case XBOOLEAN:
		return BOOLEAN
	case XARRAY:
		return ARRAY
	case XSTRUCT:
		return STRUCT
	}
	return UNKNOWN
}


<|start_filename|>internal/topo/connection/client_mqtt_test.go<|end_filename|>
package connection

import (
	"github.com/lf-edge/ekuiper/internal/conf"
	"reflect"
	"testing"
)

func TestMQTTClient_CfgValidate(t *testing.T) {

	type args struct {
		props map[string]interface{}
	}
	tests := []struct {
		name    string
		args    args
		wantErr bool
	}{
		{
			name: "config pass",
			args: args{
				props: map[string]interface{}{
					"servers": []string{"tcp:127.0.0.1"},
				},
			},
			wantErr: false,
		},
		{
			name: "config are not case sensitive",
			args: args{
				props: map[string]interface{}{
					"SERVERS": []string{"tcp:127.0.0.1"},
				},
			},
			wantErr: false,
		},
		{
			name: "config server addr key error",
			args: args{
				props: map[string]interface{}{
					"server": []string{"tcp:127.0.0.1"},
				},
			},
			wantErr: true,
		},
		{
			name: "config have unwanted topic fields",
			args: args{
				props: map[string]interface{}{
					"servers": []string{"tcp:127.0.0.1"},
					"topic":   "demo",
				},
			},
			wantErr: true,
		},
		{
			name: "config no server addr",
			args: args{
				props: map[string]interface{}{
					"username": "user1",
				},
			},
			wantErr: true,
		},
		{
			name: "config no server addr",
			args: args{
				props: map[string]interface{}{
					"servers": []string{},
				},
			},
			wantErr: true,
		},
		{
			name: "config miss cert key file",
			args: args{
				props: map[string]interface{}{
					"servers":           []string{"tcp:127.0.0.1"},
					"certificationPath": "./not_exist.crt",
					"privateKeyPath":    "./not_exist.key",
				},
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			ms := &MQTTClient{
				selector: &conf.ConSelector{
					ConnSelectorStr: "testSelector",
				},
			}
			err := ms.CfgValidate(tt.args.props)
			if (err != nil) != tt.wantErr {
				t.Errorf("CfgValidate() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}

func TestMQTTClient_CfgResult(t *testing.T) {
	props := map[string]interface{}{
		"servers":  []string{"tcp:127.0.0.1:1883"},
		"USERNAME": "demo",
		"Password": "password",
		"clientID": "clientid",
	}

	ms := &MQTTClient{
		selector: &conf.ConSelector{
			ConnSelectorStr: "testSelector",
		},
	}

	_ = ms.CfgValidate(props)

	if !reflect.DeepEqual("tcp:127.0.0.1:1883", ms.srv) {
		t.Errorf("result mismatch:\n\n got=%#v\n\n", ms.srv)
	}
	if !reflect.DeepEqual("demo", ms.uName) {
		t.Errorf("result mismatch:\n\n got=%#v\n\n", ms.uName)
	}
	if !reflect.DeepEqual("password", ms.password) {
		t.Errorf("result mismatch:\n\n got=%#v\n\n", ms.password)
	}
	if !reflect.DeepEqual("clientid", ms.clientid) {
		t.Errorf("result mismatch:\n\n got=%#v\n\n", ms.clientid)
	}
	if !reflect.DeepEqual(uint(3), ms.pVersion) {
		t.Errorf("result mismatch:\n\n got=%#v\n\n", ms.pVersion)
	}

}


<|start_filename|>internal/conf/redis_store_config_test.go<|end_filename|>
package conf

import (
	"os"
	"testing"
)

func TestRedisStorageConSelectorApply(t *testing.T) {
	type args struct {
		conf        *KuiperConf
		conSelector string
	}
	tests := []struct {
		name    string
		args    args
		wantErr bool
	}{
		{
			name: "pass",
			args: args{
				conf:        &KuiperConf{},
				conSelector: "edgex.redisMsgBus",
			},
			wantErr: false,
		},
		{
			name: "do not support mqtt message bus type, fail",
			args: args{
				conf:        &KuiperConf{},
				conSelector: "edgex.mqttMsgBus",
			},
			wantErr: true,
		},
		{
			name: "not exist connection selector",
			args: args{
				conf:        &KuiperConf{},
				conSelector: "noexist.mqtt",
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if err := RedisStorageConSelectorApply(tt.args.conSelector, tt.args.conf); (err != nil) != tt.wantErr {
				t.Errorf("RedisStorageConSelectorApply() error = %v, wantErr %v", err, tt.wantErr)
			}
		})
	}
}

func TestRedisStorageConSelector(t *testing.T) {

	envs := map[string]string{
		"KUIPER__STORE__TYPE":                                "redis",
		"KUIPER__STORE__REDIS__CONNECTIONSELECTOR":           "edgex.redisMsgBus",
		"CONNECTION__EDGEX__REDISMSGBUS__SERVER":             "edgex-redis",
		"CONNECTION__EDGEX__REDISMSGBUS__OPTIONAL__PASSWORD": "password",
	}

	for key, value := range envs {
		err := os.Setenv(key, value)
		if err != nil {
			t.Error(err)
		}
	}

	InitConf()

	if Config.Store.Type != "redis" {
		t.Errorf("env variable should set it to redis")
	}
	if Config.Store.Redis.Host != "edgex-redis" {
		t.Errorf("env variable should set it to edgex-redis")
	}
	if Config.Store.Redis.Password != "password" {
		t.Errorf("env variable should set it to password")
	}
}


<|start_filename|>internal/xsql/stmtx.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package xsql

import (
	"encoding/json"
	"fmt"
	"github.com/lf-edge/ekuiper/pkg/ast"
	"github.com/lf-edge/ekuiper/pkg/errorx"
	"github.com/lf-edge/ekuiper/pkg/kv"
	"strings"
)

func GetStreams(stmt *ast.SelectStatement) (result []string) {
	if stmt == nil {
		return nil
	}
	for _, source := range stmt.Sources {
		if s, ok := source.(*ast.Table); ok {
			result = append(result, s.Name)
		}
	}

	for _, join := range stmt.Joins {
		result = append(result, join.Name)
	}
	return
}

func GetStatementFromSql(sql string) (*ast.SelectStatement, error) {
	parser := NewParser(strings.NewReader(sql))
	if stmt, err := Language.Parse(parser); err != nil {
		return nil, fmt.Errorf("Parse SQL %s error: %s.", sql, err)
	} else {
		if r, ok := stmt.(*ast.SelectStatement); !ok {
			return nil, fmt.Errorf("SQL %s is not a select statement.", sql)
		} else {
			return r, nil
		}
	}
}

type StreamInfo struct {
	StreamType ast.StreamType `json:"streamType"`
	Statement  string         `json:"statement"`
}

func GetDataSourceStatement(m kv.KeyValue, name string) (*StreamInfo, error) {
	var (
		v  string
		vs = &StreamInfo{}
	)
	if ok, _ := m.Get(name, &v); ok {
		if err := json.Unmarshal([]byte(v), vs); err != nil {
			return nil, fmt.Errorf("error unmarshall %s, the data in db may be corrupted", name)
		} else {
			return vs, nil
		}
	}
	return nil, errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("%s is not found", name))
}

func GetDataSource(m kv.KeyValue, name string) (stmt *ast.StreamStmt, err error) {
	info, err := GetDataSourceStatement(m, name)
	if err != nil {
		return nil, err
	}
	parser := NewParser(strings.NewReader(info.Statement))
	stream, err := Language.Parse(parser)
	stmt, ok := stream.(*ast.StreamStmt)
	if !ok {
		err = fmt.Errorf("Error resolving the stream %s, the data in db may be corrupted.", name)
	}
	return
}


<|start_filename|>internal/pkg/store/encoding/encoding.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package encoding

import (
	"bytes"
	"encoding/gob"
)

func Encode(value interface{}) (error, []byte) {
	var buff bytes.Buffer
	gob.Register(value)
	enc := gob.NewEncoder(&buff)
	if err := enc.Encode(value); err != nil {
		return err, nil
	}
	return nil, buff.Bytes()
}


<|start_filename|>internal/pkg/ts/redis/redisTs.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package redis

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"github.com/gomodule/redigo/redis"
	dbRedis "github.com/lf-edge/ekuiper/internal/pkg/db/redis"
	kvEncoding "github.com/lf-edge/ekuiper/internal/pkg/store/encoding"
	"strconv"
)

const (
	TsPrefix             = "KV:TS"
	AddToSortedSet       = "ZADD"
	ReversedRangeByScore = "ZREVRANGEBYSCORE"
	RemoveRangeByScore   = "ZREMRANGEBYSCORE"
	Delete               = "DEL"
	ReversedRange        = "ZREVRANGE"
)

type ts struct {
	redis dbRedis.Instance
	table string
	last  int64
	key   string
}

func init() {
	gob.Register(make(map[string]interface{}))
}

func createRedisTs(redis dbRedis.Instance, table string) (error, *ts) {
	key := fmt.Sprintf("%s:%s", TsPrefix, table)
	err, lastTs := getLast(redis, key)
	if err != nil {
		return err, nil
	}
	s := &ts{
		redis: redis,
		table: table,
		last:  lastTs,
		key:   key,
	}
	return nil, s
}

func (t *ts) Set(key int64, value interface{}) (bool, error) {
	if key <= t.last {
		return false, nil
	}
	err, b := kvEncoding.Encode(value)
	if err != nil {
		return false, err
	}
	err = t.redis.Apply(func(conn redis.Conn) error {
		reply, err := conn.Do(AddToSortedSet, t.key, key, b)
		if err != nil {
			return err
		}
		length, err := redis.Int(reply, err)
		if err != nil {
			return err
		}
		if length == 0 {
			return fmt.Errorf("list at %s key should be non empty", t.key)
		}
		t.last = key
		return nil
	})
	if err != nil {
		return false, err
	}
	return true, nil
}

func (t ts) Get(key int64, value interface{}) (bool, error) {
	err := t.redis.Apply(func(conn redis.Conn) error {
		reply, err := conn.Do(ReversedRangeByScore, t.key, key, key)
		if err != nil {
			return err
		}
		var tmp [][]byte
		tmp, err = redis.ByteSlices(reply, err)
		if err != nil {
			return err
		}
		if len(tmp) == 0 {
			return fmt.Errorf("record under %s key and %d score not found", t.key, key)
		}
		dec := gob.NewDecoder(bytes.NewBuffer(tmp[0]))
		err = dec.Decode(value)
		return err
	})
	if err != nil {
		return false, err
	}
	return true, nil
}

func (t ts) Last(value interface{}) (int64, error) {
	var last int64 = 0
	err := t.redis.Apply(func(conn redis.Conn) error {
		reply, err := conn.Do(ReversedRange, t.key, 0, 0, "WITHSCORES")
		if err != nil {
			return err
		}
		var tmp [][]byte
		tmp, err = redis.ByteSlices(reply, err)
		if err != nil {
			return err
		}
		if len(tmp) > 0 {
			dec := gob.NewDecoder(bytes.NewBuffer(tmp[0]))
			if err = dec.Decode(value); err != nil {
				return err
			}
			last, err = strconv.ParseInt(string(tmp[1]), 10, 64)
		}
		return err
	})
	if err != nil {
		return 0, err
	}
	return last, nil
}

func (t ts) Delete(key int64) error {
	return t.redis.Apply(func(conn redis.Conn) error {
		_, err := conn.Do(RemoveRangeByScore, t.key, key, key)
		return err
	})
}

func (t ts) DeleteBefore(key int64) error {
	return t.redis.Apply(func(conn redis.Conn) error {
		bound := fmt.Sprintf("(%d", key)
		_, err := conn.Do(RemoveRangeByScore, t.key, "-INF", bound)
		return err
	})
}

func (t ts) Close() error {
	return nil
}

func (t ts) Drop() error {
	return t.redis.Apply(func(conn redis.Conn) error {
		_, err := conn.Do(Delete, t.key)
		return err
	})
}

func getLast(db dbRedis.Instance, key string) (error, int64) {
	var lastTs int64
	err := db.Apply(func(conn redis.Conn) error {
		reply, err := conn.Do(ReversedRange, key, 0, 0, "WITHSCORES")
		if err != nil {
			return err
		}
		var tmp [][]byte
		tmp, err = redis.ByteSlices(reply, err)
		if err != nil {
			return err
		}
		if len(tmp) == 0 {
			return nil
		}
		lastTs, err = strconv.ParseInt(string(tmp[1]), 10, 64)
		return err
	})
	if err != nil {
		return err, 0
	}
	return nil, lastTs
}


<|start_filename|>tools/kubernetes/util/util_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestCall(t *testing.T) {
	var tests = []struct {
		cmd command
		exp bool
	}{
		{
			cmd: command{
				Url:    `/streams`,
				Method: `post`,
				Data:   struct{ sql string }{sql: `create stream stream1 (id bigint, name string, score float) WITH ( datasource = \"topic/temperature\", FORMAT = \"json\", KEY = \"id\");`},
			},
			exp: true,
		},
		{
			cmd: command{
				Url:    `/streams`,
				Method: `get`,
			},
			exp: true,
		},
		{
			cmd: command{
				Url:    `/streams/stream1`,
				Method: `put`,
				Data:   struct{ sql string }{sql: `create stream stream1 (id bigint, name string) WITH ( datasource = \"topic/temperature\", FORMAT = \"json\", KEY = \"id\");`},
			},
			exp: true,
		},
		{
			cmd: command{
				Url:    `/rules`,
				Method: `post`,
				Data: struct {
					id      string
					sql     string
					actions []struct{ log struct{} }
				}{
					id:  `ruler1`,
					sql: `SELECT * FROM stream1`,
				},
			},
			exp: true,
		},
		{
			cmd: command{
				Url:    `/rules`,
				Method: `get`,
			},
			exp: true,
		},
		{
			cmd: command{
				Url:    `/rules/rule1`,
				Method: `get`,
			},
			exp: true,
		},
		{
			cmd: command{
				Url:    `/rules/rule2`,
				Method: `delete`,
			},
			exp: true,
		},
		{
			cmd: command{
				Url:    `/rules/rule1/stop`,
				Method: `post`,
			},
			exp: true,
		},
		{
			cmd: command{
				Url:    `/rules/rule1/start`,
				Method: `post`,
			},
			exp: true,
		},
	}

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	}))
	defer ts.Close()

	for _, v := range tests {
		ret := v.cmd.call(ts.URL)
		if v.exp != ret {
			t.Errorf("url:%s method:%s log:%s\n", v.cmd.Url, v.cmd.Method, v.cmd.getLog())
		}
	}
}


<|start_filename|>internal/binder/function/function.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package function

import (
	"fmt"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/ast"
	"strings"
)

type funcType int

const (
	NotFoundFunc funcType = iota - 1
	AggFunc
	MathFunc
	StrFunc
	ConvFunc
	HashFunc
	JsonFunc
	OtherFunc
)

var maps = []map[string]string{
	aggFuncMap, mathFuncMap, strFuncMap, convFuncMap, hashFuncMap, jsonFuncMap, otherFuncMap,
}

var aggFuncMap = map[string]string{"avg": "",
	"count": "",
	"max":   "", "min": "",
	"sum":         "",
	"collect":     "",
	"deduplicate": "",
}

var funcWithAsteriskSupportMap = map[string]string{
	"collect": "",
	"count":   "",
}

var mathFuncMap = map[string]string{"abs": "", "acos": "", "asin": "", "atan": "", "atan2": "",
	"bitand": "", "bitor": "", "bitxor": "", "bitnot": "",
	"ceil": "", "cos": "", "cosh": "",
	"exp": "",
	"ln":  "", "log": "",
	"mod":   "",
	"power": "",
	"rand":  "", "round": "",
	"sign": "", "sin": "", "sinh": "", "sqrt": "",
	"tan": "", "tanh": "",
}

var strFuncMap = map[string]string{"concat": "",
	"endswith":    "",
	"format_time": "",
	"indexof":     "",
	"length":      "", "lower": "", "lpad": "", "ltrim": "",
	"numbytes":       "",
	"regexp_matches": "", "regexp_replace": "", "regexp_substr": "", "rpad": "", "rtrim": "",
	"substring": "", "startswith": "", "split_value": "",
	"trim":  "",
	"upper": "",
}

var convFuncMap = map[string]string{"concat": "", "cast": "", "chr": "",
	"encode": "",
	"trunc":  "",
}

var hashFuncMap = map[string]string{"md5": "",
	"sha1": "", "sha256": "", "sha384": "", "sha512": "",
}

var jsonFuncMap = map[string]string{
	"json_path_query": "", "json_path_query_first": "", "json_path_exists": "",
}

var otherFuncMap = map[string]string{"isnull": "",
	"newuuid": "", "tstamp": "", "mqtt": "", "meta": "", "cardinality": "",
	"window_start": "",
	"window_end":   "",
}

func getFuncType(name string) funcType {
	for i, m := range maps {
		if _, ok := m[name]; ok {
			return funcType(i)
		}
	}
	return NotFoundFunc
}

type funcExecutor struct{}

func (f *funcExecutor) ValidateWithName(args []ast.Expr, name string) error {
	var eargs []ast.Expr
	for _, arg := range args {
		if t, ok := arg.(ast.Expr); ok {
			eargs = append(eargs, t)
		} else {
			// should never happen
			return fmt.Errorf("receive invalid arg %v", arg)
		}
	}
	return validateFuncs(name, eargs)
}

func (f *funcExecutor) Validate(_ []interface{}) error {
	return fmt.Errorf("unknow name")
}

func (f *funcExecutor) Exec(_ []interface{}, _ api.FunctionContext) (interface{}, bool) {
	return fmt.Errorf("unknow name"), false
}

func (f *funcExecutor) ExecWithName(args []interface{}, ctx api.FunctionContext, name string) (interface{}, bool) {
	switch getFuncType(name) {
	case AggFunc:
		return aggCall(name, args)
	case MathFunc:
		return mathCall(name, args)
	case ConvFunc:
		return convCall(name, args)
	case StrFunc:
		return strCall(name, args)
	case HashFunc:
		return hashCall(name, args)
	case JsonFunc:
		return jsonCall(ctx, name, args)
	case OtherFunc:
		return otherCall(name, args)
	}
	return fmt.Errorf("unknow name"), false
}

func (f *funcExecutor) IsAggregate() bool {
	return false
}

func (f *funcExecutor) IsAggregateWithName(name string) bool {
	return getFuncType(name) == AggFunc
}

var staticFuncExecutor = &funcExecutor{}

type Manager struct{}

func (m *Manager) Function(name string) (api.Function, error) {
	ft := getFuncType(name)
	if ft != NotFoundFunc {
		return staticFuncExecutor, nil
	}
	return nil, nil
}

func (m *Manager) HasFunctionSet(name string) bool {
	return name == "internal"
}

func (m *Manager) ConvName(n string) (string, bool) {
	name := strings.ToLower(n)
	ft := getFuncType(name)
	if ft != NotFoundFunc {
		return name, true
	}
	return name, false
}

var m = &Manager{}

func GetManager() *Manager {
	return m
}


<|start_filename|>internal/pkg/store/setup.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"github.com/lf-edge/ekuiper/internal/conf"
	"github.com/lf-edge/ekuiper/internal/pkg/db"
	"github.com/lf-edge/ekuiper/internal/pkg/db/redis"
	"github.com/lf-edge/ekuiper/internal/pkg/db/sql/sqlite"
)

func SetupDefault() error {
	dir, err := conf.GetDataLoc()
	if err != nil {
		return err
	}

	c := db.Config{
		Type:  "sqlite",
		Redis: redis.Config{},
		Sqlite: sqlite.Config{
			Path: dir,
			Name: "",
		},
	}

	return Setup(c)
}

func SetupWithKuiperConfig(kconf *conf.KuiperConf) error {
	dir, err := conf.GetDataLoc()
	if err != nil {
		return err
	}
	c := db.Config{
		Type: kconf.Store.Type,
		Redis: redis.Config{
			Host:     kconf.Store.Redis.Host,
			Port:     kconf.Store.Redis.Port,
			Password: kconf.Store.Redis.Password,
			Timeout:  kconf.Store.Redis.Timeout,
		},
		Sqlite: sqlite.Config{
			Path: dir,
			Name: kconf.Store.Sqlite.Name,
		},
	}
	return Setup(c)
}

func Setup(config db.Config) error {
	database, err := db.CreateDatabase(config)
	if err != nil {
		return err
	}

	err = InitGlobalStores(database)
	if err != nil {
		return err
	}

	return nil
}


<|start_filename|>internal/pkg/ts/test/common/test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"github.com/lf-edge/ekuiper/pkg/kv"
	"reflect"
	"testing"
)

var (
	Keys   = []int64{1000, 1500, 2000, 3000}
	Values = []string{"bar1", "bar15", "bar2", "bar3"}
)

func TestTsSet(ks kv.Tskv, t *testing.T) {
	load(ks, t)

	if ok, err := ks.Set(2500, "bar25"); nil != err {
		t.Error(err)
	} else if ok {
		t.Errorf("should deny key 2500 while last one is 3000")
	}
}

func TestTsLast(ks kv.Tskv, t *testing.T) {
	load(ks, t)

	var v string
	if k, err := ks.Last(&v); err != nil {
		t.Error(err)
	} else if k != 3000 || v != "bar3" {
		t.Errorf("Last expect 3000/bar3 but got %d/%s", k, v)
	}
}

func TestTsGet(ks kv.Tskv, t *testing.T) {
	load(ks, t)

	var value string
	if ok, _ := ks.Get(2000, &value); ok {
		if !reflect.DeepEqual("bar2", value) {
			t.Error("expect:bar", "get:", value)
		}
	} else {
		t.Errorf("Should find key 2000.")
	}
}

func TestTsDelete(ks kv.Tskv, t *testing.T) {
	load(ks, t)

	if err := ks.Delete(1500); nil != err {
		t.Error(err)
	}

	var value string
	if ok, _ := ks.Get(1500, &value); ok {
		t.Errorf("Should not find deleted key 1500.")
	}
}

func TestTsDeleteBefore(ks kv.Tskv, t *testing.T) {
	load(ks, t)

	if ok, err := ks.Set(3500, "bar35"); nil != err {
		t.Error(err)
	} else if !ok {
		t.Error("should allow key 3500")
	}

	if err := ks.DeleteBefore(3000); nil != err {
		t.Error(err)
	}

	var value string
	if ok, _ := ks.Get(1000, &value); ok {
		t.Errorf("Should not find deleted key 1000.")
	}
	if ok, _ := ks.Get(2000, &value); ok {
		t.Errorf("Should not find deleted key 2000.")
	}

	if ok, _ := ks.Get(3000, &value); ok {
		if !reflect.DeepEqual("bar3", value) {
			t.Error("expect:bar3", "get:", value)
		}
	} else {
		t.Errorf("Should find key 3000.")
	}

	if ok, _ := ks.Get(3500, &value); ok {
		if !reflect.DeepEqual("bar35", value) {
			t.Error("expect:bar35", "get:", value)
		}
	} else {
		t.Errorf("Should find key 3500.")
	}
}

func load(ks kv.Tskv, t *testing.T) {
	for i := 0; i < len(Keys); i++ {
		k := Keys[i]
		v := Values[i]
		if ok, err := ks.Set(k, v); nil != err {
			t.Error(err)
		} else if !ok {
			t.Errorf("should allow key %d", k)
		}
	}
}


<|start_filename|>internal/conf/redis_store_config.go<|end_filename|>
package conf

import (
	"fmt"
	"github.com/lf-edge/ekuiper/pkg/cast"
)

type optional struct {
	Password string `json:"password"`
}

type edgeXConfig struct {
	Protocol    string   `json:"protocol"`
	Server      string   `json:"server"`
	Port        int      `json:"port"`
	Type        string   `json:"type"`
	MessageType string   `json:"message_type"`
	OptionalCfg optional `json:"optional"`
}

func RedisStorageConSelectorApply(connectionSelector string, conf *KuiperConf) error {
	sel := ConSelector{
		ConnSelectorStr: connectionSelector,
	}

	err := sel.Init()
	if err != nil {
		return err
	}

	//this should be edgeX redis config
	kvs, err := sel.ReadCfgFromYaml()
	if err != nil {
		return err
	}

	redisCfg := edgeXConfig{}
	err = cast.MapToStruct(kvs, &redisCfg)
	if err != nil {
		return err
	}

	if redisCfg.Type != "redis" || redisCfg.Protocol != "redis" {
		return fmt.Errorf("redis storage connection selector %s only support redis mesage bus, but got %v", sel.ConnSelectorStr, kvs)
	}

	conf.Store.Redis.Host = redisCfg.Server
	conf.Store.Redis.Port = redisCfg.Port
	conf.Store.Redis.Password = redisCfg.OptionalCfg.Password
	return nil
}


<|start_filename|>internal/server/middleware/auth_test.go<|end_filename|>
package middleware

import (
	"github.com/lf-edge/ekuiper/internal/pkg/jwt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

func genToken(signKeyName, issuer, aud string) string {
	tkStr, _ := jwt.CreateToken(signKeyName, issuer, aud)
	return tkStr
}

func Test_AUTH(t *testing.T) {
	nextHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
	})

	handler := Auth(nextHandler)

	type args struct {
		th string
	}
	tests := []struct {
		name     string
		args     args
		req      *http.Request
		res      *httptest.ResponseRecorder
		wantCode int
	}{
		{
			name:     "token right",
			args:     args{th: genToken("sample_key", "sample_key.pub", "eKuiper")},
			req:      httptest.NewRequest("GET", "http://127.0.0.1:9081/streams", nil),
			res:      httptest.NewRecorder(),
			wantCode: 200,
		},

		{
			name:     "audience not right",
			args:     args{th: genToken("sample_key", "sample_key.pub", "Neuron")},
			req:      httptest.NewRequest("GET", "http://127.0.0.1:9081/streams", nil),
			res:      httptest.NewRecorder(),
			wantCode: 401,
		},
		{
			name:     "no token",
			args:     args{th: ""},
			req:      httptest.NewRequest("GET", "http://127.0.0.1:9081/streams", nil),
			res:      httptest.NewRecorder(),
			wantCode: 401,
		},
		{
			name:     "no need token path",
			args:     args{th: ""},
			req:      httptest.NewRequest("GET", "http://127.0.0.1:9081/ping", nil),
			res:      httptest.NewRecorder(),
			wantCode: 200,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			tt.req.Header.Set("Authorization", tt.args.th)
			handler.ServeHTTP(tt.res, tt.req)

			res := tt.res.Result()

			data, err := ioutil.ReadAll(res.Body)
			if err != nil {
				t.Errorf("expected error to be nil got %v", err)
			}

			if !reflect.DeepEqual(tt.wantCode, tt.res.Code) {
				t.Errorf("expect %d, actual %d, result %s", tt.wantCode, tt.res.Code, string(data))
			}

			_ = res.Body.Close()
		})
	}
}


<|start_filename|>internal/topo/collector/func.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
	"errors"
	"github.com/lf-edge/ekuiper/pkg/api"
)

// CollectorFunc is a function used to colllect
// incoming stream data. It can be used as a
// stream sink.
type CollectorFunc func(api.StreamContext, interface{}) error

// FuncCollector is a colletor that uses a function
// to collect data.  The specified function must be
// of type:
//   CollectorFunc
type FuncCollector struct {
	f CollectorFunc
}

// Func creates a new value *FuncCollector that
// will use the specified function parameter to
// collect streaming data.
func Func(f CollectorFunc) *FuncCollector {
	return &FuncCollector{f: f}
}

func (c *FuncCollector) Configure(props map[string]interface{}) error {
	//do nothing
	return nil
}

// Open is the starting point that starts the collector
func (c *FuncCollector) Open(ctx api.StreamContext) error {
	log := ctx.GetLogger()
	log.Infoln("Opening func collector")

	if c.f == nil {
		return errors.New("func collector missing function")
	}
	return nil
}

func (c *FuncCollector) Collect(ctx api.StreamContext, item interface{}) error {
	return c.f(ctx, item)
}

func (c *FuncCollector) Close(api.StreamContext) error {
	return nil
}


<|start_filename|>etc/sinks/zmq.json<|end_filename|>
{
	"about": {
		"trial": true,
		"author": {
			"name": "EMQ",
			"email": "<EMAIL>",
			"company": "EMQ Technologies Co., Ltd",
			"website": "https://www.emqx.io"
		},
		"helpUrl": {
			"en_US": "https://github.com/lf-edge/ekuiper/blob/master/docs/en_US/plugins/sinks/zmq.md",
			"zh_CN": "https://github.com/lf-edge/ekuiper/blob/master/docs/zh_CN/plugins/sinks/zmq.md"
		},
		"description": {
			"en_US": "This a demo 0MQ sink plugin, it cannot be used in production env.",
			"zh_CN": "本插件为演示 ZMQ 数据发送插件，不能使用在生产环境中"
		}
	},
	"libs": [
      "github.com/pebbe/zmq4@v1.0.0"
	],
	"properties": [{
		"name": "server",
		"default": "127.0.0.1:5536",
		"optional": false,
		"control": "text",
		"type": "string",
		"hint": {
			"en_US": "The url of the Zero Mq server",
			"zh_CN": "Zero Mq 服务器的 URL"
		},
		"label": {
			"en_US": "Server address",
			"zh_CN": "服务器地址"
		}
	}, {
		"name": "topic",
		"default": "",
		"optional": true,
		"control": "text",
		"type": "string",
		"hint": {
			"en_US": "The topic to publish to.",
			"zh_CN": "订阅主题"
		},
		"label": {
			"en_US": "Topic",
			"zh_CN": "主题"
		}
	}]
}


<|start_filename|>internal/plugin/portable/registry_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package portable

import (
	"github.com/lf-edge/ekuiper/internal/plugin"
	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
	"reflect"
	"sync"
	"testing"
)

func TestConcurrent(t *testing.T) {
	r := &registry{
		RWMutex:   sync.RWMutex{},
		plugins:   make(map[string]*PluginInfo),
		sources:   make(map[string]string),
		sinks:     make(map[string]string),
		functions: make(map[string]string),
	}
	allPlugins := []*PluginInfo{
		{
			PluginMeta: runtime.PluginMeta{
				Name:       "mirror",
				Version:    "1.3.0",
				Language:   "go",
				Executable: "mirror",
			},
			Sources:   []string{"random"},
			Sinks:     []string{"file"},
			Functions: []string{"echo"},
		}, {
			PluginMeta: runtime.PluginMeta{
				Name:       "next",
				Version:    "1.3.0",
				Language:   "python",
				Executable: "next",
			},
			Sinks: []string{"udp", "follower"},
		}, {
			PluginMeta: runtime.PluginMeta{
				Name:       "dummy",
				Version:    "v0.2",
				Language:   "go",
				Executable: "dummy",
			},
			Sources:   []string{"new", "can"},
			Functions: []string{"abc"},
		},
	}
	expectedPlugins := map[string]*PluginInfo{
		"mirror": allPlugins[0],
		"next":   allPlugins[1], "dummy": allPlugins[2],
	}

	expectedSources := map[string]string{
		"can": "dummy", "new": "dummy", "random": "mirror",
	}
	expectedFunctions := map[string]string{
		"abc": "dummy", "echo": "mirror",
	}
	expectedSinks := map[string]string{
		"file": "mirror", "follower": "next", "udp": "next",
	}
	// set concurrently
	var wg sync.WaitGroup
	for n, pi := range expectedPlugins {
		wg.Add(1)
		go func(name string, pluginInfo *PluginInfo) {
			defer wg.Done()
			r.Set(name, pluginInfo)
		}(n, pi)
	}
	wg.Wait()

	if !reflect.DeepEqual(expectedPlugins, r.plugins) {
		t.Errorf("plugins mismatch: expected %v, got %v", expectedPlugins, r.plugins)
		return
	}
	result := r.List()
	if !reflect.DeepEqual(len(allPlugins), len(result)) {
		t.Errorf("list plugins count mismatch: expected %v, got %v", allPlugins, result)
		return
	}
outer:
	for _, res := range result {
		for _, p := range allPlugins {
			if reflect.DeepEqual(p, res) {
				continue outer
			}
		}
		t.Errorf("list plugins mismatch: expected %v, got %v", allPlugins, result)
		return
	}

	if !reflect.DeepEqual(expectedSources, r.sources) {
		t.Errorf("sources mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", expectedSources, r.sources)
		return
	}
	if !reflect.DeepEqual(expectedFunctions, r.functions) {
		t.Errorf("functions mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", expectedFunctions, r.functions)
		return
	}
	if !reflect.DeepEqual(expectedSinks, r.sinks) {
		t.Errorf("sinks mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", expectedSinks, r.functions)
		return
	}
	pn, ok := r.GetSymbol(plugin.SOURCE, "new")
	if !ok {
		t.Error("can't find symbol new")
		return
	}
	if pn != "dummy" {
		t.Errorf("GetSymbol wrong, expect dummy but got %s", pn)
	}

	// Delete concurrently
	for n := range expectedPlugins {
		wg.Add(1)
		go func(name string) {
			defer wg.Done()
			r.Delete(name)
		}(n)
	}
	wg.Wait()
	result = r.List()
	if !reflect.DeepEqual(0, len(result)) {
		t.Errorf("list plugins count mismatch: expected no plugins, got %v", result)
		return
	}
}


<|start_filename|>internal/conf/load_test.go<|end_filename|>
// INTECH Process Automation Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conf

import (
	"os"
	"reflect"
	"testing"
)

func TestEnv(t *testing.T) {

	key := "KUIPER__BASIC__CONSOLELOG"
	value := "true"

	err := os.Setenv(key, value)
	if err != nil {
		t.Error(err)
	}

	c := KuiperConf{}
	err = LoadConfig(&c)
	if err != nil {
		t.Error(err)
	}

	if c.Basic.ConsoleLog != true {
		t.Errorf("env variable should set it to true")
	}
}

func TestJsonCamelCase(t *testing.T) {
	key := "HTTPPULL__DEFAULT__BODYTYPE"
	value := "event"

	err := os.Setenv(key, value)
	if err != nil {
		t.Error(err)
	}

	const ConfigName = "sources/httppull.yaml"
	c := make(map[string]interface{})
	err = LoadConfigByName(ConfigName, &c)
	if err != nil {
		t.Error(err)
	}

	if casted, success := c["default"].(map[string]interface{}); success {
		if casted["bodyType"] != "event" {
			t.Errorf("env variable should set it to event")
		}
	} else {
		t.Errorf("returned value does not contains map under 'Basic' key")
	}
}

func TestNestedFields(t *testing.T) {
	key := "EDGEX__DEFAULT__OPTIONAL__PASSWORD"
	value := "password"

	err := os.Setenv(key, value)
	if err != nil {
		t.Error(err)
	}

	const ConfigName = "sources/edgex.yaml"
	c := make(map[string]interface{})
	err = LoadConfigByName(ConfigName, &c)
	if err != nil {
		t.Error(err)
	}

	if casted, success := c["default"].(map[string]interface{}); success {
		if optional, ok := casted["optional"].(map[string]interface{}); ok {
			if optional["Password"] != "password" {
				t.Errorf("Password variable should set it to password")
			}
		} else {
			t.Errorf("returned value does not contains map under 'optional' key")
		}
	} else {
		t.Errorf("returned value does not contains map under 'Basic' key")
	}
}

func TestKeysReplacement(t *testing.T) {
	input := createRandomConfigMap()
	expected := createExpectedRandomConfigMap()
	list := []string{"interval", "Seed", "deduplicate", "pattern"}

	applyKeys(input, list)

	if !reflect.DeepEqual(input, expected) {
		t.Errorf("key names within list should be applied \nexpected - %s\n input   - %s", expected, input)
	}
}

func TestKeyReplacement(t *testing.T) {
	m := createRandomConfigMap()
	expected := createExpectedRandomConfigMap()

	applyKey(m, "Seed")
	applyKey(m, "interval")

	if !reflect.DeepEqual(m, expected) {
		t.Errorf("key names within list should be applied \nexpected - %s\nmap      - %s", expected, m)
	}
}

func createRandomConfigMap() map[string]interface{} {
	pattern := make(map[string]interface{})
	pattern["count"] = 50
	defaultM := make(map[string]interface{})
	defaultM["interval"] = 1000
	defaultM["seed"] = 1
	defaultM["pattern"] = pattern
	defaultM["deduplicate"] = 0
	ext := make(map[string]interface{})
	ext["interval"] = 100
	dedup := make(map[string]interface{})
	dedup["interval"] = 100
	dedup["deduplicated"] = 50
	input := make(map[string]interface{})
	input["default"] = defaultM
	input["ext"] = ext
	input["dedup"] = dedup
	return input
}

func createExpectedRandomConfigMap() map[string]interface{} {
	input := createRandomConfigMap()
	def := input["default"]
	if defMap, ok := def.(map[string]interface{}); ok {
		tmp := defMap["seed"]
		delete(defMap, "seed")
		defMap["Seed"] = tmp
	}
	return input
}


<|start_filename|>deploy/packages/Makefile<|end_filename|>
export

ifneq ($(shell uname -s),Linux)
	$(shell echo "Please execute this script under Linux")
	exit
endif
# The version-release used for package
export PKG_VSN := $(shell git describe --tags --always)

ifneq ($(shell cat /etc/*-release |grep -o -i centos),)
	ID := centos
	VERSION_ID := $(shell rpm --eval '%{centos_ver}')
else
	ID := $(shell sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g' )
	VERSION_ID := $(shell sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')
endif
SYSTEM := $(shell echo $(ID)$(VERSION_ID) | sed -r "s/([a-zA-Z]*)-.*/\1/g")
##
## Support RPM and Debian based linux systems
##
ifeq ($(ID),ubuntu)
	PKGERDIR := deb
else ifeq ($(ID),debian)
	PKGERDIR := deb
else ifeq ($(ID),raspbian)
	PKGERDIR := deb
else
	PKGERDIR := rpm
endif

.PHONY: all
all: clean
	$(if $(PKGERDIR),,$(error "Operating system '$(OS)' not supported"))
	cd $(PKGERDIR) && $(MAKE)

clean:
	rm -rf package
	make -C rpm clean
	make -C deb clean

.PHONY: deb
deb:
	make -C deb

.PHONY: rpm
rpm:
	make -C rpm



<|start_filename|>sdk/go/runtime/sink.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package runtime

import (
	context2 "context"
	"fmt"
	"github.com/lf-edge/ekuiper/sdk/go/api"
	"github.com/lf-edge/ekuiper/sdk/go/connection"
	"go.nanomsg.org/mangos/v3"
)

type sinkRuntime struct {
	s      api.Sink
	ch     connection.DataInChannel
	ctx    api.StreamContext
	cancel context2.CancelFunc
	key    string
}

func setupSinkRuntime(con *Control, s api.Sink) (*sinkRuntime, error) {
	ctx, err := parseContext(con)
	if err != nil {
		return nil, err
	}
	err = s.Configure(con.Config)
	if err != nil {
		return nil, err
	}
	ch, err := connection.CreateSinkChannel(ctx)
	if err != nil {
		return nil, err
	}
	ctx.GetLogger().Info("Setup message pipeline, start listening")
	ctx, cancel := ctx.WithCancel()
	return &sinkRuntime{
		s:      s,
		ch:     ch,
		ctx:    ctx,
		cancel: cancel,
		key:    fmt.Sprintf("%s_%s_%d_%s", con.Meta.RuleId, con.Meta.OpId, con.Meta.InstanceId, con.SymbolName),
	}, nil
}

func (s *sinkRuntime) run() {
	err := s.s.Open(s.ctx)
	if err != nil {
		_ = s.stop()
		return
	}
	for {
		var msg []byte
		// blocking read, must be interrupted when stopping
		msg, err = s.ch.Recv()
		switch err {
		case mangos.ErrClosed:
			break
		case mangos.ErrRecvTimeout:
			continue
		case nil:
			// do nothing
		default:
			s.ctx.GetLogger().Errorf("cannot receive from mangos Socket: %s", err.Error())
			_ = s.stop()
			return
		}
		err = s.s.Collect(s.ctx, msg)
		if err != nil {
			s.ctx.GetLogger().Errorf("collect error: %s", err.Error())
			_ = s.stop()
			return
		}
	}
}

func (s *sinkRuntime) stop() error {
	s.cancel()
	_ = s.s.Close(s.ctx)
	err := s.ch.Close()
	if err != nil {
		s.ctx.GetLogger().Info(err)
	}
	s.ctx.GetLogger().Info("closed sink data channel")
	reg.Delete(s.key)
	return nil
}

func (s *sinkRuntime) isRunning() bool {
	return s.ctx.Err() == nil
}


<|start_filename|>tools/kubernetes/conf/conf.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conf

import (
	"bytes"
	"encoding/json"
	"fmt"
	"gopkg.in/yaml.v3"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"time"

	"github.com/sirupsen/logrus"
)

type (
	config struct {
		Port         int    `yaml:"port"`
		Timeout      int    `yaml:"timeout"`
		IntervalTime int    `yaml:"intervalTime"`
		Ip           string `yaml:"ip"`
		ConsoleLog   bool   `yaml:"consoleLog"`
		FileLog      bool   `yaml:"fileLog"`
		LogPath      string `yaml:"logPath"`
		CommandDir   string `yaml:"commandDir"`
	}
)

var gConf config

func GetConf() *config {
	return &gConf
}
func (c *config) GetIntervalTime() int {
	return c.IntervalTime
}
func (c *config) GetIp() string {
	return c.Ip
}
func (c *config) GetPort() int {
	return c.Port
}
func (c *config) GetLogPath() string {
	return c.LogPath
}
func (c *config) GetCommandDir() string {
	return c.CommandDir
}

func processPath(path string) (string, error) {
	if abs, err := filepath.Abs(path); err != nil {
		return "", nil
	} else {
		if _, err := os.Stat(abs); os.IsNotExist(err) {
			return "", err
		}
		return abs, nil
	}
}

func (c *config) initConfig() bool {
	confPath, err := processPath(os.Args[1])
	if nil != err {
		fmt.Println("conf path err : ", err)
		return false
	}
	sliByte, err := ioutil.ReadFile(confPath)
	if nil != err {
		fmt.Println("load conf err : ", err)
		return false
	}
	err = yaml.Unmarshal(sliByte, c)
	if nil != err {
		fmt.Println("unmashal conf err : ", err)
		return false
	}

	if c.CommandDir, err = filepath.Abs(c.CommandDir); err != nil {
		fmt.Println("command dir err : ", err)
		return false
	}
	if _, err = os.Stat(c.CommandDir); os.IsNotExist(err) {
		fmt.Println("not found dir : ", c.CommandDir)
		return false
	}

	if c.LogPath, err = filepath.Abs(c.LogPath); nil != err {
		fmt.Println("log dir err : ", err)
		return false
	}
	if _, err = os.Stat(c.LogPath); os.IsNotExist(err) {
		if err = os.MkdirAll(path.Dir(c.LogPath), 0755); nil != err {
			fmt.Println("mak logdir err : ", err)
			return false
		}
	}
	return true
}

var (
	Log     *logrus.Logger
	gClient http.Client
)

func (c *config) initTimeout() {
	gClient.Timeout = time.Duration(c.Timeout) * time.Millisecond
}

func (c *config) initLog() bool {
	Log = logrus.New()
	Log.SetReportCaller(true)
	Log.SetFormatter(&logrus.TextFormatter{
		CallerPrettyfier: func(f *runtime.Frame) (string, string) {
			filename := path.Base(f.File)
			return "", fmt.Sprintf("%s:%d", filename, f.Line)
		},
		DisableColors: true,
		FullTimestamp: true,
	})
	if c.FileLog {
		logFile, err := os.OpenFile(c.LogPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
		if err != nil {
			fmt.Println("Failed to init log file settings..." + err.Error())
			Log.Infof("Failed to log to file, using default stderr.")
			return false
		} else if c.ConsoleLog {
			mw := io.MultiWriter(os.Stdout, logFile)
			Log.SetOutput(mw)
		} else if !c.ConsoleLog {
			Log.SetOutput(logFile)
		}
	} else if c.ConsoleLog {
		Log.SetOutput(os.Stdout)
	}
	return true
}

func (c *config) Init() bool {
	if !c.initConfig() {
		return false
	}

	if !c.initLog() {
		return false
	}
	c.initTimeout()
	return true
}

func fetchContents(request *http.Request) (data []byte, err error) {
	respon, err := gClient.Do(request)
	if nil != err {
		return nil, err
	}
	defer respon.Body.Close()
	data, err = ioutil.ReadAll(respon.Body)
	if nil != err {
		return nil, err
	}
	/*
		if respon.StatusCode < 200 || respon.StatusCode > 299 {
			return data, fmt.Errorf("http return code: %d and error message %s.", respon.StatusCode, string(data))
		}
	*/
	return data, err
}

func Get(inUrl string) (data []byte, err error) {
	request, err := http.NewRequest(http.MethodGet, inUrl, nil)
	if nil != err {
		return nil, err
	}
	return fetchContents(request)
}

func Post(inHead, inBody string) (data []byte, err error) {
	request, err := http.NewRequest(http.MethodPost, inHead, bytes.NewBuffer([]byte(inBody)))
	if nil != err {
		return nil, err
	}
	request.Header.Set("Content-Type", "application/json")
	return fetchContents(request)
}

func Put(inHead, inBody string) (data []byte, err error) {
	request, err := http.NewRequest(http.MethodPut, inHead, bytes.NewBuffer([]byte(inBody)))
	if nil != err {
		return nil, err
	}
	request.Header.Set("Content-Type", "application/json")
	return fetchContents(request)
}

func Delete(inUrl string) (data []byte, err error) {
	request, err := http.NewRequest(http.MethodDelete, inUrl, nil)
	if nil != err {
		return nil, err
	}
	return fetchContents(request)
}

func LoadFileUnmarshal(path string, ret interface{}) error {
	sliByte, err := ioutil.ReadFile(path)
	if nil != err {
		return err
	}
	err = json.Unmarshal(sliByte, ret)
	if nil != err {
		return err
	}
	return nil
}

func SaveFileMarshal(path string, content interface{}) error {
	data, err := json.Marshal(content)
	if nil != err {
		return err
	}
	return ioutil.WriteFile(path, data, 0666)
}


<|start_filename|>internal/topo/operator/math_func_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package operator

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/conf"
	"github.com/lf-edge/ekuiper/internal/topo/context"
	"github.com/lf-edge/ekuiper/internal/xsql"
	"reflect"
	"strings"
	"testing"
)

func TestMathAndConversionFunc_Apply1(t *testing.T) {
	var tests = []struct {
		sql    string
		data   *xsql.Tuple
		result []map[string]interface{}
	}{
		{
			sql: "SELECT abs(a) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: xsql.Message{
					"a": -1,
				},
			},
			result: []map[string]interface{}{{
				"a": 1,
			}},
		},

		{
			sql: "SELECT abs(a) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: xsql.Message{
					"a": -1.1,
				},
			},
			result: []map[string]interface{}{{
				"a": 1.1,
			}},
		},

		{
			sql: "SELECT abs(a) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: xsql.Message{
					"a": 1.1,
				},
			},
			result: []map[string]interface{}{{
				"a": 1.1,
			}},
		},

		{
			sql: "SELECT acos(1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(0),
			}},
		},

		{
			sql: "SELECT asin(1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(1.5707963267948966),
			}},
		},

		{
			sql: "SELECT atan(1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(0.7853981633974483),
			}},
		},

		{
			sql: "SELECT atan2(1,1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(0.7853981633974483),
			}},
		},

		{
			sql: "SELECT bitand(1,1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": 1,
			}},
		},

		{
			sql: "SELECT bitand(1.0,1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: nil,
		},

		{
			sql: "SELECT bitor(1,1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": 1,
			}},
		},

		{
			sql: "SELECT bitxor(1,1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": 0,
			}},
		},

		{
			sql: "SELECT bitnot(1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": -2,
			}},
		},

		{
			sql: "SELECT ceil(1.6) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(2),
			}},
		},

		{
			sql: "SELECT cos(0) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(1),
			}},
		},

		{
			sql: "SELECT cosh(0) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(1),
			}},
		},

		{
			sql: "SELECT exp(1.2) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(3.3201169227365472),
			}},
		},

		{
			sql: "SELECT ln(1) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(0),
			}},
		},

		{
			sql: "SELECT log(10) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(1),
			}},
		},

		{
			sql: "SELECT mod(10, 3) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(1),
			}},
		},

		{
			sql: "SELECT power(10, 3) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(1000),
			}},
		},

		{
			sql: "SELECT round(10.2) AS a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(10),
			}},
		},

		{
			sql: "SELECT sign(10.2) AS a, sign(-2) as b, sign(0) as c FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": 1,
				"b": -1,
				"c": 0,
			}},
		},

		{
			sql: "SELECT sin(0) as a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(0),
			}},
		},

		{
			sql: "SELECT sinh(0) as a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(0),
			}},
		},

		{
			sql: "SELECT sqrt(4) as a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(2),
			}},
		},

		{
			sql: "SELECT tan(0) as a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(0),
			}},
		},

		{
			sql: "SELECT tanh(1) as a FROM test",
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(0.7615941559557649),
			}},
		},

		{
			sql: `SELECT cast(1.2, "bigint") as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": 1,
			}},
		},

		{
			sql: `SELECT cast(5, "bigint") as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": 5,
			}},
		},

		{
			sql: `SELECT cast(1.2, "string") as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": "1.2",
			}},
		},

		{
			sql: `SELECT cast(true, "string") as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": "true",
			}},
		},

		{
			sql: `SELECT cast("true", "boolean") as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": true,
			}},
		},

		{
			sql: `SELECT cast("1", "boolean") as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": true,
			}},
		},

		{
			sql: `SELECT cast(0.0, "boolean") as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": false,
			}},
		},

		{
			sql: `SELECT chr(0) as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": int32(0),
			}},
		},

		{
			sql: `SELECT chr("a") as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": int32(97),
			}},
		},

		{
			sql: `SELECT encode("hello", "base64") as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": "aGVsbG8=",
			}},
		},

		{
			sql: `SELECT trunc(3.1415, 2) as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(3.14),
			}},
		},

		{
			sql: `SELECT trunc(3, 2) as a FROM test`,
			data: &xsql.Tuple{
				Emitter: "test",
				Message: nil,
			},
			result: []map[string]interface{}{{
				"a": float64(3.00),
			}},
		},
	}

	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	contextLogger := conf.Log.WithField("rule", "TestMathAndConversionFunc_Apply1")
	ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger)
	for i, tt := range tests {
		//fmt.Println("Running test " + strconv.Itoa(i))
		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
		if err != nil && tt.result == nil {
			continue
		} else if err != nil && tt.result != nil {
			t.Errorf("%q", err)
			continue
		}
		pp := &ProjectOp{Fields: stmt.Fields}
		fv, afv := xsql.NewFunctionValuersForOp(nil)
		result := pp.Apply(ctx, tt.data, fv, afv)
		if !reflect.DeepEqual(tt.result, result) {
			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
		}
	}
}


<|start_filename|>internal/topo/connection/client_wrapper.go<|end_filename|>
package connection

import (
	"github.com/lf-edge/ekuiper/internal/conf"
)

type ClientFactoryFunc func(super *conf.ConSelector) Client

type Client interface {
	CfgValidate(map[string]interface{}) error
	GetClient() (interface{}, error)
	CloseClient() error
}

type clientWrapper struct {
	cli    Client
	conn   interface{}
	refCnt uint32
}

func NewClientWrapper(client Client, selector *conf.ConSelector) (*clientWrapper, error) {
	props, err := selector.ReadCfgFromYaml()
	if err != nil {
		return nil, err
	}
	err = client.CfgValidate(props)
	if err != nil {
		return nil, err
	}
	var con interface{}

	con, err = client.GetClient()
	if err != nil {
		return nil, err
	}

	cliWpr := &clientWrapper{
		cli:    client,
		conn:   con,
		refCnt: 1,
	}

	return cliWpr, nil
}

func (c *clientWrapper) addRef() {
	c.refCnt = c.refCnt + 1
}

func (c *clientWrapper) subRef() {
	c.refCnt = c.refCnt - 1
}

func (c *clientWrapper) IsRefEmpty() bool {
	return c.refCnt == 0
}

func (c *clientWrapper) clean() {
	_ = c.cli.CloseClient()
}

func (c *clientWrapper) getInstance() interface{} {
	return c.conn
}


<|start_filename|>pkg/ast/statement.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ast

type Statement interface {
	stmt()
	Node
}

type SelectStatement struct {
	Fields     Fields
	Sources    Sources
	Joins      Joins
	Condition  Expr
	Dimensions Dimensions
	Having     Expr
	SortFields SortFields

	Statement
}

type Fields []Field

func (f Fields) node() {}

func (f Fields) Len() int {
	return len(f)
}
func (f Fields) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}
func (f Fields) Less(i int, j int) bool {
	m := f[i].AName
	if m == "" {
		m = f[i].Name
	}
	n := f[j].AName
	if n == "" {
		n = f[j].Name
	}
	return m < n
}

type Field struct {
	Name  string
	AName string
	Expr  Expr

	Node
}

func (f *Field) GetName() string {
	if f.AName != "" {
		return f.AName
	} else {
		return f.Name
	}
}

func (f *Field) IsSelectionField() bool {
	if f.AName != "" {
		return true
	}
	_, ok := f.Expr.(*FieldRef)
	if ok {
		return true
	}
	return false
}

func (f *Field) IsColumn() bool {
	if f.AName != "" {
		return false
	}
	_, ok := f.Expr.(*FieldRef)
	if ok {
		return true
	}
	return false
}

type Sources []Source

func (s Sources) node() {}

type Source interface {
	Node
	source()
}

type Table struct {
	Name  string
	Alias string
	Source
}

type JoinType int

const (
	LEFT_JOIN JoinType = iota
	INNER_JOIN
	RIGHT_JOIN
	FULL_JOIN
	CROSS_JOIN
)

type Join struct {
	Name     string
	Alias    string
	JoinType JoinType
	Expr     Expr

	Node
}

type Joins []Join

func (j Joins) node() {}

type Dimension struct {
	Expr Expr

	Node
}

type Dimensions []Dimension

func (d Dimensions) node() {}

func (d *Dimensions) GetWindow() *Window {
	for _, child := range *d {
		if w, ok := child.Expr.(*Window); ok {
			return w
		}
	}
	return nil
}
func (d *Dimensions) GetGroups() Dimensions {
	var nd Dimensions
	for _, child := range *d {
		if _, ok := child.Expr.(*Window); !ok {
			nd = append(nd, child)
		}
	}
	return nd
}

type WindowType int

const (
	NOT_WINDOW WindowType = iota
	TUMBLING_WINDOW
	HOPPING_WINDOW
	SLIDING_WINDOW
	SESSION_WINDOW
	COUNT_WINDOW
)

type Window struct {
	WindowType WindowType
	Length     *IntegerLiteral
	Interval   *IntegerLiteral
	Filter     Expr
	Expr
}

type SortField struct {
	Name       string
	StreamName StreamName
	Uname      string // unique name of a field
	Ascending  bool

	Expr
}

type SortFields []SortField

func (d SortFields) node() {}


<|start_filename|>internal/binder/function/binder.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package function

import (
	"github.com/lf-edge/ekuiper/internal/binder"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/errorx"
)

var ( // init once and read only
	funcFactories      []binder.FuncFactory
	funcFactoriesNames []string
)

func init() {
	f := binder.FactoryEntry{
		Name:    "built-in",
		Factory: GetManager(),
	}
	applyFactory(f)
}

// Initialize Only call once when server starts
func Initialize(factories []binder.FactoryEntry) error {
	for _, f := range factories {
		applyFactory(f)
	}
	return nil
}

func applyFactory(f binder.FactoryEntry) {
	if s, ok := f.Factory.(binder.FuncFactory); ok {
		funcFactories = append(funcFactories, s)
		funcFactoriesNames = append(funcFactoriesNames, f.Name)
	}
}

func Function(name string) (api.Function, error) {
	e := make(errorx.MultiError)
	for i, sf := range funcFactories {
		r, err := sf.Function(name)
		if err != nil {
			e[funcFactoriesNames[i]] = err
		}
		if r != nil {
			return r, e.GetError()
		}
	}
	return nil, e.GetError()
}

func HasFunctionSet(name string) bool {
	for _, sf := range funcFactories {
		r := sf.HasFunctionSet(name)
		if r {
			return r
		}
	}
	return false
}

func ConvName(name string) (string, bool) {
	for _, sf := range funcFactories {
		r, ok := sf.ConvName(name)
		if ok {
			return r, ok
		}
	}
	return name, false
}

type multiAggFunc interface {
	IsAggregateWithName(name string) bool
}

func IsAggFunc(funcName string) bool {
	f, _ := Function(funcName)
	if f != nil {
		if mf, ok := f.(multiAggFunc); ok {
			return mf.IsAggregateWithName(funcName)
		} else {
			return f.IsAggregate()
		}
	}
	return false
}


<|start_filename|>internal/pkg/ts/sql/sqlTs_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"github.com/lf-edge/ekuiper/internal/pkg/db/sql/sqlite"
	"github.com/lf-edge/ekuiper/internal/pkg/ts/test/common"
	ts2 "github.com/lf-edge/ekuiper/pkg/kv"
	"os"
	"path"
	"path/filepath"
	"testing"
)

const (
	DbName = "sqliteTS.db"
	Table  = "test"
)

func TestSqlTsSet(t *testing.T) {
	ks, db, abs := setupSqlKv()
	defer cleanSqlKv(db, abs)

	common.TestTsSet(ks, t)
}

func TestSqlTsLast(t *testing.T) {
	ks, db, abs := setupSqlKv()
	defer cleanSqlKv(db, abs)

	common.TestTsLast(ks, t)
}

func TestSqlTsGet(t *testing.T) {
	ks, db, abs := setupSqlKv()
	defer cleanSqlKv(db, abs)

	common.TestTsGet(ks, t)
}

func TestSqlTsDelete(t *testing.T) {
	ks, db, abs := setupSqlKv()
	defer cleanSqlKv(db, abs)

	common.TestTsDelete(ks, t)
}

func TestSqlTsDeleteBefore(t *testing.T) {
	ks, db, abs := setupSqlKv()
	defer cleanSqlKv(db, abs)

	common.TestTsDeleteBefore(ks, t)
}

func deleteIfExists(abs string) error {
	absPath := path.Join(abs, DbName)
	if f, _ := os.Stat(absPath); f != nil {
		return os.Remove(absPath)
	}
	return nil
}

func setupSqlKv() (ts2.Tskv, *sqlite.Database, string) {
	absPath, err := filepath.Abs("test")
	if err != nil {
		panic(err)
	}
	err = deleteIfExists(absPath)
	if err != nil {
		panic(err)
	}
	config := sqlite.Config{
		Path: absPath,
		Name: DbName,
	}
	_, db := sqlite.NewSqliteDatabase(config)
	err = db.Connect()
	if err != nil {
		panic(err)
	}

	builder := NewTsBuilder(db)
	if err != nil {
		panic(err)
	}
	var store ts2.Tskv
	err, store = builder.CreateTs(Table)
	if err != nil {
		panic(err)
	}
	return store, db, absPath
}

func cleanSqlKv(db *sqlite.Database, abs string) {
	if err := db.Disconnect(); err != nil {
		panic(err)
	}
	if err := deleteIfExists(abs); err != nil {
		panic(err)
	}
}


<|start_filename|>internal/topo/node/watermark.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"context"
	"fmt"
	"github.com/lf-edge/ekuiper/internal/xsql"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/ast"
	"math"
	"sort"
)

type WatermarkTuple struct {
	Timestamp int64
}

func (t *WatermarkTuple) GetTimestamp() int64 {
	return t.Timestamp
}

func (t *WatermarkTuple) IsWatermark() bool {
	return true
}

const WATERMARK_KEY = "$$wartermark"

type WatermarkGenerator struct {
	inputTopics   []string
	topicToTs     map[string]int64
	window        *WindowConfig
	lateTolerance int64
	interval      int
	//ticker          *clock.Ticker
	stream chan<- interface{}
	//state
	lastWatermarkTs int64
}

func NewWatermarkGenerator(window *WindowConfig, l int64, s []string, stream chan<- interface{}) (*WatermarkGenerator, error) {
	w := &WatermarkGenerator{
		window:        window,
		topicToTs:     make(map[string]int64),
		lateTolerance: l,
		inputTopics:   s,
		stream:        stream,
	}
	switch window.Type {
	case ast.NOT_WINDOW:
	case ast.TUMBLING_WINDOW:
		w.interval = window.Length
	case ast.HOPPING_WINDOW:
		w.interval = window.Interval
	case ast.SLIDING_WINDOW:
		w.interval = window.Length
	case ast.SESSION_WINDOW:
		//Use timeout to update watermark
		w.interval = window.Interval
	default:
		return nil, fmt.Errorf("unsupported window type %d", window.Type)
	}
	return w, nil
}

func (w *WatermarkGenerator) track(s string, ts int64, ctx api.StreamContext) bool {
	log := ctx.GetLogger()
	log.Debugf("watermark generator track event from topic %s at %d", s, ts)
	currentVal, ok := w.topicToTs[s]
	if !ok || ts > currentVal {
		w.topicToTs[s] = ts
	}
	r := ts >= w.lastWatermarkTs
	if r {
		w.trigger(ctx)
	}
	return r
}

func (w *WatermarkGenerator) trigger(ctx api.StreamContext) {
	log := ctx.GetLogger()
	watermark := w.computeWatermarkTs(ctx)
	log.Debugf("compute watermark event at %d with last %d", watermark, w.lastWatermarkTs)
	if watermark > w.lastWatermarkTs {
		t := &WatermarkTuple{Timestamp: watermark}
		select {
		case w.stream <- t:
		default: //TODO need to set buffer
		}
		w.lastWatermarkTs = watermark
		ctx.PutState(WATERMARK_KEY, w.lastWatermarkTs)
		log.Debugf("scan watermark event at %d", watermark)
	}
}

func (w *WatermarkGenerator) computeWatermarkTs(_ context.Context) int64 {
	var ts int64
	if len(w.topicToTs) >= len(w.inputTopics) {
		ts = math.MaxInt64
		for _, key := range w.inputTopics {
			if ts > w.topicToTs[key] {
				ts = w.topicToTs[key]
			}
		}
	}
	return ts - w.lateTolerance
}

//If window end cannot be determined yet, return max int64 so that it can be recalculated for the next watermark
func (w *WatermarkGenerator) getNextWindow(inputs []*xsql.Tuple, current int64, watermark int64, triggered bool) int64 {
	switch w.window.Type {
	case ast.TUMBLING_WINDOW, ast.HOPPING_WINDOW:
		if triggered {
			return current + int64(w.interval)
		} else {
			interval := int64(w.interval)
			nextTs := getEarliestEventTs(inputs, current, watermark)
			if nextTs == math.MaxInt64 || nextTs%interval == 0 {
				return nextTs
			}
			return nextTs + (interval - nextTs%interval)
		}
	case ast.SLIDING_WINDOW:
		nextTs := getEarliestEventTs(inputs, current, watermark)
		return nextTs
	default:
		return math.MaxInt64
	}
}

func (w *WatermarkGenerator) getNextSessionWindow(inputs []*xsql.Tuple, current int64, watermark int64, triggered bool) (int64, bool) {
	if len(inputs) > 0 {
		timeout, duration := int64(w.window.Interval), int64(w.window.Length)
		sort.SliceStable(inputs, func(i, j int) bool {
			return inputs[i].Timestamp < inputs[j].Timestamp
		})
		et := inputs[0].Timestamp
		tick := et + (duration - et%duration)
		if et%duration == 0 {
			tick = et
		}
		var p int64
		ticked := false
		for _, tuple := range inputs {
			var r int64 = math.MaxInt64
			if p > 0 {
				if tuple.Timestamp-p > timeout {
					r = p + timeout
				}
			}
			if tuple.Timestamp > tick {
				if tick-duration > et && tick < r {
					r = tick
					ticked = true
				}
				tick += duration
			}
			if r < math.MaxInt64 {
				return r, ticked
			}
			p = tuple.Timestamp
		}
	}
	return math.MaxInt64, false
}

func (o *WindowOperator) execEventWindow(ctx api.StreamContext, inputs []*xsql.Tuple, errCh chan<- error) {
	log := ctx.GetLogger()
	var (
		triggered       bool
		nextWindowEndTs int64
		prevWindowEndTs int64
		lastTicked      bool
	)

	o.watermarkGenerator.lastWatermarkTs = 0
	if s, err := ctx.GetState(WATERMARK_KEY); err == nil && s != nil {
		if si, ok := s.(int64); ok {
			o.watermarkGenerator.lastWatermarkTs = si
		} else {
			errCh <- fmt.Errorf("restore window state `lastWatermarkTs` %v error, invalid type", s)
		}
	}
	log.Infof("Start with window state lastWatermarkTs: %d", o.watermarkGenerator.lastWatermarkTs)
	for {
		select {
		// process incoming item
		case item, opened := <-o.input:
			processed := false
			if item, processed = o.preprocess(item); processed {
				break
			}
			o.statManager.ProcessTimeStart()
			if !opened {
				o.statManager.IncTotalExceptions()
				break
			}
			switch d := item.(type) {
			case error:
				o.statManager.IncTotalRecordsIn()
				o.Broadcast(d)
				o.statManager.IncTotalExceptions()
			case xsql.Event:
				if d.IsWatermark() {
					watermarkTs := d.GetTimestamp()
					windowEndTs := nextWindowEndTs
					ticked := false
					//Session window needs a recalculation of window because its window end depends the inputs
					if windowEndTs == math.MaxInt64 || o.window.Type == ast.SESSION_WINDOW || o.window.Type == ast.SLIDING_WINDOW {
						if o.window.Type == ast.SESSION_WINDOW {
							windowEndTs, ticked = o.watermarkGenerator.getNextSessionWindow(inputs, prevWindowEndTs, watermarkTs, triggered)
						} else {
							windowEndTs = o.watermarkGenerator.getNextWindow(inputs, prevWindowEndTs, watermarkTs, triggered)
						}
					}
					for windowEndTs <= watermarkTs && windowEndTs >= 0 {
						log.Debugf("Window end ts %d Watermark ts %d", windowEndTs, watermarkTs)
						log.Debugf("Current input count %d", len(inputs))
						//scan all events and find out the event in the current window
						if o.window.Type == ast.SESSION_WINDOW && !lastTicked {
							o.triggerTime = inputs[0].Timestamp
						}
						inputs, triggered = o.scan(inputs, windowEndTs, ctx)
						prevWindowEndTs = windowEndTs
						lastTicked = ticked
						if o.window.Type == ast.SESSION_WINDOW {
							windowEndTs, ticked = o.watermarkGenerator.getNextSessionWindow(inputs, prevWindowEndTs, watermarkTs, triggered)
						} else {
							windowEndTs = o.watermarkGenerator.getNextWindow(inputs, prevWindowEndTs, watermarkTs, triggered)
						}
					}
					nextWindowEndTs = windowEndTs
					log.Debugf("next window end %d", nextWindowEndTs)
				} else {
					o.statManager.IncTotalRecordsIn()
					tuple, ok := d.(*xsql.Tuple)
					if !ok {
						log.Debugf("receive non tuple element %v", d)
					}
					log.Debugf("event window receive tuple %s", tuple.Message)
					if o.watermarkGenerator.track(tuple.Emitter, d.GetTimestamp(), ctx) {
						inputs = append(inputs, tuple)
					}
				}
				o.statManager.ProcessTimeEnd()
				ctx.PutState(WINDOW_INPUTS_KEY, inputs)
			default:
				o.statManager.IncTotalRecordsIn()
				o.Broadcast(fmt.Errorf("run Window error: expect xsql.Event type but got %[1]T(%[1]v)", d))
				o.statManager.IncTotalExceptions()
			}
		// is cancelling
		case <-ctx.Done():
			log.Infoln("Cancelling window....")
			if o.ticker != nil {
				o.ticker.Stop()
			}
			return
		}
	}
}

func getEarliestEventTs(inputs []*xsql.Tuple, startTs int64, endTs int64) int64 {
	var minTs int64 = math.MaxInt64
	for _, t := range inputs {
		if t.Timestamp > startTs && t.Timestamp <= endTs && t.Timestamp < minTs {
			minTs = t.Timestamp
		}
	}
	return minTs
}


<|start_filename|>internal/pkg/store/redis/redisKv.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package redis

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"github.com/gomodule/redigo/redis"
	dbRedis "github.com/lf-edge/ekuiper/internal/pkg/db/redis"
	kvEncoding "github.com/lf-edge/ekuiper/internal/pkg/store/encoding"
	"strings"
)

const KvPrefix = "KV:STORE"

type redisKvStore struct {
	database  dbRedis.Instance
	table     string
	keyPrefix string
}

func CreateRedisKvStore(redis dbRedis.Instance, table string) (*redisKvStore, error) {
	store := &redisKvStore{
		database:  redis,
		table:     table,
		keyPrefix: fmt.Sprintf("%s:%s", KvPrefix, table),
	}
	return store, nil
}

func (kv redisKvStore) Setnx(key string, value interface{}) error {
	return kv.database.Apply(func(conn redis.Conn) error {
		err, b := kvEncoding.Encode(value)
		if nil != err {
			return err
		}
		tKey := kv.tableKey(key)
		reply, err := conn.Do("SETNX", tKey, b)
		if err != nil {
			return err
		}
		code, err := redis.Int(reply, err)
		if code == 0 {
			return fmt.Errorf("item %s already exists under %s key because of %s", key, tKey, err)
		}
		return nil
	})
}

func (kv redisKvStore) Set(key string, value interface{}) error {
	err, b := kvEncoding.Encode(value)
	if nil != err {
		return err
	}
	err = kv.database.Apply(func(conn redis.Conn) error {
		tKey := kv.tableKey(key)
		reply, err := conn.Do("SET", tKey, b)
		code, err := redis.String(reply, err)
		if err != nil {
			return err
		}
		if code != "OK" {
			return fmt.Errorf("item %s (under key %s) not set because of %s", key, tKey, err)
		}
		return nil
	})
	return err
}

func (kv redisKvStore) Get(key string, value interface{}) (bool, error) {
	result := false
	err := kv.database.Apply(func(conn redis.Conn) error {
		tKey := kv.tableKey(key)
		reply, err := conn.Do("GET", tKey)
		if err != nil {
			return err
		}
		buff, err := redis.Bytes(reply, err)
		if err != nil {
			result = false
			return nil
		}
		dec := gob.NewDecoder(bytes.NewBuffer(buff))
		if err := dec.Decode(value); err != nil {
			return err
		}
		result = true
		return nil
	})
	return result, err
}

func (kv redisKvStore) Delete(key string) error {
	return kv.database.Apply(func(conn redis.Conn) error {
		tKey := kv.tableKey(key)
		_, err := conn.Do("DEL", tKey)
		return err
	})
}

func (kv redisKvStore) Keys() ([]string, error) {
	keys := make([]string, 0)
	err := kv.database.Apply(func(conn redis.Conn) error {
		pattern := fmt.Sprintf("%s:*", kv.keyPrefix)
		reply, err := conn.Do("KEYS", pattern)
		keys, err = redis.Strings(reply, err)
		return err
	})
	result := make([]string, 0)
	for _, k := range keys {
		result = append(result, kv.trimPrefix(k))
	}
	return result, err
}

func (kv redisKvStore) Clean() error {
	keys, err := kv.Keys()
	if err != nil {
		return err
	}
	keysToRemove := make([]interface{}, len(keys))
	for i, v := range keysToRemove {
		keysToRemove[i] = v
	}
	err = kv.database.Apply(func(conn redis.Conn) error {
		_, err := conn.Do("DEL", keysToRemove...)
		return err
	})
	return err
}

func (kv redisKvStore) tableKey(key string) string {
	return fmt.Sprintf("%s:%s:%s", KvPrefix, kv.table, key)
}

func (kv redisKvStore) trimPrefix(fullKey string) string {
	prefixToTrim := fmt.Sprintf("%s:%s:", KvPrefix, kv.table)
	return strings.TrimPrefix(fullKey, prefixToTrim)
}


<|start_filename|>internal/processor/rule_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package processor

import (
	"github.com/lf-edge/ekuiper/pkg/api"
	"reflect"
	"testing"
)

func TestRuleActionParse_Apply(t *testing.T) {
	var tests = []struct {
		ruleStr string
		result  *api.Rule
	}{
		{
			ruleStr: `{
			  "id": "ruleTest",
			  "sql": "SELECT * from demo",
			  "actions": [
				{
				  	"funcName": "RFC_READ_TABLE",
					"ashost":   "192.168.1.100",
					"sysnr":    "02",
					"client":   "900",
					"user":     "SPERF",
					"passwd":   "<PASSWORD>",
					"params": {
						"QUERY_TABLE": "VBAP",
						"ROWCOUNT":    10,
						"FIELDS": [
							{"FIELDNAME": "MANDT"},
							{"FIELDNAME": "VBELN"},
							{"FIELDNAME": "POSNR"}
						]
					}
				}
			  ]
			}`,
			result: &api.Rule{
				Triggered: false,
				Id:        "ruleTest",
				Sql:       "SELECT * from demo",
				Actions: []map[string]interface{}{
					{
						"funcName": "RFC_READ_TABLE",
						"ashost":   "192.168.1.100",
						"sysnr":    "02",
						"client":   "900",
						"user":     "SPERF",
						"passwd":   "<PASSWORD>",
						"params": map[string]interface{}{
							"QUERY_TABLE": "VBAP",
							"ROWCOUNT":    float64(10),
							"FIELDS": []interface{}{
								map[string]interface{}{"FIELDNAME": "MANDT"},
								map[string]interface{}{"FIELDNAME": "VBELN"},
								map[string]interface{}{"FIELDNAME": "POSNR"},
							},
						},
					},
				},
				Options: &api.RuleOption{
					IsEventTime:        false,
					LateTol:            1000,
					Concurrency:        1,
					BufferLength:       1024,
					SendMetaToSink:     false,
					Qos:                api.AtMostOnce,
					CheckpointInterval: 300000,
					SendError:          true,
				},
			},
		}, {
			ruleStr: `{
				"id": "ruleTest2",
				"sql": "SELECT * from demo",
				"actions": [
					{
						"log": ""
					},
					{
						"sap": {
							"funcName": "RFC_READ_TABLE",
							"ashost": "192.168.100.10",
							"sysnr": "02",
							"client": "900",
							"user": "uuu",
							"passwd": "<PASSWORD>."
						}
					}
				],
				"options": {
					"isEventTime": true,
					"lateTolerance": 1000,
					"bufferLength": 10240,
					"qos": 2,
					"checkpointInterval": 60000
				}
			}`,
			result: &api.Rule{
				Triggered: false,
				Id:        "ruleTest2",
				Sql:       "SELECT * from demo",
				Actions: []map[string]interface{}{
					{
						"log": "",
					}, {
						"sap": map[string]interface{}{
							"funcName": "RFC_READ_TABLE",
							"ashost":   "192.168.100.10",
							"sysnr":    "02",
							"client":   "900",
							"user":     "uuu",
							"passwd":   "<PASSWORD>.",
						},
					},
				},
				Options: &api.RuleOption{
					IsEventTime:        true,
					LateTol:            1000,
					Concurrency:        1,
					BufferLength:       10240,
					SendMetaToSink:     false,
					Qos:                api.ExactlyOnce,
					CheckpointInterval: 60000,
					SendError:          true,
				},
			},
		},
	}

	p := NewRuleProcessor()
	for i, tt := range tests {
		r, err := p.getRuleByJson(tt.result.Id, tt.ruleStr)
		if err != nil {
			t.Errorf("get rule error: %s", err)
		}
		if !reflect.DeepEqual(tt.result, r) {
			t.Errorf("%d \tresult mismatch:\n\nexp=%+v\n\ngot=%+v\n\n", i, tt.result, r)
		}
	}

}


<|start_filename|>internal/topo/sink/log_sink.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sink

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/topo/collector"
	"github.com/lf-edge/ekuiper/pkg/api"
	"sync"
	"time"
)

// NewLogSink log action, no properties now
// example: {"log":{}}
func NewLogSink() api.Sink {
	return collector.Func(func(ctx api.StreamContext, data interface{}) error {
		log := ctx.GetLogger()
		if v, _, err := ctx.TransformOutput(data); err == nil {
			log.Infof("sink result for rule %s: %s", ctx.GetRuleId(), v)
			return nil
		} else {
			return fmt.Errorf("transform data error: %v", err)
		}
	})
}

type QueryResult struct {
	Results   []string
	LastFetch time.Time
	Mux       sync.Mutex
}

var QR = &QueryResult{LastFetch: time.Now()}

func NewLogSinkToMemory() api.Sink {
	QR.Results = make([]string, 10)
	return collector.Func(func(ctx api.StreamContext, data interface{}) error {
		var result string
		if v, _, err := ctx.TransformOutput(data); err == nil {
			result = string(v)
		} else {
			return fmt.Errorf("transform data error: %v", err)
		}
		QR.Mux.Lock()
		QR.Results = append(QR.Results, result)
		QR.Mux.Unlock()
		return nil
	})
}


<|start_filename|>internal/plugin/plugin_test.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package plugin

import (
	"encoding/json"
	"fmt"
	"reflect"
	"testing"
)

func TestDecode(t *testing.T) {
	var tests = []struct {
		t          PluginType
		j          string
		name       string
		file       string
		shellParas []string
		symbols    []string
	}{
		{
			t:    PORTABLE,
			j:    "{\"name\":\"mirror\",\"file\":\"mirror_win.zip\"}",
			name: "mirror",
			file: "mirror_win.zip",
		}, {
			t:          SINK,
			j:          `{"name":"tdengine","file":"https://packages.emqx.io/kuiper-plugins/1.3.1/debian/sinks/tdengine_amd64.zip","shellParas": ["2.0.3.1"]}`,
			name:       "tdengine",
			file:       "https://packages.emqx.io/kuiper-plugins/1.3.1/debian/sinks/tdengine_amd64.zip",
			shellParas: []string{"2.0.3.1"},
		}, {
			t:       FUNCTION,
			j:       `{"name":"image","file":"https://packages.emqx.io/kuiper-plugins/1.3.1/debian/functions/image_amd64.zip","functions": ["resize","thumbnail"]}`,
			name:    "image",
			file:    "https://packages.emqx.io/kuiper-plugins/1.3.1/debian/functions/image_amd64.zip",
			symbols: []string{"resize", "thumbnail"},
		}, {
			t: FUNCTION,
			j: "{\"name1\":\"image\",\"file1\":\"abc\"}",
		},
	}
	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
	for i, tt := range tests {
		p := NewPluginByType(tt.t)
		err := json.Unmarshal([]byte(tt.j), p)
		if err != nil {
			t.Error(err)
			return
		}
		if tt.name != p.GetName() {
			t.Errorf("%d name mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.name, p.GetName())
		}
		if tt.file != p.GetFile() {
			t.Errorf("%d file mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.file, p.GetFile())
		}
		if !reflect.DeepEqual(tt.shellParas, p.GetShellParas()) {
			t.Errorf("%d shellParas mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.shellParas, p.GetShellParas())
		}
		if !reflect.DeepEqual(tt.symbols, p.GetSymbols()) {
			t.Errorf("%d symbols mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.symbols, p.GetSymbols())
		}
	}
}


<|start_filename|>internal/plugin/portable/model.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package portable

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
)

type PluginInfo struct {
	runtime.PluginMeta
	Sources   []string `json:"sources"`
	Sinks     []string `json:"sinks"`
	Functions []string `json:"functions"`
}

var langMap = map[string]bool{
	"go":     true,
	"python": true,
}

// Validate TODO validate duplication of source, sink and functions
func (p *PluginInfo) Validate(expectedName string) error {
	if p.Name != expectedName {
		return fmt.Errorf("invalid plugin, expect name '%s' but got '%s'", expectedName, p.Name)
	}
	if p.Language == "" {
		return fmt.Errorf("invalid plugin, missing language")
	}
	if p.Executable == "" {
		return fmt.Errorf("invalid plugin, missing executable")
	}
	if len(p.Sources)+len(p.Sinks)+len(p.Functions) == 0 {
		return fmt.Errorf("invalid plugin, must define at lease one source, sink or function")
	}
	if l, ok := langMap[p.Language]; !ok || !l {
		return fmt.Errorf("invalid plugin, language '%s' is not supported", p.Language)
	}
	return nil
}


<|start_filename|>sdk/go/mock/mock.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mock

import (
	"context"
	filename "github.com/keepeye/logrus-filename"
	"github.com/lf-edge/ekuiper/sdk/go/api"
	"github.com/sirupsen/logrus"
	"time"
)

type mockContext struct {
	Ctx    context.Context
	RuleId string
	OpId   string
}

//Implement context interface
func (c *mockContext) Deadline() (deadline time.Time, ok bool) {
	return c.Ctx.Deadline()
}

func (c *mockContext) Done() <-chan struct{} {
	return c.Ctx.Done()
}

func (c *mockContext) Err() error {
	return c.Ctx.Err()
}

func (c *mockContext) Value(key interface{}) interface{} {
	return c.Ctx.Value(key)
}

// Stream metas
func (c *mockContext) GetContext() context.Context {
	return c.Ctx
}

func (c *mockContext) GetLogger() api.Logger {
	return Logger
}

func (c *mockContext) GetRuleId() string {
	return c.RuleId
}

func (c *mockContext) GetOpId() string {
	return c.OpId
}

func (c *mockContext) GetInstanceId() int {
	return 0
}

func (c *mockContext) GetRootPath() string {
	//loc, _ := conf.GetLoc("")
	return "root path"
}

func (c *mockContext) SetError(err error) {

}

func (c *mockContext) WithMeta(ruleId string, opId string) api.StreamContext {
	return c
}

func (c *mockContext) WithInstance(_ int) api.StreamContext {
	return c
}

func (c *mockContext) WithCancel() (api.StreamContext, context.CancelFunc) {
	ctx, cancel := context.WithCancel(c.Ctx)
	return &mockContext{
		RuleId: c.RuleId,
		OpId:   c.OpId,
		Ctx:    ctx,
	}, cancel
}

func (c *mockContext) IncrCounter(key string, amount int) error {
	return nil
}

func (c *mockContext) GetCounter(key string) (int, error) {
	return 0, nil
}

func (c *mockContext) PutState(key string, value interface{}) error {
	return nil
}

func (c *mockContext) GetState(key string) (interface{}, error) {
	return nil, nil
}

func (c *mockContext) DeleteState(key string) error {
	return nil
}

func (c *mockContext) Snapshot() error {
	return nil
}

func (c *mockContext) SaveState(checkpointId int64) error {
	return nil
}

func newMockContext(ruleId string, opId string) api.StreamContext {
	return &mockContext{Ctx: context.Background(), RuleId: ruleId, OpId: opId}
}

type mockFuncContext struct {
	api.StreamContext
	funcId int
}

func (fc *mockFuncContext) GetFuncId() int {
	return fc.funcId
}

func newMockFuncContext(ctx api.StreamContext, id int) api.FunctionContext {
	return &mockFuncContext{
		StreamContext: ctx,
		funcId:        id,
	}
}

var Logger *logrus.Logger

func init() {
	l := logrus.New()
	filenameHook := filename.NewHook()
	filenameHook.Field = "file"
	l.AddHook(filenameHook)
	l.SetFormatter(&logrus.TextFormatter{
		TimestampFormat: "2006-01-02 15:04:05",
		DisableColors:   true,
		FullTimestamp:   true,
	})
	l.WithField("type", "main")
	Logger = l
}


<|start_filename|>Makefile<|end_filename|>
BUILD_PATH ?= _build
PACKAGES_PATH ?= _packages

VERSION := $(shell git describe --tags --always)
ARCH := $(shell go env GOARCH)
OS := $(shell go env GOOS)
PACKAGE_NAME := kuiper-$(VERSION)-$(OS)-$(ARCH)

TARGET ?= lfedge/ekuiper

export KUIPER_SOURCE := $(shell pwd)

.PHONY: build
build: build_without_edgex

.PHONY:pkg
pkg: pkg_without_edgex
	@if [ "$$(uname -s)" = "Linux" ]; then make -C deploy/packages; fi

.PHONY: build_prepare
build_prepare:
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/bin
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/etc
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/etc/sources
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/etc/sinks
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/etc/services
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/etc/services/schemas
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/data
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/plugins
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/plugins/sources
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/plugins/sinks
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/plugins/functions
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/plugins/portable
	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/log

	@cp -r etc/* $(BUILD_PATH)/$(PACKAGE_NAME)/etc

.PHONY: build_without_edgex
build_without_edgex: build_prepare
	GO111MODULE=on CGO_ENABLED=1 go build -ldflags="-s -w -X main.Version=$(VERSION) -X main.LoadFileType=relative" -o kuiper cmd/kuiper/main.go
	GO111MODULE=on CGO_ENABLED=1 go build -trimpath -ldflags="-s -w -X main.Version=$(VERSION) -X main.LoadFileType=relative" -o kuiperd cmd/kuiperd/main.go
	@if [ ! -z $$(which upx) ]; then upx ./kuiper; upx ./kuiperd; fi
	@mv ./kuiper ./kuiperd $(BUILD_PATH)/$(PACKAGE_NAME)/bin
	@echo "Build successfully"

.PHONY: pkg_without_edgex
pkg_without_edgex: build_without_edgex
	@make real_pkg

.PHONY: build_with_edgex
build_with_edgex: build_prepare
	GO111MODULE=on CGO_ENABLED=1 go build -ldflags="-s -w -X main.Version=$(VERSION) -X main.LoadFileType=relative" -tags edgex -o kuiper cmd/kuiper/main.go
	GO111MODULE=on CGO_ENABLED=1 go build -trimpath -ldflags="-s -w -X main.Version=$(VERSION) -X main.LoadFileType=relative" -tags edgex -o kuiperd cmd/kuiperd/main.go
	@if [ ! -z $$(which upx) ]; then upx ./kuiper; upx ./kuiperd; fi
	@mv ./kuiper ./kuiperd $(BUILD_PATH)/$(PACKAGE_NAME)/bin
	@echo "Build successfully"

.PHONY: pkg_with_edgex
pkg_with_edgex: build_with_edgex
	@make real_pkg

.PHONY: real_pkg
real_pkg:
	@mkdir -p $(PACKAGES_PATH)
	@cd $(BUILD_PATH) && zip -rq $(PACKAGE_NAME).zip $(PACKAGE_NAME)
	@cd $(BUILD_PATH) && tar -czf $(PACKAGE_NAME).tar.gz $(PACKAGE_NAME)
	@mv $(BUILD_PATH)/$(PACKAGE_NAME).zip $(BUILD_PATH)/$(PACKAGE_NAME).tar.gz $(PACKAGES_PATH)
	@echo "Package build success"

.PHONY: docker
docker:
	docker buildx build --no-cache --platform=linux/amd64 -t $(TARGET):$(VERSION) -f deploy/docker/Dockerfile . --load
	docker buildx build --no-cache --platform=linux/amd64 -t $(TARGET):$(VERSION)-slim -f deploy/docker/Dockerfile-slim . --load
	docker buildx build --no-cache --platform=linux/amd64 -t $(TARGET):$(VERSION)-alpine -f deploy/docker/Dockerfile-alpine . --load

PLUGINS := sinks/file \
	sinks/influx \
	sinks/zmq \
	sinks/image \
	sinks/redis \
	sources/random \
	sources/zmq \
	sinks/tdengine \
	functions/accumulateWordCount \
	functions/countPlusOne \
	functions/image \
	functions/geohash \
	functions/echo \
	functions/labelImage

.PHONY: plugins $(PLUGINS)
plugins: $(PLUGINS)

$(PLUGINS): PLUGIN_TYPE = $(word 1, $(subst /, , $@))
$(PLUGINS): PLUGIN_NAME = $(word 2, $(subst /, , $@))
$(PLUGINS):
	@$(CURDIR)/build-plugins.sh $(PLUGIN_TYPE) $(PLUGIN_NAME)

.PHONY: clean
clean:
	@rm -rf cross_build.tar linux_amd64 linux_arm64 linux_arm_v7 linux_386
	@rm -rf _build _packages _plugins


<|start_filename|>internal/conf/logger.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conf

import (
	filename "github.com/keepeye/logrus-filename"
	"github.com/sirupsen/logrus"
	"os"
	"strings"
)

const (
	logFileName = "stream.log"
)

var (
	Log     *logrus.Logger
	logFile *os.File
)

func InitLogger() {
	Log = logrus.New()
	initSyslog()
	filenameHook := filename.NewHook()
	filenameHook.Field = "file"
	Log.AddHook(filenameHook)

	Log.SetFormatter(&logrus.TextFormatter{
		TimestampFormat: "2006-01-02 15:04:05",
		DisableColors:   true,
		FullTimestamp:   true,
	})

	Log.Debugf("init with args %s", os.Args)
	for _, arg := range os.Args {
		if strings.HasPrefix(arg, "-test.") {
			IsTesting = true
			break
		}
	}
}

func CloseLogger() {
	if logFile != nil {
		logFile.Close()
	}
}


<|start_filename|>internal/plugin/portable/runtime/source.go<|end_filename|>
// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package runtime

import (
	"encoding/json"
	"fmt"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/errorx"
	"go.nanomsg.org/mangos/v3"
)

// Error handling: wrap all error in a function to handle

type PortableSource struct {
	symbolName string
	reg        *PluginMeta
	clean      func() error

	topic string
	props map[string]interface{}
}

func NewPortableSource(symbolName string, reg *PluginMeta) *PortableSource {
	return &PortableSource{
		symbolName: symbolName,
		reg:        reg,
	}
}

func (ps *PortableSource) Open(ctx api.StreamContext, consumer chan<- api.SourceTuple, errCh chan<- error) {
	ctx.GetLogger().Infof("Start running portable source %s with datasource %s and conf %+v", ps.symbolName, ps.topic, ps.props)
	pm := GetPluginInsManager()
	ins, err := pm.getOrStartProcess(ps.reg, PortbleConf)
	if err != nil {
		errCh <- err
		return
	}
	ctx.GetLogger().Infof("Plugin started successfully")

	// wait for plugin data
	dataCh, err := CreateSourceChannel(ctx)
	if err != nil {
		errCh <- err
		return
	}

	// Control: send message to plugin to ask starting symbol
	c := &Control{
		Meta: &Meta{
			RuleId:     ctx.GetRuleId(),
			OpId:       ctx.GetOpId(),
			InstanceId: ctx.GetInstanceId(),
		},
		SymbolName: ps.symbolName,
		PluginType: TYPE_SOURCE,
		DataSource: ps.topic,
		Config:     ps.props,
	}
	err = ins.StartSymbol(ctx, c)
	if err != nil {
		errCh <- err
		_ = dataCh.Close()
		return
	}
	ps.clean = func() error {
		ctx.GetLogger().Info("clean up source")
		err1 := dataCh.Close()
		err2 := ins.StopSymbol(ctx, c)
		e := make(errorx.MultiError)
		if err1 != nil {
			e["dataCh"] = err1
		}
		if err != nil {
			e["symbol"] = err2
		}
		return e.GetError()
	}

	for {
		var msg []byte
		// make sure recv has timeout
		msg, err = dataCh.Recv()
		switch err {
		case mangos.ErrClosed:
			ctx.GetLogger().Info("stop source after close")
			return
		case mangos.ErrRecvTimeout:
			ctx.GetLogger().Debug("source receive timeout, retry")
			continue
		case nil:
			// do nothing
		default:
			errCh <- fmt.Errorf("cannot receive from mangos Socket: %s", err.Error())
			return
		}
		result := &api.DefaultSourceTuple{}
		e := json.Unmarshal(msg, result)
		if e != nil {
			ctx.GetLogger().Errorf("Invalid data format, cannot decode %s to json format with error %s", string(msg), e)
			continue
		}
		select {
		case consumer <- result:
			ctx.GetLogger().Debugf("send data to source node")
		case <-ctx.Done():
			ctx.GetLogger().Info("stop source")
			return
		}
	}
}

func (ps *PortableSource) Configure(topic string, props map[string]interface{}) error {
	ps.topic = topic
	ps.props = props
	return nil
}

func (ps *PortableSource) Close(ctx api.StreamContext) error {
	ctx.GetLogger().Infof("Closing source %s", ps.symbolName)
	if ps.clean != nil {
		return ps.clean()
	}
	return nil
}


<|start_filename|>internal/topo/connection/manager.go<|end_filename|>
package connection

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/conf"
	"sync"
)

var m = clientManager{
	clientFactory:       make(map[string]ClientFactoryFunc),
	lock:                sync.Mutex{},
	supportedClientType: make([]string, 0),
	clientMap:           make(map[string]*clientWrapper),
}

type clientManager struct {
	lock                sync.Mutex
	supportedClientType []string
	clientFactory       map[string]ClientFactoryFunc
	clientMap           map[string]*clientWrapper
}

func registerClientFactory(clientType string, creatorFunc ClientFactoryFunc) {
	m.lock.Lock()
	m.clientFactory[clientType] = creatorFunc
	m.supportedClientType = append(m.supportedClientType, clientType)
	m.lock.Unlock()
}

func GetConnection(connectSelector string) (interface{}, error) {

	m.lock.Lock()
	defer m.lock.Unlock()

	var cliWpr *clientWrapper
	var found bool

	cliWpr, found = m.clientMap[connectSelector]
	if found {
		cliWpr.addRef()
	} else {
		selectCfg := &conf.ConSelector{
			ConnSelectorStr: connectSelector,
		}
		err := selectCfg.Init()
		if err != nil {
			conf.Log.Errorf("connection selector: %s have error %s.", connectSelector, err)
			return nil, err
		}

		clientCreator, ok := m.clientFactory[selectCfg.Type]
		if !ok {
			conf.Log.Errorf("can not find clientCreator for connection selector : %s only support %s", connectSelector, m.supportedClientType)
			return nil, fmt.Errorf("can not find clientCreator for connection selector : %s. only support %s", connectSelector, m.supportedClientType)
		}

		client := clientCreator(selectCfg)

		cliWpr, err = NewClientWrapper(client, selectCfg)
		if err != nil {
			conf.Log.Errorf("can not create client for connection selector : %s have error %s", connectSelector, err)
			return nil, err
		}

		m.clientMap[connectSelector] = cliWpr
	}

	conf.Log.Infof("connection selector: %s GetConnection count %d.", connectSelector, cliWpr.refCnt)

	return cliWpr.getInstance(), nil
}

func ReleaseConnection(connectSelector string) {
	m.lock.Lock()
	defer m.lock.Unlock()
	if v, ok := m.clientMap[connectSelector]; ok {
		v.subRef()
		conf.Log.Infof("connection selector: %s ReleaseConnection count %d.", connectSelector, v.refCnt)
		if v.IsRefEmpty() {
			v.clean()
			delete(m.clientMap, connectSelector)
		}
	}
}


<|start_filename|>internal/topo/sink/rest_sink.go<|end_filename|>
// Copyright 2022 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sink

import (
	"encoding/json"
	"fmt"
	"github.com/lf-edge/ekuiper/internal/pkg/cert"
	"github.com/lf-edge/ekuiper/internal/pkg/httpx"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/errorx"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type RestSink struct {
	method             string
	url                string
	headers            map[string]string
	headersTemplate    string
	bodyType           string
	timeout            int64
	sendSingle         bool
	debugResp          bool
	insecureSkipVerify bool
	certificationPath  string
	privateKeyPath     string
	rootCaPath         string

	client *http.Client
}

var methodsMap = map[string]bool{"GET": true, "HEAD": true, "POST": true, "PUT": true, "DELETE": true, "PATCH": true}

func (ms *RestSink) Configure(ps map[string]interface{}) error {
	temp, ok := ps["method"]
	if ok {
		ms.method, ok = temp.(string)
		if !ok {
			return fmt.Errorf("rest sink property method %v is not a string", temp)
		}
		ms.method = strings.ToUpper(strings.Trim(ms.method, ""))
	} else {
		ms.method = "GET"
	}
	if _, ok = methodsMap[ms.method]; !ok {
		return fmt.Errorf("invalid property method: %s", ms.method)
	}
	switch ms.method {
	case "GET", "HEAD":
		ms.bodyType = "none"
	default:
		ms.bodyType = "json"
	}

	temp, ok = ps["url"]
	if !ok {
		return fmt.Errorf("rest sink is missing property url")
	}
	ms.url, ok = temp.(string)
	if !ok {
		return fmt.Errorf("rest sink property url %v is not a string", temp)
	}
	ms.url = strings.Trim(ms.url, "")

	temp, ok = ps["headers"]
	if ok {
		switch h := temp.(type) {
		case map[string]interface{}:
			ms.headers = make(map[string]string)
			for k, v := range h {
				if v1, ok1 := v.(string); ok1 {
					ms.headers[k] = v1
				} else {
					return fmt.Errorf("header value %s for header %s is not a string", v, k)
				}
			}
		case string:
			ms.headersTemplate = h
		default:
			return fmt.Errorf("rest sink property headers %v is not a map[string]interface", temp)
		}
	}

	temp, ok = ps["bodyType"]
	if ok {
		ms.bodyType, ok = temp.(string)
		if !ok {
			return fmt.Errorf("rest sink property bodyType %v is not a string", temp)
		}
		ms.bodyType = strings.ToLower(strings.Trim(ms.bodyType, ""))
	}
	if _, ok = httpx.BodyTypeMap[ms.bodyType]; !ok {
		return fmt.Errorf("invalid property bodyType: %s, should be \"none\" or \"form\"", ms.bodyType)
	}

	temp, ok = ps["timeout"]
	if !ok {
		ms.timeout = 5000
	} else {
		to, ok := temp.(float64)
		if !ok {
			return fmt.Errorf("rest sink property timeout %v is not a number", temp)
		}
		ms.timeout = int64(to)
	}

	temp, ok = ps["sendSingle"]
	if !ok {
		ms.sendSingle = false
	} else {
		ms.sendSingle, ok = temp.(bool)
		if !ok {
			return fmt.Errorf("rest sink property sendSingle %v is not a bool", temp)
		}
	}

	temp, ok = ps["debugResp"]
	if !ok {
		ms.debugResp = false
	} else {
		ms.debugResp, ok = temp.(bool)
		if !ok {
			return fmt.Errorf("rest sink property debugResp %v is not a bool", temp)
		}
	}

	temp, ok = ps["insecureSkipVerify"]
	if !ok {
		ms.insecureSkipVerify = true
	} else {
		ms.insecureSkipVerify, ok = temp.(bool)
		if !ok {
			return fmt.Errorf("rest sink property insecureSkipVerify %v is not a bool", temp)
		}
	}

	if certPath, ok := ps["certificationPath"]; ok {
		if certPath1, ok1 := certPath.(string); ok1 {
			ms.certificationPath = certPath1
		} else {
			return fmt.Errorf("not valid rest sink property certificationPath value %v", certPath)
		}
	}

	if privPath, ok := ps["privateKeyPath"]; ok {
		if privPath1, ok1 := privPath.(string); ok1 {
			ms.privateKeyPath = privPath1
		} else {
			return fmt.Errorf("not valid rest sink property privateKeyPath value %v", privPath)
		}
	}

	if rootPath, ok := ps["rootCaPath"]; ok {
		if rootPath1, ok1 := rootPath.(string); ok1 {
			ms.rootCaPath = rootPath1
		} else {
			return fmt.Errorf("not valid rest sink property rootCaPath value %v", rootPath)
		}
	}

	return nil
}

func (ms *RestSink) Open(ctx api.StreamContext) error {
	logger := ctx.GetLogger()

	tlsOpts := cert.TlsConfigurationOptions{
		SkipCertVerify: ms.insecureSkipVerify,
		CertFile:       ms.certificationPath,
		KeyFile:        ms.privateKeyPath,
		CaFile:         ms.rootCaPath,
	}
	tlscfg, err := cert.GenerateTLSForClient(tlsOpts)
	if err != nil {
		return err
	}

	tr := &http.Transport{
		TLSClientConfig: tlscfg,
	}

	ms.client = &http.Client{
		Transport: tr,
		Timeout:   time.Duration(ms.timeout) * time.Millisecond}
	logger.Infof("open rest sink with configuration: {method: %s, url: %s, bodyType: %s, timeout: %d, header: %v, sendSingle: %v, tls cfg: %v", ms.method, ms.url, ms.bodyType, ms.timeout, ms.headers, ms.sendSingle, tlsOpts)

	if _, err := url.Parse(ms.url); err != nil {
		return err
	}
	return nil
}

type MultiErrors []error

func (me MultiErrors) AddError(err error) MultiErrors {
	me = append(me, err)
	return me
}

func (me MultiErrors) Error() string {
	s := make([]string, len(me))
	for i, v := range me {
		s = append(s, fmt.Sprintf("Error %d with info %s. \n", i, v))
	}
	return strings.Join(s, "  ")
}

func (ms *RestSink) Collect(ctx api.StreamContext, item interface{}) error {
	logger := ctx.GetLogger()
	logger.Debugf("rest sink receive %s", item)
	output, transed, err := ctx.TransformOutput(item)
	if err != nil {
		logger.Warnf("rest sink decode data error: %v", err)
		return nil
	}
	var d = item
	if transed {
		d = output
	}
	resp, err := ms.Send(ctx, d, logger)
	if err != nil {
		return fmt.Errorf("rest sink fails to send out the data: %s", err)
	} else {
		defer resp.Body.Close()
		logger.Debugf("rest sink got response %v", resp)
		if resp.StatusCode < 200 || resp.StatusCode > 299 {
			if buf, bodyErr := ioutil.ReadAll(resp.Body); bodyErr != nil {
				logger.Errorf("%s\n", bodyErr)
				return fmt.Errorf("%s: http return code: %d and error message %s", errorx.IOErr, resp.StatusCode, bodyErr)
			} else {
				logger.Errorf("%s\n", string(buf))
				return fmt.Errorf("%s: http return code: %d and error message %s", errorx.IOErr, resp.StatusCode, string(buf))
			}
		} else {
			if ms.debugResp {
				if buf, bodyErr := ioutil.ReadAll(resp.Body); bodyErr != nil {
					logger.Errorf("%s\n", bodyErr)
				} else {
					logger.Infof("Response content: %s\n", string(buf))
				}
			}
		}
	}
	return nil
}

func (ms *RestSink) Send(ctx api.StreamContext, v interface{}, logger api.Logger) (*http.Response, error) {
	temp, err := ctx.ParseDynamicProp(ms.bodyType, v)
	if err != nil {
		return nil, err
	}
	bodyType, ok := temp.(string)
	if !ok {
		return nil, fmt.Errorf("the value %v of dynamic prop %s for bodyType is not a string", ms.bodyType, temp)
	}
	temp, err = ctx.ParseDynamicProp(ms.method, v)
	if err != nil {
		return nil, err
	}
	method, ok := temp.(string)
	if !ok {
		return nil, fmt.Errorf("the value %v of dynamic prop %s for method is not a string", ms.method, temp)
	}
	temp, err = ctx.ParseDynamicProp(ms.url, v)
	if err != nil {
		return nil, err
	}
	u, ok := temp.(string)
	if !ok {
		return nil, fmt.Errorf("the value %v of dynamic prop %s for url is not a string", ms.url, temp)
	}
	var headers map[string]string
	if ms.headers != nil {
		headers = ms.headers
	} else if ms.headersTemplate != "" {
		temp, err = ctx.ParseDynamicProp(ms.headersTemplate, v)
		if err != nil {
			return nil, err
		}
		tstr, ok := temp.(string)
		if !ok {
			return nil, fmt.Errorf("the value %v of dynamic prop %s for headersTemplate is not a string", ms.headersTemplate, temp)
		}
		err = json.Unmarshal([]byte(tstr), &headers)
		if err != nil {
			return nil, fmt.Errorf("rest sink headers template decode error: %v", err)
		}
	}
	return httpx.Send(logger, ms.client, bodyType, method, u, headers, ms.sendSingle, v)
}

func (ms *RestSink) Close(ctx api.StreamContext) error {
	logger := ctx.GetLogger()
	logger.Infof("Closing rest sink")
	return nil
}
