<|start_filename|>pkg/compute/tasks/guest_save_instance_image_task.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
	"context"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type GuestSaveGuestImageTask struct {
	SGuestBaseTask
}

func init() {
	taskman.RegisterTask(GuestSaveGuestImageTask{})
}

func (self *GuestSaveGuestImageTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
	// prepare save image
	guest := obj.(*models.SGuest)

	self.SetStage("OnSaveRootImageComplete", nil)
	disks := guest.CategorizeDisks()
	imageIds, _ := self.Params.GetArray("image_ids")
	self.Params.Remove("image_ids")

	// data disk
	for index, dataDisk := range disks.Data {
		params := jsonutils.DeepCopy(self.Params).(*jsonutils.JSONDict)
		params.Add(imageIds[index], "image_id")
		opts := api.DiskSaveInput{ImageId: imageIds[index].String()}
		if err := dataDisk.StartDiskSaveTask(ctx, self.UserCred, opts, self.GetTaskId()); err != nil {
			self.taskFailed(ctx, guest, jsonutils.NewString(err.Error()))
		}
	}

	self.Params.Add(imageIds[len(imageIds)-1], "image_id")
	opts := api.DiskSaveInput{ImageId: imageIds[len(imageIds)-1].String()}
	if err := disks.Root.StartDiskSaveTask(ctx, self.UserCred, opts, self.GetTaskId()); err != nil {
		self.taskFailed(ctx, guest, jsonutils.NewString(err.Error()))
	}
}

func (self *GuestSaveGuestImageTask) OnSaveRootImageComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
	subTasks := taskman.SubTaskManager.GetTotalSubtasks(self.Id, "on_save_root_image_complete", taskman.SUBTASK_FAIL)

	if len(subTasks) > 0 {
		self.taskFailed(ctx, guest, jsonutils.NewString("subtask failed"))
		// ??? return ???
	}

	if restart, _ := self.GetParams().Bool("auto_start"); restart {
		self.SetStage("on_start_server_complete", nil)
		guest.StartGueststartTask(ctx, self.GetUserCred(), nil, self.GetTaskId())
	} else {
		guest.SetStatus(self.UserCred, api.VM_READY, "")
		self.taskSuc(ctx, guest)
	}
}

func (self *GuestSaveGuestImageTask) OnSaveRootImageCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
	log.Errorf("Guest save image failed: %s", data.PrettyString())
	self.taskFailed(ctx, guest, data)
}

func (self *GuestSaveGuestImageTask) OnStartServerComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
	self.taskSuc(ctx, guest)
}

func (self *GuestSaveGuestImageTask) OnStartServerCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
	// even if start server failed, the task that save guest image is successful
	self.taskSuc(ctx, guest)
}

func (self *GuestSaveGuestImageTask) taskSuc(ctx context.Context, guest *models.SGuest) {
	self.SetStageComplete(ctx, nil)
}

func (self *GuestSaveGuestImageTask) taskFailed(ctx context.Context, guest *models.SGuest, reason jsonutils.JSONObject) {

	guest.SetStatus(self.UserCred, api.VM_SAVE_DISK_FAILED, reason.String())
	db.OpsLog.LogEvent(guest, db.ACT_GUEST_SAVE_GUEST_IMAGE_FAIL, reason, self.UserCred)
	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_IMAGE_SAVE, reason, self.UserCred, false)

	self.SetStageFailed(ctx, reason)
}


<|start_filename|>pkg/esxi/handler/handlers.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package handler

import (
	"context"
	"fmt"
	"net/http"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	"yunion.io/x/onecloud/pkg/appctx"
	"yunion.io/x/onecloud/pkg/appsrv"
	"yunion.io/x/onecloud/pkg/esxi"
	"yunion.io/x/onecloud/pkg/hostman/hostutils"
	"yunion.io/x/onecloud/pkg/hostman/storageman"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/mcclient/auth"
)

const (
	AGENT_PREFIX = "disks/agent"
)

func InitHandlers(app *appsrv.Application) {
	initESXIHandler(app)
}

var defaultHandler = func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	httperrors.NotImplementedError(ctx, w, "")
	return
}

func IdAgentPrefix(action string) string {
	return fmt.Sprintf("%s/%s/<disk_id>", AGENT_PREFIX, action)
}

func AgentPrefix(action string) string {
	return fmt.Sprintf("%s/%s", AGENT_PREFIX, action)
}

func initESXIHandler(app *appsrv.Application) {

	app.AddHandler("POST", AgentPrefix("upload"), auth.Authenticate(uploadHandler))
	app.AddHandler("POST", AgentPrefix("deploy"), auth.Authenticate(deployHandler))
	app.AddHandler("POST", IdAgentPrefix("delete"), auth.Authenticate(deleteHandler))
	app.AddHandler("POST", IdAgentPrefix("create"), auth.Authenticate(createHandler))
	app.AddHandler("POST", IdAgentPrefix("save-prepare"), auth.Authenticate(savePrepareHandler))
	app.AddHandler("POST", IdAgentPrefix("resize"), auth.Authenticate(resizeHandler))
	app.AddHandler("POST", IdAgentPrefix("clone"), auth.Authenticate(defaultHandler))
	app.AddHandler("POST", IdAgentPrefix("fetch"), auth.Authenticate(defaultHandler))
	app.AddHandler("POST", IdAgentPrefix("post-migrate"), auth.Authenticate(defaultHandler))
	app.AddHandler("POST", IdAgentPrefix("snapshot"), auth.Authenticate(defaultHandler))
	app.AddHandler("POST", IdAgentPrefix("reset"), auth.Authenticate(defaultHandler))
	app.AddHandler("POST", IdAgentPrefix("cleanup-snapshots"), auth.Authenticate(defaultHandler))
}

func uploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	_, _, body := appsrv.FetchEnv(ctx, w, r)
	disk, err := body.Get("disk")
	if err != nil {
		httperrors.MissingParameterError(ctx, w, "miss disk")
		return
	}
	hostutils.DelayTaskWithoutReqctx(ctx, esxi.EsxiAgent.AgentStorage.SaveToGlance, disk)
	hostutils.ResponseOk(ctx, w)
}

func deployHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	log.Debugf("enter deployHandler")
	_, _, body := appsrv.FetchEnv(ctx, w, r)
	disk, err := body.Get("disk")
	if err != nil {
		httperrors.MissingParameterError(ctx, w, "miss disk")
		return
	}
	hostutils.DelayTask(ctx, esxi.EsxiAgent.AgentStorage.AgentDeployGuest, disk)
	hostutils.ResponseOk(ctx, w)
}

func deleteHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	params, _, _ := appsrv.FetchEnv(ctx, w, r)
	diskId := params["<disk_id>"]
	disk, err := esxi.EsxiAgent.AgentStorage.GetDiskById(diskId)
	if err != nil {
		httperrors.GeneralServerError(ctx, w, errors.Wrapf(err, "GetDiskById(%s)", diskId))
		return
	}
	if taskId := ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID); taskId == nil {
		if disk != nil {
			_, err := disk.Delete(ctx, nil)
			if err != nil {
				httperrors.GeneralServerError(ctx, w, err)
				return
			}
			hostutils.ResponseOk(ctx, w)
			return
		}
	}
	hostutils.ResponseOk(ctx, w)
	hostutils.DelayTask(ctx, disk.Delete, nil)
}

func createHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	disk, diskInfo, err := diskAndDiskInfo(ctx, w, r)
	if err != nil {
		httperrors.GeneralServerError(ctx, w, err)
		return
	}
	hostutils.DelayTask(ctx, esxi.EsxiAgent.AgentStorage.CreateDiskByDiskInfo,
		storageman.SDiskCreateByDiskinfo{DiskId: disk.GetId(), Disk: disk, DiskInfo: diskInfo})
	hostutils.ResponseOk(ctx, w)
}

func savePrepareHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	taskId := ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID).(string)
	disk, diskInfo, err := diskAndDiskInfo(ctx, w, r)
	if err != nil {
		httperrors.GeneralServerError(ctx, w, err)
		return
	}
	hostutils.DelayTask(ctx, disk.PrepareSaveToGlance, storageman.PrepareSaveToGlanceParams{
		TaskId:   taskId,
		DiskInfo: diskInfo,
	})
	hostutils.ResponseOk(ctx, w)
}

func resizeHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	disk, diskInfo, err := diskAndDiskInfo(ctx, w, r)
	if err != nil {
		httperrors.GeneralServerError(ctx, w, err)
		return
	}
	hostutils.DelayTask(ctx, disk.Resize, diskInfo)
	hostutils.ResponseOk(ctx, w)
}

/*
func fetchHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	params, _, body := appsrv.FetchEnv(ctx, w, r)
	diskId := params["<disk_id>"]
	disk := esxi.EsxiAgent.AgentStorage.GetDiskById(diskId)
	if disk != nil {
		httperrors.GeneralServerError(ctx, w, httperrors.NewDuplicateResourceError("disk '%s'", diskId))
		return
	}
	disk := esxi.EsxiAgent.AgentStorage.CreateDisk(diskId)
	diskInfo, err := body.Get("disk")
	if err != nil {
		httperrors.InputParameterError(ctx, w, "miss disk")
	}
	url, err := diskInfo.GetString("url")
	if err != nil {
		httperrors.InputParameterError(ctx, w, "miss disk.url")
	}
	hostutils.DelayTask(ctx, disk.CreateFromUrl)
	hostutils.ResponseOk(ctx, w)
}
*/

func diskAndDiskInfo(ctx context.Context, w http.ResponseWriter, r *http.Request) (storageman.IDisk, jsonutils.JSONObject, error) {
	params, _, body := appsrv.FetchEnv(ctx, w, r)
	diskId := params["<disk_id>"]
	disk, err := esxi.EsxiAgent.AgentStorage.GetDiskById(diskId)
	if err != nil {
		return nil, nil, httperrors.NewGeneralError(errors.Wrapf(err, "GetDiskById(%s)", diskId))
	}
	diskInfo, err := body.Get("disk")
	if err != nil {
		return nil, nil, httperrors.NewMissingParameterError("miss disk")
	}
	return disk, diskInfo, nil
}


<|start_filename|>pkg/compute/models/networkaddresses.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package models

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/util/delayedwork"
	"yunion.io/x/sqlchemy"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/util/rbacutils"
	"yunion.io/x/onecloud/pkg/util/stringutils2"
)

type SNetworkAddressManager struct {
	db.SStandaloneAnonResourceBaseManager
	SNetworkResourceBaseManager

	delayedWorkManager *delayedwork.DelayedWorkManager
}

var NetworkAddressManager *SNetworkAddressManager

func init() {
	NetworkAddressManager = &SNetworkAddressManager{
		SStandaloneAnonResourceBaseManager: db.NewStandaloneAnonResourceBaseManager(
			SNetworkAddress{},
			"networkaddresses_tbl",
			"networkaddress",
			"networkaddresses",
		),
	}
	NetworkAddressManager.SetVirtualObject(NetworkAddressManager)
	NetworkAddressManager.delayedWorkManager = delayedwork.NewDelayedWorkManager()
}

type SNetworkAddress struct {
	db.SStandaloneAnonResourceBase

	Type       string `width:"16" charset:"ascii" list:"user" create:"required"`
	ParentType string `width:"16" charset:"ascii" list:"user" create:"required"`
	ParentId   string `width:"16" charset:"ascii" list:"user" create:"optional"`
	SNetworkResourceBase
	IpAddr string `width:"16" charset:"ascii" list:"user" create:"optional"`

	SubCtrVid int `create:"optional"`
}

func (man *SNetworkAddressManager) InitializeData() error {
	go man.delayedWorkManager.Start(context.Background())
	return nil
}

func (man *SNetworkAddressManager) queryByParentTypeId(ctx context.Context, typ string, id string) *sqlchemy.SQuery {
	q := NetworkAddressManager.Query().
		Equals("parent_type", typ).
		Equals("parent_id", id)
	return q
}

func (man *SNetworkAddressManager) queryByGuestnetworkId(ctx context.Context, rowid int64) *sqlchemy.SQuery {
	id := strconv.FormatInt(rowid, 10)
	return man.queryByParentTypeId(ctx, api.NetworkAddressParentTypeGuestnetwork, id)
}

func (man *SNetworkAddressManager) fetchByParentTypeId(ctx context.Context, typ string, id string) ([]SNetworkAddress, error) {
	var (
		q   = man.queryByParentTypeId(ctx, typ, id)
		nas []SNetworkAddress
	)
	if err := db.FetchModelObjects(man, q, &nas); err != nil {
		return nil, err
	}
	return nas, nil
}

func (man *SNetworkAddressManager) fetchByGuestnetworkId(ctx context.Context, rowid int64) ([]SNetworkAddress, error) {
	id := strconv.FormatInt(rowid, 10)
	return man.fetchByParentTypeId(ctx, api.NetworkAddressParentTypeGuestnetwork, id)
}

type addrConf struct {
	Type    string `json:"type"`
	IpAddr  string `json:"ip_addr"`
	Masklen int    `json:"masklen"`
	Gateway string `json:"gateway"`
}

func (man *SNetworkAddressManager) fetchAddressesByGuestnetworkId(ctx context.Context, rowid int64) ([]addrConf, error) {
	var (
		naq     = man.queryByGuestnetworkId(ctx, rowid).SubQuery()
		nq      = NetworkManager.Query().SubQuery()
		ipnetsq = naq.Query(
			naq.Field("type"),
			naq.Field("ip_addr"),
			nq.Field("guest_ip_mask").Label("masklen"),
			nq.Field("guest_gateway").Label("gateway"),
		).Join(nq, sqlchemy.Equals(
			naq.Field("network_id"),
			nq.Field("id")),
		)
		ipnets []addrConf
	)
	if err := ipnetsq.All(&ipnets); err != nil {
		return nil, errors.Wrapf(err, "fetch addresses ipnets by guestnetwork row id: %d", rowid)
	}
	return ipnets, nil
}

func (man *SNetworkAddressManager) deleteByGuestnetworkId(ctx context.Context, userCred mcclient.TokenCredential, rowid int64) error {
	nas, err := NetworkAddressManager.fetchByGuestnetworkId(ctx, rowid)
	if err != nil {
		return errors.Wrap(err, "fetch attached network addresses")
	}
	var errs []error
	for i := range nas {
		na := &nas[i]
		if err := na.remoteUnassignAddress(ctx, userCred); err != nil {
			errs = append(errs, err)
			continue
		}
		if err := db.DeleteModel(ctx, userCred, na); err != nil {
			errs = append(errs, err)
			continue
		}
	}
	return errors.NewAggregate(errs)
}

func (man *SNetworkAddressManager) syncGuestnetworkICloudNic(ctx context.Context, userCred mcclient.TokenCredential, guestnetwork *SGuestnetwork, iNic cloudprovider.ICloudNic) error {
	ipAddrs, err := iNic.GetSubAddress()
	if err != nil {
		return err
	}
	if err := man.syncGuestnetworkSubIPs(ctx, userCred, guestnetwork, ipAddrs); err != nil {
		return err
	}
	return nil
}

func (man *SNetworkAddressManager) syncGuestnetworkSubIPs(ctx context.Context, userCred mcclient.TokenCredential, guestnetwork *SGuestnetwork, ipAddrs []string) error {
	nas, err := man.fetchByGuestnetworkId(ctx, guestnetwork.RowId)
	if err != nil {
		return errors.Wrap(err, "fetchByGuestnetworkId")
	}
	var gotIpAddrs []string
	for i := range nas {
		gotIpAddrs = append(gotIpAddrs, nas[i].IpAddr)
	}

	var removes, adds []string
	for _, ip0 := range gotIpAddrs {
		ok := false
		for _, ip1 := range ipAddrs {
			if ip0 == ip1 {
				ok = true
				break
			}
		}
		if !ok {
			removes = append(removes, ip0)
		}
	}
	for _, ip1 := range ipAddrs {
		ok := false
		for _, ip0 := range gotIpAddrs {
			if ip0 == ip1 {
				ok = true
			}
		}
		if !ok {
			adds = append(adds, ip1)
		}
	}
	if err := man.removeGuestnetworkSubIPs(ctx, userCred, guestnetwork, removes); err != nil {
		return err
	}
	if err := man.addGuestnetworkSubIPs(ctx, userCred, guestnetwork, adds); err != nil {
		return err
	}
	return nil
}

func (man *SNetworkAddressManager) removeGuestnetworkSubIPs(ctx context.Context, userCred mcclient.TokenCredential, guestnetwork *SGuestnetwork, ipAddrs []string) error {
	q := man.queryByGuestnetworkId(ctx, guestnetwork.RowId)
	q = q.In("ip_addr", ipAddrs)

	var nas []SNetworkAddress
	if err := db.FetchModelObjects(man, q, &nas); err != nil {
		return errors.Wrapf(err, "removeGuestnetworkSubIPs by ipAddrs %s", strings.Join(ipAddrs, ", "))
	}

	for i := range nas {
		na := &nas[i]
		if err := db.DeleteModel(ctx, userCred, na); err != nil {
			return err
		}
	}
	return nil
}

func (man *SNetworkAddressManager) addGuestnetworkSubIPs(ctx context.Context, userCred mcclient.TokenCredential, guestnetwork *SGuestnetwork, ipAddrs []string) error {
	net := guestnetwork.GetNetwork()
	if net == nil {
		return errors.Wrapf(errors.ErrNotFound, "find network %s of guestnetwork %d",
			guestnetwork.NetworkId, guestnetwork.RowId)
	}

	lockman.LockObject(ctx, net)
	defer lockman.ReleaseObject(ctx, net)
	var (
		usedAddrMap = net.GetUsedAddresses()
		usedAddrs   []string
	)
	for _, ipAddr := range ipAddrs {
		if _, ok := usedAddrMap[ipAddr]; ok {
			usedAddrs = append(usedAddrs, ipAddr)
		}
	}
	if len(usedAddrs) > 0 {
		return errors.Errorf("addGuestnetworkSubIPs: ipaddr %s already used", strings.Join(usedAddrs, ", "))
	}
	for _, ipAddr := range ipAddrs {
		m, err := db.NewModelObject(man)
		if err != nil {
			return errors.Wrapf(err, "addGuestnetworkSubIPs")
		}
		na := m.(*SNetworkAddress)
		na.ParentType = api.NetworkAddressParentTypeGuestnetwork
		na.ParentId = strconv.FormatInt(guestnetwork.RowId, 10)
		na.Type = api.NetworkAddressTypeSubIP
		na.IpAddr = ipAddr
		if err := man.TableSpec().Insert(ctx, na); err != nil {
			return errors.Wrapf(err, "addGuestnetworkSubIPs")
		}
	}
	return nil
}

func (man *SNetworkAddressManager) BatchPreValidate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict, count int) error {
	var (
		input api.NetworkAddressCreateInput
		err   error
	)
	if err = man.SStandaloneAnonResourceBaseManager.BatchPreValidate(ctx, userCred, ownerId, query, data, count); err != nil {
		return err
	}
	if err = data.Unmarshal(&input); err != nil {
		return err
	}
	input, err = man.validateCreateData(ctx, userCred, ownerId, query, input, count)
	if err != nil {
		return err
	}
	data.Update(input.JSON(input))
	return nil
}

func (man *SNetworkAddressManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.NetworkAddressCreateInput) (api.NetworkAddressCreateInput, error) {
	return man.validateCreateData(ctx, userCred, ownerId, query, input, 1)
}

func (man *SNetworkAddressManager) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.NetworkAddressCreateInput, count int) (api.NetworkAddressCreateInput, error) {
	if _, err := man.SStandaloneAnonResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StandaloneAnonResourceCreateInput); err != nil {
		return input, err
	}
	var (
		net *SNetwork
	)
	switch input.ParentType {
	case api.NetworkAddressParentTypeGuestnetwork:
		switch input.Type {
		case api.NetworkAddressTypeSubIP:
			if input.GuestId != "" {
				gM, err := GuestManager.FetchByIdOrName(userCred, input.GuestId)
				if err != nil {
					return input, httperrors.NewInputParameterError("fetch guest %s: %v", input.GuestId, err)
				}
				gn, err := GuestnetworkManager.FetchByGuestIdIndex(gM.GetId(), input.GuestnetworkIndex)
				if err != nil {
					return input, httperrors.NewInputParameterError("fetch guest nic: %v", err)
				}
				net = gn.GetNetwork()
				if net == nil {
					return input, httperrors.NewInternalServerError("cannot fetch network of guestnetwork %d", gn.RowId)
				}
				if net.IsManaged() && count > 1 {
					// barriers for batch create
					// - dispatcher has policy of one fail,
					//   all fail for batch creation
					// - the revert action there only marks
					//   local models as deleted without
					//   cleanup of already assigned private
					//   addresses
					return input, httperrors.NewNotSupportedError("batch create is not supported for external resources")
				}
				input.ParentId = gn.RowId
				input.NetworkId = net.Id
			} else {
				return input, httperrors.NewInputParameterError("unknown parent object id spec")
			}

		default:
			return input, httperrors.NewInputParameterError("got unknown type %q, expect %s",
				input.Type, api.NetworkAddressTypes)
		}
	default:
		return input, httperrors.NewInputParameterError("got unknown parent type %q, expect %s",
			input.ParentType, api.NetworkAddressParentTypes)
	}

	if net == nil {
		panic("fatal error: no network")
	}
	{
		tryReserved := false
		if input.IPAddr != "" {
			tryReserved = true
		}
		ipAddr, err := net.GetFreeIPWithLock(ctx, userCred, nil, nil, input.IPAddr, "", tryReserved)
		if err != nil {
			return input, httperrors.NewInputParameterError("allocate ip addr: %v", err)
		}
		input.IPAddr = ipAddr
	}

	return input, nil
}

func (na *SNetworkAddress) parentIdInt64() int64 {
	r, err := strconv.ParseInt(na.ParentId, 10, 64)
	if err != nil {
		panic(fmt.Sprintf("networkaddress %s: invalid parent_id %s", na.Id, na.ParentId))
	}
	return r
}

func (na *SNetworkAddress) getGuestnetwork(ctx context.Context, userCred mcclient.TokenCredential) (*SGuestnetwork, error) {
	if na.ParentType == api.NetworkAddressParentTypeGuestnetwork {
		guestnetwork, err := GuestnetworkManager.fetchByRowId(ctx, userCred, na.parentIdInt64())
		if err != nil {
			return nil, err
		}
		return guestnetwork, nil
	}
	return nil, errors.Errorf("getGuestnetwork: expect parent_type %s, got %s", api.NetworkAddressParentTypeGuestnetwork, na.ParentType)
}

func (na *SNetworkAddress) getGuest(ctx context.Context, userCred mcclient.TokenCredential) (*SGuest, error) {
	guestnetwork, err := na.getGuestnetwork(ctx, userCred)
	if err != nil {
		return nil, err
	}
	guest := guestnetwork.GetGuest()
	if guest != nil {
		return guest, nil
	}
	return nil, errors.Wrapf(errors.ErrNotFound, "getGuest: guestnetwork guest_id %s", guestnetwork.GuestId)
}

func (na *SNetworkAddress) getIVM(ctx context.Context, userCred mcclient.TokenCredential) (cloudprovider.ICloudVM, error) {
	guest, err := na.getGuest(ctx, userCred)
	if err != nil {
		return nil, err
	}
	if guest.ExternalId == "" {
		return nil, errors.Errorf("getIVM: guest %s(%s) has empty external_id", guest.Name, guest.Id)
	}
	ivm, err := guest.GetIVM()
	if err != nil {
		return nil, err
	}
	return ivm, nil
}

func (na *SNetworkAddress) getICloudNic(ctx context.Context, userCred mcclient.TokenCredential) (cloudprovider.ICloudNic, error) {
	guestnetwork, err := na.getGuestnetwork(ctx, userCred)
	if err != nil {
		return nil, err
	}

	iVM, err := na.getIVM(ctx, userCred)
	if err != nil {
		return nil, err
	}
	iNics, err := iVM.GetINics()
	if err != nil {
		return nil, err
	}
	for _, iNic := range iNics {
		if iNic.GetIP() == guestnetwork.IpAddr && iNic.GetMAC() == guestnetwork.MacAddr {
			return iNic, nil
		}
	}
	return nil, errors.Wrapf(errors.ErrNotFound, "getICloudNic: no cloud nic with ip %s, mac %s", guestnetwork.IpAddr, guestnetwork.MacAddr)
}

func (na *SNetworkAddress) remoteAssignAddress(ctx context.Context, userCred mcclient.TokenCredential) error {
	if na.ParentType == api.NetworkAddressParentTypeGuestnetwork {
		guest, err := na.getGuest(ctx, userCred)
		if err != nil {
			return err
		}
		if guest.ExternalId != "" {
			iNic, err := na.getICloudNic(ctx, userCred)
			if err != nil {
				return err
			}
			if err := iNic.AssignAddress([]string{na.IpAddr}); err != nil {
				if errors.Cause(err) == cloudprovider.ErrAddressCountExceed {
					return httperrors.NewNotAcceptableError("exceed address count limit: %v", err)
				}
				return err
			}
		} else {
			NetworkAddressManager.submitGuestSyncTask(ctx, userCred, guest)
		}
	}
	return nil
}

func (na *SNetworkAddress) remoteUnassignAddress(ctx context.Context, userCred mcclient.TokenCredential) error {
	if na.ParentType == api.NetworkAddressParentTypeGuestnetwork {
		guest, err := na.getGuest(ctx, userCred)
		if err != nil {
			if errors.Cause(err) == errors.ErrNotFound {
				return nil
			}
			return err
		}
		if guest.ExternalId != "" {
			iNic, err := na.getICloudNic(ctx, userCred)
			if err != nil {
				if errors.Cause(err) == errors.ErrNotFound {
					return nil
				}
				return err
			}
			if err := iNic.UnassignAddress([]string{na.IpAddr}); err != nil {
				return err
			}
		} else {
			NetworkAddressManager.submitGuestSyncTask(ctx, userCred, guest)
		}
	}
	return nil
}

func (na *SNetworkAddress) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
	if err := na.remoteAssignAddress(ctx, userCred); err != nil {
		return err
	}
	return nil
}

func (na *SNetworkAddress) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
	if err := na.remoteUnassignAddress(ctx, userCred); err != nil {
		return err
	}
	return nil
}

func (man *SNetworkAddressManager) ListItemExportKeys(ctx context.Context,
	q *sqlchemy.SQuery,
	userCred mcclient.TokenCredential,
	keys stringutils2.SSortedStrings,
) (*sqlchemy.SQuery, error) {
	return db.ApplyListItemExportKeys(ctx, q, userCred, keys,
		&man.SStandaloneAnonResourceBaseManager,
		&man.SNetworkResourceBaseManager,
	)
}

func (man *SNetworkAddressManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
	return db.ApplyQueryDistinctExtraField(q, field,
		&man.SStandaloneAnonResourceBaseManager,
		&man.SNetworkResourceBaseManager,
	)
}

func (man *SNetworkAddressManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.NetworkAddressListInput) (*sqlchemy.SQuery, error) {
	var err error

	q, err = man.SStandaloneAnonResourceBaseManager.ListItemFilter(ctx, q, userCred, input.StandaloneAnonResourceListInput)
	if err != nil {
		return nil, err
	}

	q, err = man.SNetworkResourceBaseManager.ListItemFilter(ctx, q, userCred, input.NetworkFilterListInput)
	if err != nil {
		return nil, err
	}

	if len(input.GuestId) > 0 {
		gnq := GuestnetworkManager.Query().SubQuery()
		gq := GuestManager.Query().
			In("id", input.GuestId).
			SubQuery()
		idq := man.Query("id").
			Equals("parent_type", api.NetworkAddressParentTypeGuestnetwork)
		idq = idq.Join(gnq, sqlchemy.Equals(gnq.Field("row_id"), idq.Field("parent_id")))
		idq = idq.Join(gq, sqlchemy.Equals(gq.Field("id"), gnq.Field("guest_id")))
		q = q.In("id", idq.SubQuery())
	}
	return q, nil
}

func (man *SNetworkAddressManager) OrderByExtraFields(
	ctx context.Context,
	q *sqlchemy.SQuery,
	userCred mcclient.TokenCredential,
	query api.NetworkAddressListInput,
) (retq *sqlchemy.SQuery, err error) {
	retq, err = man.SNetworkResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.NetworkFilterListInput)
	if err != nil {
		return nil, err
	}
	retq, err = man.SStandaloneAnonResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneAnonResourceListInput)
	return retq, nil
}

func (man *SNetworkAddressManager) FetchCustomizeColumns(
	ctx context.Context,
	userCred mcclient.TokenCredential,
	query jsonutils.JSONObject,
	objs []interface{},
	fields stringutils2.SSortedStrings,
	isList bool,
) []api.NetworkAddressDetails {
	ret := make([]api.NetworkAddressDetails, len(objs))
	netCols := man.SNetworkResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
	for i := range netCols {
		ret[i].NetworkResourceInfo = netCols[i]
	}
	stdaCols := man.SStandaloneAnonResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
	for i := range stdaCols {
		ret[i].StandaloneAnonResourceDetails = stdaCols[i]
	}

	{
		guestnetworkRowIds := make([]int64, 0, len(objs))
		for i := range objs {
			na := objs[i].(*SNetworkAddress)
			if na.ParentType == api.NetworkAddressParentTypeGuestnetwork {
				rowId, _ := strconv.ParseInt(na.ParentId, 10, 64)
				guestnetworkRowIds = append(guestnetworkRowIds, rowId)
			}
		}
		if len(guestnetworkRowIds) > 0 {
			gns, err := GuestnetworkManager.fetchByRowIds(ctx, userCred, guestnetworkRowIds)
			if err == nil && len(gns) > 0 {
				gnObjs := make([]interface{}, len(gns))
				for i := range gns {
					gnObjs[i] = &gns[i]
				}
				gnds := GuestnetworkManager.FetchCustomizeColumns(ctx, userCred, query, gnObjs, fields, isList)
				for i, j := 0, 0; i < len(objs); i++ {
					na := objs[i].(*SNetworkAddress)
					if na.ParentType == api.NetworkAddressParentTypeGuestnetwork {
						ret[i].Guestnetwork = gnds[j]
						j++
					}
				}
			}
		}
	}
	return ret
}

func (man *SNetworkAddressManager) FilterByOwner(q *sqlchemy.SQuery, owner mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
	q = db.ApplyFilterByOwner(q, owner, scope,
		&man.SStandaloneAnonResourceBaseManager,
	)
	if owner != nil {
		var condVar, condVal string
		switch scope {
		case rbacutils.ScopeProject:
			condVar, condVal = "tenant_id", owner.GetProjectId()
		case rbacutils.ScopeDomain:
			condVar, condVal = "domain_id", owner.GetProjectDomainId()
		default:
			return q
		}
		{
			var (
				gnq = GuestnetworkManager.Query().SubQuery()
				gq  = GuestManager.Query().SubQuery()
				naq = NetworkAddressManager.Query("id")
			)
			naq = naq.
				Join(gnq, sqlchemy.Equals(naq.Field("parent_id"), gnq.Field("row_id"))).
				Join(gq, sqlchemy.Equals(gnq.Field("guest_id"), gq.Field("id")))
			naq = naq.Equals("parent_type", api.NetworkAddressParentTypeGuestnetwork)
			naq = naq.Filter(sqlchemy.Equals(gq.Field(condVar), condVal))
			q = q.In("id", naq.SubQuery())
		}
	}
	return q
}

func (man *SNetworkAddressManager) submitGuestSyncTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) {
	man.delayedWorkManager.Submit(ctx, delayedwork.DelayedWorkRequest{
		ID:        guest.Id,
		SoftDelay: 2 * time.Second,
		HardDelay: 5 * time.Second,
		Func: func(ctx context.Context) {
			var (
				fwOnly       = false
				parentTaskId = ""
			)
			err := guest.StartSyncTask(ctx, userCred, fwOnly, parentTaskId)
			if err != nil {
				log.Errorf("guest StartSyncTask: %v", err)
			}
		},
	})
}


<|start_filename|>pkg/util/panicutils/panicutil.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package panicutils

import (
	"context"
	"runtime/debug"

	"yunion.io/x/jsonutils"

	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
	npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
)

const PANIC_ERROR = "PANIC_ERROR"

func SendPanicMessage(ctx context.Context, err error) {
	iNotes := jsonutils.Marshal(map[string]string{"stack": string(debug.Stack()), "error": err.Error()})
	notifyclient.SystemNotifyWithCtx(ctx, npk.NotifyPriorityCritical, PANIC_ERROR, iNotes)
}


<|start_filename|>pkg/compute/tasks/server_sku_create_task.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
	"context"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type ServerSkuCreateTask struct {
	taskman.STask
}

func init() {
	taskman.RegisterTask(ServerSkuCreateTask{})
}

func (self *ServerSkuCreateTask) taskFail(ctx context.Context, sku *models.SServerSku, err error) {
	sku.SetStatus(self.UserCred, api.SkuStatusCreatFailed, err.Error())
	db.OpsLog.LogEvent(sku, db.ACT_ALLOCATE, err, self.GetUserCred())
	logclient.AddActionLogWithStartable(self, sku, logclient.ACT_ALLOCATE, err, self.UserCred, false)
	self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}

func (self *ServerSkuCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	sku := obj.(*models.SServerSku)
	region, err := sku.GetRegion()
	if err != nil {
		self.taskFail(ctx, sku, errors.Wrapf(err, "GetRegion"))
		return
	}

	provider, err := region.GetCloudprovider()
	if err != nil {
		self.taskFail(ctx, sku, errors.Wrapf(err, "GetCloudprovider"))
		return
	}

	driver, err := provider.GetProvider()
	if err != nil {
		self.taskFail(ctx, sku, errors.Wrapf(err, "GetDriver"))
		return
	}

	iRegion, err := driver.GetIRegionById(region.ExternalId)
	if err != nil {
		self.taskFail(ctx, sku, errors.Wrapf(err, "GetIRegionById"))
		return
	}

	opts := cloudprovider.SServerSkuCreateOption{
		Name:             sku.Name,
		CpuCount:         sku.CpuCoreCount,
		VmemSizeMb:       sku.MemorySizeMB,
		SysDiskMinSizeGb: sku.SysDiskMinSizeGB,
		SysDiskMaxSizeGb: sku.SysDiskMaxSizeGB,
	}

	iSku, err := iRegion.CreateISku(&opts)
	if err != nil {
		self.taskFail(ctx, sku, errors.Wrapf(err, "CreateISku"))
		return
	}

	sku.SyncWithPrivateCloudSku(ctx, self.GetUserCred(), iSku)
	self.SetStageComplete(ctx, nil)
}


<|start_filename|>cmd/climc/shell/compute/cachedimages.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compute

import (
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/mcclient/modules"
	"yunion.io/x/onecloud/pkg/mcclient/options"
)

func init() {
	type CachedImageListOptions struct {
		options.BaseListOptions
		ImageType string `help:"image type" choices:"system|customized|shared|market"`

		Region string `help:"show images cached at cloud region"`
		Zone   string `help:"show images cached at zone"`

		HostSchedtagId string `help:"filter cached image with host schedtag"`
		Valid          *bool  `help:"valid cachedimage"`
	}
	R(&CachedImageListOptions{}, "cached-image-list", "List cached images", func(s *mcclient.ClientSession, args *CachedImageListOptions) error {
		params, err := options.ListStructToParams(args)
		if err != nil {
			return err
		}
		result, err := modules.Cachedimages.List(s, params)
		if err != nil {
			return err
		}
		printList(result, modules.Cachedimages.GetColumns(s))
		return nil
	})

	type CachedImageShowOptions struct {
		ID string `help:"ID or Name of the cached image to show"`
	}
	R(&CachedImageShowOptions{}, "cached-image-show", "Show cached image details", func(s *mcclient.ClientSession, args *CachedImageShowOptions) error {
		result, err := modules.Cachedimages.Get(s, args.ID, nil)
		if err != nil {
			return err
		}
		printObject(result)
		return nil
	})

	R(&CachedImageShowOptions{}, "cached-image-refresh", "Refresh cached image details", func(s *mcclient.ClientSession, args *CachedImageShowOptions) error {
		result, err := modules.Cachedimages.PerformAction(s, args.ID, "refresh", nil)
		if err != nil {
			return err
		}
		printObject(result)
		return nil
	})

	type CachedImageDeleteOptions struct {
		ID []string `help:"ID or Name of the cached image to show"`
	}
	R(&CachedImageDeleteOptions{}, "cached-image-delete", "Remove cached image information", func(s *mcclient.ClientSession, args *CachedImageDeleteOptions) error {
		results := modules.Cachedimages.BatchDelete(s, args.ID, nil)
		printBatchResults(results, modules.Cachedimages.GetColumns(s))
		return nil
	})
}


<|start_filename|>pkg/multicloud/esxi/shell/virtualmachine.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shell

import (
	"context"
	"fmt"

	"yunion.io/x/pkg/errors"

	"yunion.io/x/onecloud/pkg/multicloud/esxi"
	"yunion.io/x/onecloud/pkg/util/printutils"
	"yunion.io/x/onecloud/pkg/util/shellutils"
)

func init() {
	type VirtualMachineListOptions struct {
		Datacenter string `help:"Datacenter"`
		HostIP     string `help:"HostIP"`
		FakeRegex  string `help:"regex for fake template' name"`
		Datastore  string `help:"Datastore"`
		Template   bool   `help:"Whether it is tempalte virtual machine, default:false"`
	}
	shellutils.R(&VirtualMachineListOptions{}, "vm-list", "List vms of a host", func(cli *esxi.SESXiClient, args *VirtualMachineListOptions) error {
		var (
			vms []*esxi.SVirtualMachine
			err error
		)
		switch {
		case len(args.HostIP) > 0:
			host, err := cli.FindHostByIp(args.HostIP)
			if err != nil {
				return err
			}
			if args.Template {
				vms, err := host.GetTemplateVMs()
				if err != nil {
					return err
				}
				printList(vms, []string{})
				return nil
			}
			vms, err := host.GetIVMs2()
			if err != nil {
				return err
			}
			printList(vms, []string{})
			return nil
		case len(args.Datacenter) > 0:
			var fetcher esxi.VMFetcher
			if len(args.Datastore) > 0 {
				ds, err := getDatastore(cli, args.Datacenter, args.Datastore)
				if err != nil {
					return err
				}
				fetcher = ds
			} else {
				dc, err := cli.FindDatacenterByMoId(args.Datacenter)
				if err != nil {
					return errors.Wrap(err, "FindDatacenterByMoId")
				}
				fetcher = dc
			}
			if args.Template {
				if len(args.FakeRegex) > 0 {
					fmt.Printf("regex: %s\n", args.FakeRegex)
					vms, err = fetcher.FetchFakeTempateVMs(args.FakeRegex)
					if err != nil {
						return errors.Wrap(err, "FetchFakeTempateVMs")
					}
				} else {
					vms, err = fetcher.FetchTemplateVMs()
					if err != nil {
						return errors.Wrap(err, "FetchTemplateVMs")
					}
				}
			} else {
				vms, err = fetcher.FetchNoTemplateVMs()
				if err != nil {
					return errors.Wrap(err, "FetchNoTemplateVMs")
				}
			}
		default:
			return fmt.Errorf("Both Datacenter and HostIP cannot be empty")
		}
		printList(vms, []string{})
		return nil
	})

	type VirtualMachineCloneOptions struct {
		HOSTIP     string `help:"Host IP"`
		TEMPLATEID string `help:"id of template ma"`
		NAME       string `help:"New VM's name'"`
		Uuid       string `help:"Uuid of new VM"`
		CpuNum     int    `help:"Number of CPU"`
		MemSize    int    `help:"Size of Memory(MB)"`
	}
	shellutils.R(&VirtualMachineCloneOptions{}, "vm-clone", "Clone vm", func(cli *esxi.SESXiClient,
		args *VirtualMachineCloneOptions) error {
		host, err := cli.FindHostByIp(args.HOSTIP)
		if err != nil {
			return err
		}
		idss, err := host.GetDataStores()
		if err != nil {
			return err
		}
		if len(idss) == 0 {
			return fmt.Errorf("no datastore")
		}
		temVm, err := host.GetTemplateVMById(args.TEMPLATEID)
		if err != nil {
			return err
		}
		createParams := esxi.SCreateVMParam{
			Name: args.NAME,
			Uuid: args.Uuid,
			Cpu:  args.CpuNum,
			Mem:  args.MemSize,
		}
		vm, err := host.CloneVM(context.Background(), temVm, nil, idss[0].(*esxi.SDatastore), createParams)
		if err != nil {
			return errors.Wrap(err, "SHost.CloneVMFromTemplate")
		}
		printObject(vm)
		return nil
	})

	type VirtualMachineShowOptions struct {
		Datacenter string `help:"Datacenter"`
		HostIP     string `help:"Host IP"`
		VMID       string `help:"VM ID"`
	}
	getVM := func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) (*esxi.SVirtualMachine, error) {
		var vm *esxi.SVirtualMachine
		switch {
		case len(args.HostIP) > 0:
			host, err := cli.FindHostByIp(args.HostIP)
			if err != nil {
				return nil, errors.Wrap(err, "FindHostByIp")
			}
			ivm, err := host.GetIVMById(args.VMID)
			if err != nil && errors.Cause(err) != errors.ErrNotFound {
				return nil, err
			}
			if err != nil {
				vm, err = host.GetTemplateVMById(args.VMID)
				if err != nil {
					return nil, errors.Wrap(err, "GetTemplateVMById")
				}
			}
			vm = ivm.(*esxi.SVirtualMachine)
		case len(args.Datacenter) > 0:
			dc, err := cli.FindDatacenterByMoId(args.Datacenter)
			if err != nil {
				return nil, errors.Wrap(err, "FindDatacenterByMoId")
			}
			vm, err = dc.FetchVMById(args.VMID)
			if err != nil {
				return nil, errors.Wrap(err, "FetchVMById")
			}
		default:
			return nil, fmt.Errorf("Both Datacenter and HostIP cannot be empty")
		}
		return vm, nil
	}
	shellutils.R(&VirtualMachineShowOptions{}, "vm-show", "Show vm details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
		vm, err := getVM(cli, args)
		if err != nil {
			return err
		}
		printObject(vm)
		return nil
	})

	shellutils.R(&VirtualMachineShowOptions{}, "vm-nics", "Show vm nics details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
		vm, err := getVM(cli, args)
		if err != nil {
			return err
		}
		vmnics, err := vm.GetINics()
		if err != nil {
			return err
		}
		printList(vmnics, []string{})
		return nil
	})

	shellutils.R(&VirtualMachineShowOptions{}, "vm-snapshots", "Show vm snapshots details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
		vm, err := getVM(cli, args)
		if err != nil {
			return err
		}
		vmsps, err := vm.GetInstanceSnapshots()
		if err != nil {
			return err
		}
		printList(vmsps, []string{})
		return nil
	})

	type VirtualMachineSnapshotCreateOptions struct {
		VirtualMachineShowOptions
		NAME string `help:"Name of snapshot"`
		Desc string `help:"Description of snapshot"`
	}
	shellutils.R(&VirtualMachineSnapshotCreateOptions{}, "vm-snapshot-create", "Create vm snapshot", func(cli *esxi.SESXiClient, args *VirtualMachineSnapshotCreateOptions) error {
		vm, err := getVM(cli, &args.VirtualMachineShowOptions)
		if err != nil {
			return err
		}
		sp, err := vm.CreateInstanceSnapshot(context.Background(), args.NAME, args.Desc)
		if err != nil {
			return err
		}
		printObject(sp)
		return nil
	})

	type VirtualMachineSnapshotOptions struct {
		VirtualMachineShowOptions
		ID string `help:"ID of snapshot"`
	}
	shellutils.R(&VirtualMachineSnapshotOptions{}, "vm-snapshot-delete", "Delete vm snapshot", func(cli *esxi.SESXiClient, args *VirtualMachineSnapshotOptions) error {
		vm, err := getVM(cli, &args.VirtualMachineShowOptions)
		if err != nil {
			return err
		}
		sp, err := vm.GetInstanceSnapshot(args.ID)
		if err != nil {
			return err
		}
		err = sp.Delete()
		if err != nil {
			return err
		}
		return nil
	})
	shellutils.R(&VirtualMachineSnapshotOptions{}, "vm-snapshot-reset", "Reset vm to snapshot", func(cli *esxi.SESXiClient, args *VirtualMachineSnapshotOptions) error {
		vm, err := getVM(cli, &args.VirtualMachineShowOptions)
		if err != nil {
			return err
		}
		err = vm.ResetToInstanceSnapshot(context.Background(), args.ID)
		if err != nil {
			return err
		}
		return nil
	})

	shellutils.R(&VirtualMachineShowOptions{}, "vm-disks", "Show vm disks details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
		vm, err := getVM(cli, args)
		if err != nil {
			return err
		}
		vmdisks, err := vm.GetIDisks()
		if err != nil {
			return err
		}
		printList(vmdisks, []string{})
		return nil
	})

	type VirtualMachineDiskResizeOptions struct {
		VirtualMachineShowOptions
		DISKIDX int   `help:"disk index"`
		SIZEGB  int64 `help:"new size of disk"`
	}
	shellutils.R(&VirtualMachineDiskResizeOptions{}, "vm-disk-resize", "Resize a vm disk", func(cli *esxi.SESXiClient, args *VirtualMachineDiskResizeOptions) error {
		vm, err := getVM(cli, &args.VirtualMachineShowOptions)
		if err != nil {
			return err
		}
		vmdisks, err := vm.GetIDisks()
		if err != nil {
			return err
		}
		if args.DISKIDX < 0 || args.DISKIDX >= len(vmdisks) {
			return fmt.Errorf("Out of index: %d", args.DISKIDX)
		}
		disk := vmdisks[args.DISKIDX]
		ctx := context.Background()
		return disk.Resize(ctx, args.SIZEGB*1024)
	})

	shellutils.R(&VirtualMachineShowOptions{}, "vm-vnc", "Show vm VNC details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
		vm, err := getVM(cli, args)
		if err != nil {
			return err
		}
		info, err := vm.GetVNCInfo()
		if err != nil {
			return err
		}
		printutils.PrintJSONObject(info)
		return nil
	})

	shellutils.R(&VirtualMachineShowOptions{}, "vm-vmrc", "Show vm VMRC connection", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
		vm, err := getVM(cli, args)
		if err != nil {
			return err
		}
		info, err := vm.GetVmrcInfo()
		if err != nil {
			return err
		}
		printutils.PrintJSONObject(info)
		return nil
	})

	shellutils.R(&VirtualMachineShowOptions{}, "vm-webmks", "Show vm webmks connection", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
		vm, err := getVM(cli, args)
		if err != nil {
			return err
		}
		info, err := vm.GetWebmksInfo()
		if err != nil {
			return err
		}
		printutils.PrintJSONObject(info)
		return nil
	})

	shellutils.R(&VirtualMachineShowOptions{}, "vm-file-status", "Show vm files details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
		vm, err := getVM(cli, args)
		if err != nil {
			return err
		}
		err = vm.CheckFileInfo(context.Background())
		if err != nil {
			return err
		}
		return nil
	})

}


<|start_filename|>pkg/cloudcommon/db/tablespec.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package db

import (
	"context"
	"reflect"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/sqlchemy"

	"yunion.io/x/onecloud/pkg/cloudcommon/informer"
	"yunion.io/x/onecloud/pkg/util/nopanic"
	"yunion.io/x/onecloud/pkg/util/splitable"
)

type ITableSpec interface {
	Name() string
	Columns() []sqlchemy.IColumnSpec
	PrimaryColumns() []sqlchemy.IColumnSpec
	DataType() reflect.Type
	// CreateSQL() string
	Instance() *sqlchemy.STable
	ColumnSpec(name string) sqlchemy.IColumnSpec
	Insert(ctx context.Context, dt interface{}) error
	InsertOrUpdate(ctx context.Context, dt interface{}) error
	Update(ctx context.Context, dt interface{}, doUpdate func() error) (sqlchemy.UpdateDiffs, error)
	Fetch(dt interface{}) error
	// FetchAll(dest interface{}) error
	SyncSQL() []string
	DropForeignKeySQL() []string
	AddIndex(unique bool, cols ...string) bool
	Increment(ctx context.Context, diff interface{}, target interface{}) error
	Decrement(ctx context.Context, diff interface{}, target interface{}) error

	GetSplitTable() *splitable.SSplitTableSpec
}

type sTableSpec struct {
	sqlchemy.ITableSpec
}

func newTableSpec(model interface{}, tableName string, indexField string, dateField string, maxDuration time.Duration, maxSegments int) ITableSpec {
	var itbl sqlchemy.ITableSpec
	if len(indexField) > 0 && len(dateField) > 0 {
		var err error
		itbl, err = splitable.NewSplitTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments)
		if err != nil {
			log.Errorf("NewSplitTableSpec %s %s", tableName, err)
			return nil
		} else {
			log.Debugf("table %s maxDuration %d hour maxSegements %d", tableName, maxDuration/time.Hour, maxSegments)
		}
	} else {
		itbl = sqlchemy.NewTableSpecFromStruct(model, tableName)
	}
	return &sTableSpec{
		ITableSpec: itbl,
	}
}

func (ts *sTableSpec) GetSplitTable() *splitable.SSplitTableSpec {
	sts, ok := ts.ITableSpec.(*splitable.SSplitTableSpec)
	if ok {
		return sts
	}
	return nil
}

func (ts *sTableSpec) newInformerModel(dt interface{}) (*informer.ModelObject, error) {
	obj, ok := dt.(IModel)
	if !ok {
		return nil, errors.Errorf("informer model is not IModel")
	}
	if obj.GetVirtualObject() == nil {
		return nil, errors.Errorf("object %#v virtual object is nil", obj)
	}
	if obj.GetModelManager() == nil {
		return nil, errors.Errorf("object %#v model manager is nil", obj)
	}
	jointObj, isJoint := obj.(IJointModel)
	if isJoint {
		mObj := JointMaster(jointObj)
		sObj := JointSlave(jointObj)
		return informer.NewJointModel(jointObj, jointObj.KeywordPlural(), mObj.GetId(), sObj.GetId()), nil
	}
	return informer.NewModel(obj, obj.KeywordPlural(), obj.GetId()), nil
}

func (ts *sTableSpec) isMarkDeleted(dt interface{}) (bool, error) {
	if vObj, ok := dt.(IVirtualModel); ok {
		if vObj.GetPendingDeleted() {
			return true, nil
		}
	}
	obj, ok := dt.(IModel)
	if !ok {
		return false, errors.Errorf("informer model is not IModel")
	}
	return obj.GetDeleted(), nil
}

func (ts *sTableSpec) Insert(ctx context.Context, dt interface{}) error {
	if err := ts.ITableSpec.Insert(dt); err != nil {
		return err
	}
	ts.inform(ctx, dt, informer.Create)
	return nil
}

func (ts *sTableSpec) InsertOrUpdate(ctx context.Context, dt interface{}) error {
	if err := ts.ITableSpec.InsertOrUpdate(dt); err != nil {
		return err
	}
	ts.inform(ctx, dt, informer.Create)
	return nil
}

func (ts *sTableSpec) Update(ctx context.Context, dt interface{}, doUpdate func() error) (sqlchemy.UpdateDiffs, error) {
	oldObj := jsonutils.Marshal(dt)
	diffs, err := ts.ITableSpec.Update(dt, doUpdate)
	if err != nil {
		return nil, err
	}
	if diffs == nil {
		// no data to update
		return nil, nil
	}
	isDeleted, err := ts.isMarkDeleted(dt)
	if err != nil {
		return nil, errors.Wrap(err, "check is mark deleted")
	}
	if isDeleted {
		ts.inform(ctx, dt, informer.Delete)
	} else {
		ts.informUpdate(ctx, dt, oldObj.(*jsonutils.JSONDict))
	}
	return diffs, nil
}

func (ts *sTableSpec) Increment(ctx context.Context, diff, target interface{}) error {
	oldObj := jsonutils.Marshal(target)
	err := ts.ITableSpec.Increment(diff, target)
	if err != nil {
		return errors.Wrap(err, "Increment")
	}
	ts.informUpdate(ctx, target, oldObj.(*jsonutils.JSONDict))
	return nil
}

func (ts *sTableSpec) Decrement(ctx context.Context, diff, target interface{}) error {
	oldObj := jsonutils.Marshal(target)
	err := ts.ITableSpec.Decrement(diff, target)
	if err != nil {
		return err
	}
	ts.informUpdate(ctx, target, oldObj.(*jsonutils.JSONDict))
	return nil
}

func (ts *sTableSpec) inform(ctx context.Context, dt interface{}, f func(ctx context.Context, obj *informer.ModelObject) error) {
	if !informer.IsInit() {
		return
	}
	nf := func() {
		obj, err := ts.newInformerModel(dt)
		if err != nil {
			log.Warningf("newInformerModel error: %v", err)
			return
		}
		if err := f(ctx, obj); err != nil {
			if errors.Cause(err) == informer.ErrBackendNotInit {
				log.V(4).Warningf("informer backend not init")
			} else {
				log.Errorf("call informer func error: %v", err)
			}
		}
	}
	nopanic.Run(nf)
}

func (ts *sTableSpec) informUpdate(ctx context.Context, dt interface{}, oldObj *jsonutils.JSONDict) {
	if !informer.IsInit() {
		return
	}
	nf := func() {
		obj, err := ts.newInformerModel(dt)
		if err != nil {
			log.Warningf("newInformerModel error: %v", err)
			return
		}
		if err := informer.Update(ctx, obj, oldObj); err != nil {
			if errors.Cause(err) == informer.ErrBackendNotInit {
				log.V(4).Warningf("informer backend not init")
			} else {
				log.Errorf("call informer update func error: %v", err)
			}
		}
	}
	nopanic.Run(nf)
}


<|start_filename|>vendor/github.com/tencentyun/cos-go-sdk-v5/batch.go<|end_filename|>
package cos

import (
	"context"
	"encoding/xml"
	"fmt"
	"net/http"
)

type BatchService service

type BatchRequestHeaders struct {
	XCosAppid     int          `header:"x-cos-appid" xml:"-" url:"-"`
	ContentLength string       `header:"Content-Length,omitempty" xml:"-" url:"-"`
	ContentType   string       `header:"Content-Type,omitempty" xml:"-" url:"-"`
	Headers       *http.Header `header:"-" xml:"-", url:"-"`
}

// BatchProgressSummary
type BatchProgressSummary struct {
	NumberOfTasksFailed    int `xml:"NumberOfTasksFailed" header:"-" url:"-"`
	NumberOfTasksSucceeded int `xml:"NumberOfTasksSucceeded" header:"-" url:"-"`
	TotalNumberOfTasks     int `xml:"TotalNumberOfTasks" header:"-" url:"-"`
}

// BatchJobReport
type BatchJobReport struct {
	Bucket      string `xml:"Bucket" header:"-" url:"-"`
	Enabled     string `xml:"Enabled" header:"-" url:"-"`
	Format      string `xml:"Format" header:"-" url:"-"`
	Prefix      string `xml:"Prefix,omitempty" header:"-" url:"-"`
	ReportScope string `xml:"ReportScope" header:"-" url:"-"`
}

// BatchJobOperationCopy
type BatchMetadata struct {
	Key   string `xml:"Key" header:"-" url:"-"`
	Value string `xml:"Value" header:"-" url:"-"`
}
type BatchNewObjectMetadata struct {
	CacheControl       string          `xml:"CacheControl,omitempty" header:"-" url:"-"`
	ContentDisposition string          `xml:"ContentDisposition,omitempty" header:"-" url:"-"`
	ContentEncoding    string          `xml:"ContentEncoding,omitempty" header:"-" url:"-"`
	ContentType        string          `xml:"ContentType,omitempty" header:"-" url:"-"`
	HttpExpiresDate    string          `xml:"HttpExpiresDate,omitempty" header:"-" url:"-"`
	SSEAlgorithm       string          `xml:"SSEAlgorithm,omitempty" header:"-" url:"-"`
	UserMetadata       []BatchMetadata `xml:"UserMetadata>member,omitempty" header:"-" url:"-"`
}
type BatchGrantee struct {
	DisplayName    string `xml:"DisplayName,omitempty" header:"-" url:"-"`
	Identifier     string `xml:"Identifier" header:"-" url:"-"`
	TypeIdentifier string `xml:"TypeIdentifier" header:"-" url:"-"`
}
type BatchCOSGrant struct {
	Grantee    *BatchGrantee `xml:"Grantee" header:"-" url:"-"`
	Permission string        `xml:"Permission" header:"-" url:"-"`
}
type BatchAccessControlGrants struct {
	COSGrants *BatchCOSGrant `xml:"COSGrant,omitempty" header:"-" url:"-"`
}
type BatchJobOperationCopy struct {
	AccessControlGrants       *BatchAccessControlGrants `xml:"AccessControlGrants,omitempty" header:"-" url:"-"`
	CannedAccessControlList   string                    `xml:"CannedAccessControlList,omitempty" header:"-" url:"-"`
	MetadataDirective         string                    `xml:"MetadataDirective,omitempty" header:"-" url:"-"`
	ModifiedSinceConstraint   int64                     `xml:"ModifiedSinceConstraint,omitempty" header:"-" url:"-"`
	UnModifiedSinceConstraint int64                     `xml:"UnModifiedSinceConstraint,omitempty" header:"-" url:"-"`
	NewObjectMetadata         *BatchNewObjectMetadata   `xml:"NewObjectMetadata,omitempty" header:"-" url:"-"`
	StorageClass              string                    `xml:"StorageClass,omitempty" header:"-" url:"-"`
	TargetResource            string                    `xml:"TargetResource" header:"-" url:"-"`
}

// BatchInitiateRestoreObject
type BatchInitiateRestoreObject struct {
	ExpirationInDays int    `xml:"ExpirationInDays"`
	JobTier          string `xml:"JobTier"`
}

// BatchJobOperation
type BatchJobOperation struct {
	PutObjectCopy *BatchJobOperationCopy      `xml:"COSPutObjectCopy,omitempty" header:"-" url:"-"`
	RestoreObject *BatchInitiateRestoreObject `xml:"COSInitiateRestoreObject,omitempty" header:"-" url:"-"`
}

// BatchJobManifest
type BatchJobManifestLocation struct {
	ETag            string `xml:"ETag" header:"-" url:"-"`
	ObjectArn       string `xml:"ObjectArn" header:"-" url:"-"`
	ObjectVersionId string `xml:"ObjectVersionId,omitempty" header:"-" url:"-"`
}
type BatchJobManifestSpec struct {
	Fields []string `xml:"Fields>member,omitempty" header:"-" url:"-"`
	Format string   `xml:"Format" header:"-" url:"-"`
}
type BatchJobManifest struct {
	Location *BatchJobManifestLocation `xml:"Location" header:"-" url:"-"`
	Spec     *BatchJobManifestSpec     `xml:"Spec" header:"-" url:"-"`
}

type BatchCreateJobOptions struct {
	XMLName              xml.Name           `xml:"CreateJobRequest" header:"-" url:"-"`
	ClientRequestToken   string             `xml:"ClientRequestToken" header:"-" url:"-"`
	ConfirmationRequired string             `xml:"ConfirmationRequired,omitempty" header:"-" url:"-"`
	Description          string             `xml:"Description,omitempty" header:"-" url:"-"`
	Manifest             *BatchJobManifest  `xml:"Manifest" header:"-" url:"-"`
	Operation            *BatchJobOperation `xml:"Operation" header:"-" url:"-"`
	Priority             int                `xml:"Priority" header:"-" url:"-"`
	Report               *BatchJobReport    `xml:"Report" header:"-" url:"-"`
	RoleArn              string             `xml:"RoleArn" header:"-" url:"-"`
}

type BatchCreateJobResult struct {
	XMLName xml.Name `xml:"CreateJobResult"`
	JobId   string   `xml:"JobId,omitempty"`
}

func processETag(opt *BatchCreateJobOptions) *BatchCreateJobOptions {
	if opt != nil && opt.Manifest != nil && opt.Manifest.Location != nil {
		opt.Manifest.Location.ETag = "<ETag>" + opt.Manifest.Location.ETag + "</ETag>"
	}
	return opt
}

func (s *BatchService) CreateJob(ctx context.Context, opt *BatchCreateJobOptions, headers *BatchRequestHeaders) (*BatchCreateJobResult, *Response, error) {
	var res BatchCreateJobResult
	sendOpt := sendOptions{
		baseURL:   s.client.BaseURL.BatchURL,
		uri:       "/jobs",
		method:    http.MethodPost,
		optHeader: headers,
		body:      opt,
		result:    &res,
	}

	resp, err := s.client.send(ctx, &sendOpt)
	return &res, resp, err
}

type BatchJobFailureReasons struct {
	FailureCode   string `xml:"FailureCode" header:"-" url:"-"`
	FailureReason string `xml:"FailureReason" header:"-" url:"-"`
}

type BatchDescribeJob struct {
	ConfirmationRequired string                  `xml:"ConfirmationRequired,omitempty" header:"-" url:"-"`
	CreationTime         string                  `xml:"CreationTime,omitempty" header:"-" url:"-"`
	Description          string                  `xml:"Description,omitempty" header:"-" url:"-"`
	FailureReasons       *BatchJobFailureReasons `xml:"FailureReasons>JobFailure,omitempty" header:"-" url:"-"`
	JobId                string                  `xml:"JobId" header:"-" url:"-"`
	Manifest             *BatchJobManifest       `xml:"Manifest" header:"-" url:"-"`
	Operation            *BatchJobOperation      `xml:"Operation" header:"-" url:"-"`
	Priority             int                     `xml:"Priority" header:"-" url:"-"`
	ProgressSummary      *BatchProgressSummary   `xml:"ProgressSummary" header:"-" url:"-"`
	Report               *BatchJobReport         `xml:"Report,omitempty" header:"-" url:"-"`
	RoleArn              string                  `xml:"RoleArn,omitempty" header:"-" url:"-"`
	Status               string                  `xml:"Status,omitempty" header:"-" url:"-"`
	StatusUpdateReason   string                  `xml:"StatusUpdateReason,omitempty" header:"-" url:"-"`
	SuspendedCause       string                  `xml:"SuspendedCause,omitempty" header:"-" url:"-"`
	SuspendedDate        string                  `xml:"SuspendedDate,omitempty" header:"-" url:"-"`
	TerminationDate      string                  `xml:"TerminationDate,omitempty" header:"-" url:"-"`
}
type BatchDescribeJobResult struct {
	XMLName xml.Name          `xml:"DescribeJobResult"`
	Job     *BatchDescribeJob `xml:"Job,omitempty"`
}

func (s *BatchService) DescribeJob(ctx context.Context, id string, headers *BatchRequestHeaders) (*BatchDescribeJobResult, *Response, error) {
	var res BatchDescribeJobResult
	u := fmt.Sprintf("/jobs/%s", id)
	sendOpt := sendOptions{
		baseURL:   s.client.BaseURL.BatchURL,
		uri:       u,
		method:    http.MethodGet,
		optHeader: headers,
		result:    &res,
	}
	resp, err := s.client.send(ctx, &sendOpt)
	return &res, resp, err
}

type BatchListJobsOptions struct {
	JobStatuses string `url:"jobStatuses,omitempty" header:"-" xml:"-"`
	MaxResults  int    `url:"maxResults,omitempty" header:"-" xml:"-"`
	NextToken   string `url:"nextToken,omitempty" header:"-" xml:"-"`
}

type BatchListJobsMember struct {
	CreationTime    string                `xml:"CreationTime,omitempty" header:"-" url:"-"`
	Description     string                `xml:"Description,omitempty" header:"-" url:"-"`
	JobId           string                `xml:"JobId,omitempty" header:"-" url:"-"`
	Operation       string                `xml:"Operation,omitempty" header:"-" url:"-"`
	Priority        int                   `xml:"Priority,omitempty" header:"-" url:"-"`
	ProgressSummary *BatchProgressSummary `xml:"ProgressSummary,omitempty" header:"-" url:"-"`
	Status          string                `xml:"Status,omitempty" header:"-" url:"-"`
	TerminationDate string                `xml:"TerminationDate,omitempty" header:"-" url:"-"`
}
type BatchListJobs struct {
	Members []BatchListJobsMember `xml:"member,omitempty" header:"-" url:"-"`
}
type BatchListJobsResult struct {
	XMLName   xml.Name       `xml:"ListJobsResult"`
	Jobs      *BatchListJobs `xml:"Jobs,omitempty"`
	NextToken string         `xml:"NextToken,omitempty"`
}

func (s *BatchService) ListJobs(ctx context.Context, opt *BatchListJobsOptions, headers *BatchRequestHeaders) (*BatchListJobsResult, *Response, error) {
	var res BatchListJobsResult
	sendOpt := sendOptions{
		baseURL:   s.client.BaseURL.BatchURL,
		uri:       "/jobs",
		method:    http.MethodGet,
		optQuery:  opt,
		optHeader: headers,
		result:    &res,
	}
	resp, err := s.client.send(ctx, &sendOpt)
	return &res, resp, err
}

type BatchUpdatePriorityOptions struct {
	JobId    string `url:"-" header:"-" xml:"-"`
	Priority int    `url:"priority" header:"-" xml:"-"`
}
type BatchUpdatePriorityResult struct {
	XMLName  xml.Name `xml:"UpdateJobPriorityResult"`
	JobId    string   `xml:"JobId,omitempty"`
	Priority int      `xml:"Priority,omitempty"`
}

func (s *BatchService) UpdateJobPriority(ctx context.Context, opt *BatchUpdatePriorityOptions, headers *BatchRequestHeaders) (*BatchUpdatePriorityResult, *Response, error) {
	u := fmt.Sprintf("/jobs/%s/priority", opt.JobId)
	var res BatchUpdatePriorityResult
	sendOpt := sendOptions{
		baseURL:   s.client.BaseURL.BatchURL,
		uri:       u,
		method:    http.MethodPost,
		optQuery:  opt,
		optHeader: headers,
		result:    &res,
	}
	resp, err := s.client.send(ctx, &sendOpt)
	return &res, resp, err
}

type BatchUpdateStatusOptions struct {
	JobId              string `header:"-" url:"-" xml:"-"`
	RequestedJobStatus string `url:"requestedJobStatus" header:"-" xml:"-"`
	StatusUpdateReason string `url:"statusUpdateReason,omitempty" header:"-", xml:"-"`
}
type BatchUpdateStatusResult struct {
	XMLName            xml.Name `xml:"UpdateJobStatusResult"`
	JobId              string   `xml:"JobId,omitempty"`
	Status             string   `xml:"Status,omitempty"`
	StatusUpdateReason string   `xml:"StatusUpdateReason,omitempty"`
}

func (s *BatchService) UpdateJobStatus(ctx context.Context, opt *BatchUpdateStatusOptions, headers *BatchRequestHeaders) (*BatchUpdateStatusResult, *Response, error) {
	u := fmt.Sprintf("/jobs/%s/status", opt.JobId)
	var res BatchUpdateStatusResult
	sendOpt := sendOptions{
		baseURL:   s.client.BaseURL.BatchURL,
		uri:       u,
		method:    http.MethodPost,
		optQuery:  opt,
		optHeader: headers,
		result:    &res,
	}
	resp, err := s.client.send(ctx, &sendOpt)
	return &res, resp, err
}


<|start_filename|>cmd/climc/shell/compute/quotas.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compute

import (
	"yunion.io/x/jsonutils"
	"yunion.io/x/log"

	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
	"yunion.io/x/onecloud/pkg/mcclient/modules"
)

type ComputeQuotaKeys struct {
	ZoneQuotaKeys

	Hypervisor string `help:"hypervisor" choices:"kvm|baremetal"`
}

type ComputeQuotaOptions struct {
	ComputeQuotaKeys

	Count          int64 `help:"server count" json:"count,omitzero"`
	Cpu            int64 `help:"CPU count" json:"cpu,omitzero"`
	Memory         int64 `help:"Memory size in MB" json:"memory,omitzero"`
	Storage        int64 `help:"Storage size in MB" json:"storage,omitzero"`
	IsolatedDevice int64 `help:"Isolated device count" json:"isolated_device,omitzero"`
}

type RegionQuotaKeys struct {
	Provider  string `help:"cloud provider" json:"provider,omitempty"`
	Brand     string `help:"cloud brand" json:"brand,omitempty"`
	CloudEnv  string `help:"cloud environment" json:"cloud_env,omitempty" choices:"onpremise|private|public"`
	AccountId string `help:"cloud account id" json:"account_id,omitempty"`
	ManagerId string `help:"cloud provider id" json:"manager_id,omitempty"`
	RegionId  string `help:"region id" json:"region_id,omitempty"`
}

type RegionQuotaOptions struct {
	RegionQuotaKeys

	Port  int64 `help:"Internal NIC count" json:"port,omitzero"`
	Eport int64 `help:"External NIC count" json:"eport,omitzero"`
	// Bw    int64 `help:"Internal bandwidth in Mbps" json:"bw,omitzero"`
	// Ebw   int64 `help:"External bandwidth in Mbps" json:"ebw,omitzero"`

	Eip int64 `help:"Elastic IP count" json:"eip,omitzero"`

	Snapshot         int64 `help:"Snapshot count" json:"snapshot,omitzero"`
	InstanceSnapshot int64 `help:"Instance snapshot count" json:"instance_snapshot,omitzero"`

	Bucket    int64 `help:"bucket count" json:"bucket,omitzero"`
	ObjectGB  int64 `help:"object size in GB" json:"object_gb,omitzero"`
	ObjectCnt int64 `help:"object count" json:"object_cnt,omitzero"`

	Rds   int64 `help:"Rds count" json:"rds,omitzero"`
	Cache int64 `help:"redis count" json:"cache,omitzero"`

	Loadbalancer int64 `help:"loadbalancer instance count" json:"loadbalancer,omitzero"`
}

type ZoneQuotaKeys struct {
	RegionQuotaKeys

	ZoneId string `help:"zone id" json:"zone_id,omitempty"`
}

type ZoneQuotaOptions struct {
	ZoneQuotaKeys
}

type ProjectQuotaOptions struct {
	Secgroup int64 `help:"Secgroup count" json:"secgroup,omitzero"`
}

type ImageQuotaKeys struct {
	Type string `help:"image type, either iso or image" choices:"iso|image" json:"type,omitempty"`
}

type ImageQuotaOptions struct {
	ImageQuotaKeys

	Image int64 `help:"Template count" json:"image,omitzero"`
}

type DomainQuotaOptions struct {
	Cloudaccount int64 `help:"cloudaccount count" json:"cloudaccount,omitzero"`
}

type InfrasQuotaOptions struct {
	RegionQuotaKeys

	Host int64 `help:"host count" json:"host,omitzero"`
	Vpc  int64 `help:"vpc count" json:"vpc,omitzero"`
}

type IdentityQuotaOptions struct {
	User    int64 `help:"user count" json:"user,omitzero"`
	Group   int64 `help:"group count" json:"group,omitzero"`
	Project int64 `help:"project count" json:"project,omitzero"`
	Role    int64 `help:"role count" json:"role,omitzero"`
	Policy  int64 `help:"policy count" json:"policy,omitzero"`
}

type QuotaSetBaseOptions struct {
	Project string `help:"Tenant name or ID to set quota" json:"tenant,omitempty"`
	Domain  string `help:"Domain name or ID to set quota" json:"domain,omitempty"`
	Action  string `help:"quota set action" choices:"add|sub|reset|replace|delete|update"`
}

func printQuotaList(result jsonutils.JSONObject) {
	printList(modulebase.JSON2ListResult(result), nil)
}

func init() {
	type QuotaOptions struct {
		Scope   string `help:"scope" choices:"domain|project"`
		Project string `help:"Tenant name or ID" json:"tenant"`
		Domain  string `help:"Domain name or ID" json:"domain"`
		Refresh bool   `help:"refresh" json:"refresh,omitfalse"`
		Primary bool   `help:"get primary quota ONLY" json:"primary,omitfalse"`
	}
	R(&QuotaOptions{}, "quota", "Show quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error {
		params := jsonutils.Marshal(args)
		log.Debugf("%s", params)
		quotas, err := modules.Quotas.GetQuota(s, params)
		if err != nil {
			return err
		}
		printQuotaList(quotas)
		return nil
	})
	R(&QuotaOptions{}, "project-quota", "Show project-quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error {
		params := jsonutils.Marshal(args)
		quotas, err := modules.ProjectQuotas.GetQuota(s, params)
		if err != nil {
			return err
		}
		printQuotaList(quotas)
		return nil
	})
	R(&QuotaOptions{}, "region-quota", "Show region-quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error {
		params := jsonutils.Marshal(args)
		quotas, err := modules.RegionQuotas.GetQuota(s, params)
		if err != nil {
			return err
		}
		printQuotaList(quotas)
		return nil
	})
	R(&QuotaOptions{}, "zone-quota", "Show zone-quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error {
		params := jsonutils.Marshal(args)
		quotas, err := modules.ZoneQuotas.GetQuota(s, params)
		if err != nil {
			return err
		}
		printQuotaList(quotas)
		return nil
	})
	R(&QuotaOptions{}, "project-quota", "Show project-quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error {
		params := jsonutils.Marshal(args)
		result, err := modules.ProjectQuotas.GetQuota(s, params)
		if err != nil {
			return err
		}
		printQuotaList(result)
		return nil
	})
	R(&QuotaOptions{}, "image-quota", "Show image quota for current user or tenant", func(s *mcclient.ClientSession, args *QuotaOptions) error {
		params := jsonutils.Marshal(args)
		quotas, err := modules.ImageQuotas.GetQuota(s, params)
		if err != nil {
			return err
		}
		printQuotaList(quotas)
		return nil
	})
	R(&QuotaOptions{}, "domain-quota", "Show domain-quota for current domain", func(s *mcclient.ClientSession, args *QuotaOptions) error {
		params := jsonutils.Marshal(args)
		result, err := modules.DomainQuotas.GetQuota(s, params)
		if err != nil {
			return err
		}
		printQuotaList(result)
		return nil
	})
	R(&QuotaOptions{}, "infras-quota", "Show infras-quota for current domain", func(s *mcclient.ClientSession, args *QuotaOptions) error {
		params := jsonutils.Marshal(args)
		result, err := modules.InfrasQuotas.GetQuota(s, params)
		if err != nil {
			return err
		}
		printQuotaList(result)
		return nil
	})
	R(&QuotaOptions{}, "identity-quota", "Show identity-quota for current domain", func(s *mcclient.ClientSession, args *QuotaOptions) error {
		params := jsonutils.Marshal(args)
		result, err := modules.IdentityQuotas.GetQuota(s, params)
		if err != nil {
			return err
		}
		printQuotaList(result)
		return nil
	})

	type ComputeQuotaSetOptions struct {
		QuotaSetBaseOptions
		ComputeQuotaOptions
	}
	R(&ComputeQuotaSetOptions{}, "quota-set", "Set quota for tenant", func(s *mcclient.ClientSession, args *ComputeQuotaSetOptions) error {
		params := jsonutils.Marshal(args)
		quotas, e := modules.Quotas.DoQuotaSet(s, params)
		if e != nil {
			return e
		}
		printQuotaList(quotas)
		return nil
	})

	type RegionQuotaSetOptions struct {
		QuotaSetBaseOptions
		RegionQuotaOptions
	}
	R(&RegionQuotaSetOptions{}, "region-quota-set", "Set regional quota for tenant", func(s *mcclient.ClientSession, args *RegionQuotaSetOptions) error {
		params := jsonutils.Marshal(args)
		quotas, e := modules.RegionQuotas.DoQuotaSet(s, params)
		if e != nil {
			return e
		}
		printQuotaList(quotas)
		return nil
	})

	type ZoneQuotaSetOptions struct {
		QuotaSetBaseOptions
		ZoneQuotaOptions
	}
	R(&ZoneQuotaSetOptions{}, "zone-quota-set", "Set zonal quota for tenant", func(s *mcclient.ClientSession, args *ZoneQuotaSetOptions) error {
		params := jsonutils.Marshal(args)
		quotas, e := modules.ZoneQuotas.DoQuotaSet(s, params)
		if e != nil {
			return e
		}
		printQuotaList(quotas)
		return nil
	})

	type ProjectQuotaSetOptions struct {
		QuotaSetBaseOptions
		ProjectQuotaOptions
	}
	R(&ProjectQuotaSetOptions{}, "project-quota-set", "Set project quota for tenant", func(s *mcclient.ClientSession, args *ProjectQuotaSetOptions) error {
		params := jsonutils.Marshal(args)
		quotas, e := modules.ProjectQuotas.DoQuotaSet(s, params)
		if e != nil {
			return e
		}
		printQuotaList(quotas)
		return nil
	})

	type ImageQuotaSetOptions struct {
		QuotaSetBaseOptions
		ImageQuotaOptions
	}
	R(&ImageQuotaSetOptions{}, "image-quota-set", "Set image quota for tenant", func(s *mcclient.ClientSession, args *ImageQuotaSetOptions) error {
		params := jsonutils.Marshal(args)
		quotas, e := modules.ImageQuotas.DoQuotaSet(s, params)
		if e != nil {
			return e
		}
		printQuotaList(quotas)
		return nil
	})

	type DomainQuotaSetOptions struct {
		QuotaSetBaseOptions
		DomainQuotaOptions
	}
	R(&DomainQuotaSetOptions{}, "domain-quota-set", "Set domain quota for domain", func(s *mcclient.ClientSession, args *DomainQuotaSetOptions) error {
		params := jsonutils.Marshal(args)
		quotas, e := modules.DomainQuotas.DoQuotaSet(s, params)
		if e != nil {
			return e
		}
		printQuotaList(quotas)
		return nil
	})

	type InfrasQuotaSetOptions struct {
		QuotaSetBaseOptions
		InfrasQuotaOptions
	}
	R(&InfrasQuotaSetOptions{}, "infras-quota-set", "Set infrastructure quota for domain", func(s *mcclient.ClientSession, args *InfrasQuotaSetOptions) error {
		params := jsonutils.Marshal(args)
		quotas, e := modules.InfrasQuotas.DoQuotaSet(s, params)
		if e != nil {
			return e
		}
		printQuotaList(quotas)
		return nil
	})

	type IdentityQuotaSetOptions struct {
		QuotaSetBaseOptions
		IdentityQuotaOptions
	}
	R(&IdentityQuotaSetOptions{}, "identity-quota-set", "Set identity quota for domain", func(s *mcclient.ClientSession, args *IdentityQuotaSetOptions) error {
		params := jsonutils.Marshal(args)
		quotas, e := modules.IdentityQuotas.DoQuotaSet(s, params)
		if e != nil {
			return e
		}
		printQuotaList(quotas)
		return nil
	})

	type QuotaListOptions struct {
		ProjectDomain string `help:"domain name or ID to query project quotas"`
		Refresh       bool   `help:"refresh" json:"refresh,omitfalse"`
		Primary       bool   `help:"list primary quota ONLY" json:"primary,omitfalse"`
	}
	R(&QuotaListOptions{}, "quota-list", "List quota of domains or projects of a domain", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
		params := jsonutils.Marshal(args)
		result, e := modules.Quotas.GetQuotaList(s, params)
		if e != nil {
			return e
		}
		printQuotaList(result)
		return nil
	})

	R(&QuotaListOptions{}, "region-quota-list", "List region quota of domains or projects of a domain", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
		params := jsonutils.Marshal(args)
		result, e := modules.RegionQuotas.GetQuotaList(s, params)
		if e != nil {
			return e
		}
		printQuotaList(result)
		return nil
	})

	R(&QuotaListOptions{}, "zone-quota-list", "List zone quota of domains or projects of a domain", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
		params := jsonutils.Marshal(args)
		result, e := modules.ZoneQuotas.GetQuotaList(s, params)
		if e != nil {
			return e
		}
		printQuotaList(result)
		return nil
	})

	R(&QuotaListOptions{}, "project-quota-list", "List project quota of domains or projects of a domain", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
		params := jsonutils.Marshal(args)
		result, e := modules.ProjectQuotas.GetQuotaList(s, params)
		if e != nil {
			return e
		}
		printQuotaList(result)
		return nil
	})

	R(&QuotaListOptions{}, "image-quota-list", "List image quota of domains or projects of a domain", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
		params := jsonutils.Marshal(args)
		result, e := modules.ImageQuotas.GetQuotaList(s, params)
		if e != nil {
			return e
		}
		printQuotaList(result)
		return nil
	})

	R(&QuotaListOptions{}, "domain-quota-list", "List domain quota of domains", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
		params := jsonutils.Marshal(args)
		result, e := modules.DomainQuotas.GetQuotaList(s, params)
		if e != nil {
			return e
		}
		printQuotaList(result)
		return nil
	})

	R(&QuotaListOptions{}, "infras-quota-list", "List infrastructure quota of domains", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
		params := jsonutils.Marshal(args)
		result, e := modules.InfrasQuotas.GetQuotaList(s, params)
		if e != nil {
			return e
		}
		printQuotaList(result)
		return nil
	})

	R(&QuotaListOptions{}, "identity-quota-list", "List identity quota of domains", func(s *mcclient.ClientSession, args *QuotaListOptions) error {
		params := jsonutils.Marshal(args)
		result, e := modules.IdentityQuotas.GetQuotaList(s, params)
		if e != nil {
			return e
		}
		printQuotaList(result)
		return nil
	})

	type CleanPendingUsageOptions struct {
		Scope   string `help:"scope" choices:"domain|project"`
		Project string `help:"Tenant name or ID" json:"tenant"`
		Domain  string `help:"Domain name or ID" json:"domain"`
	}
	R(&CleanPendingUsageOptions{}, "clean-pending-usage", "Clean pending usage for project or domain", func(s *mcclient.ClientSession, args *CleanPendingUsageOptions) error {
		params := jsonutils.Marshal(args)
		_, err := modules.Quotas.DoCleanPendingUsage(s, params)
		if err != nil {
			return err
		}
		return nil
	})

	R(&CleanPendingUsageOptions{}, "clean-region-pending-usage", "Clean pending usage for project or domain", func(s *mcclient.ClientSession, args *CleanPendingUsageOptions) error {
		params := jsonutils.Marshal(args)
		_, err := modules.RegionQuotas.DoCleanPendingUsage(s, params)
		if err != nil {
			return err
		}
		return nil
	})

	R(&CleanPendingUsageOptions{}, "clean-zone-pending-usage", "Clean pending usage for project or domain", func(s *mcclient.ClientSession, args *CleanPendingUsageOptions) error {
		params := jsonutils.Marshal(args)
		_, err := modules.ZoneQuotas.DoCleanPendingUsage(s, params)
		if err != nil {
			return err
		}
		return nil
	})

	R(&CleanPendingUsageOptions{}, "clean-project-pending-usage", "Clean pending usage for project or domain", func(s *mcclient.ClientSession, args *CleanPendingUsageOptions) error {
		params := jsonutils.Marshal(args)
		_, err := modules.ProjectQuotas.DoCleanPendingUsage(s, params)
		if err != nil {
			return err
		}
		return nil
	})

	R(&CleanPendingUsageOptions{}, "clean-image-pending-usage", "Clean pending usage for project or domain", func(s *mcclient.ClientSession, args *CleanPendingUsageOptions) error {
		params := jsonutils.Marshal(args)
		_, err := modules.ImageQuotas.DoCleanPendingUsage(s, params)
		if err != nil {
			return err
		}
		return nil
	})

	R(&CleanPendingUsageOptions{}, "clean-domain-pending-usage", "Clean pending usage for project or domain", func(s *mcclient.ClientSession, args *CleanPendingUsageOptions) error {
		params := jsonutils.Marshal(args)
		_, err := modules.DomainQuotas.DoCleanPendingUsage(s, params)
		if err != nil {
			return err
		}
		return nil
	})

	R(&CleanPendingUsageOptions{}, "clean-infras-pending-usage", "Clean pending usage for project or domain", func(s *mcclient.ClientSession, args *CleanPendingUsageOptions) error {
		params := jsonutils.Marshal(args)
		_, err := modules.InfrasQuotas.DoCleanPendingUsage(s, params)
		if err != nil {
			return err
		}
		return nil
	})

	R(&CleanPendingUsageOptions{}, "clean-identity-pending-usage", "Clean pending usage for project or domain", func(s *mcclient.ClientSession, args *CleanPendingUsageOptions) error {
		params := jsonutils.Marshal(args)
		_, err := modules.IdentityQuotas.DoCleanPendingUsage(s, params)
		if err != nil {
			return err
		}
		return nil
	})
}


<|start_filename|>pkg/scheduler/algorithm/predicates/isolated_device_predicate.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package predicates

import (
	"fmt"

	"yunion.io/x/onecloud/pkg/scheduler/core"
)

// IsolatedDevicePredicate check mode, and number of scheduled
// device configurations and current resources.
type IsolatedDevicePredicate struct {
	BasePredicate
}

func (f *IsolatedDevicePredicate) Name() string {
	return "host_isolated_device"
}

func (f *IsolatedDevicePredicate) Clone() core.FitPredicate {
	return &IsolatedDevicePredicate{}
}

func (f *IsolatedDevicePredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) {
	data := u.SchedData()
	if len(data.IsolatedDevices) == 0 {
		return false, nil
	}
	return true, nil
}

func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
	h := NewPredicateHelper(f, u, c)
	reqIsoDevs := u.SchedData().IsolatedDevices
	getter := c.Getter()
	minCapacity := int64(0xFFFFFFFF)

	// check by specify device id
	for _, dev := range reqIsoDevs {
		if len(dev.Id) == 0 {
			continue
		}
		if fDev := getter.GetIsolatedDevice(dev.Id); fDev != nil {
			if len(fDev.GuestID) != 0 {
				h.Exclude(fmt.Sprintf("IsolatedDevice %q already used by guest %q", dev.Id, fDev.GuestID))
				return h.GetResult()
			}
		} else {
			h.Exclude(fmt.Sprintf("Not found IsolatedDevice %q", dev.Id))
			return h.GetResult()
		}
		minCapacity = 1
	}

	reqCount := len(reqIsoDevs)
	freeCount := len(getter.UnusedIsolatedDevices()) - getter.GetPendingUsage().IsolatedDevice
	totalCount := len(getter.GetIsolatedDevices())

	// check host isolated device count
	if freeCount < reqCount {
		h.AppendInsufficientResourceError(int64(reqCount), int64(totalCount), int64(freeCount))
		h.Exclude(fmt.Sprintf(
			"IsolatedDevice count not enough, request: %d, hostTotal: %d, hostFree: %d",
			reqCount, totalCount, freeCount))
		return h.GetResult()
	}

	// check host device by type
	devTypeRequest := make(map[string]int, 0)
	for _, dev := range reqIsoDevs {
		if len(dev.DevType) != 0 {
			devTypeRequest[dev.DevType] += 1
		}
	}
	for devType, reqCount := range devTypeRequest {
		freeCount := len(getter.UnusedIsolatedDevicesByType(devType))
		if freeCount < reqCount {
			h.Exclude(fmt.Sprintf("IsolatedDevice type %q not enough, request: %d, hostFree: %d", devType, reqCount, freeCount))
			return h.GetResult()
		}
		cap := freeCount / reqCount
		if int64(cap) < minCapacity {
			minCapacity = int64(cap)
		}
	}

	// check host device by model
	devVendorModelRequest := make(map[string]int, 0)
	for _, dev := range reqIsoDevs {
		if len(dev.Model) != 0 {
			devVendorModelRequest[fmt.Sprintf("%s:%s", dev.Vendor, dev.Model)] += 1
		}
	}
	for vendorModel, reqCount := range devVendorModelRequest {
		freeCount := len(getter.UnusedIsolatedDevicesByVendorModel(vendorModel))
		if freeCount < reqCount {
			h.Exclude(fmt.Sprintf("IsolatedDevice vendor:model %q not enough, request: %d, hostFree: %d", vendorModel, reqCount, freeCount))
			return h.GetResult()
		}
		cap := freeCount / reqCount
		if int64(cap) < minCapacity {
			minCapacity = int64(cap)
		}
	}

	h.SetCapacity(minCapacity)
	return h.GetResult()
}


<|start_filename|>pkg/compute/tasks/dbinstance_set_auto_renew_task.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
	"context"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type DBInstanceSetAutoRenewTask struct {
	taskman.STask
}

func init() {
	taskman.RegisterTask(DBInstanceSetAutoRenewTask{})
}

func (self *DBInstanceSetAutoRenewTask) taskFailed(ctx context.Context, rds *models.SDBInstance, err error) {
	db.OpsLog.LogEvent(rds, db.ACT_SET_AUTO_RENEW_FAIL, err, self.GetUserCred())
	logclient.AddActionLogWithStartable(self, rds, logclient.ACT_SET_AUTO_RENEW, err, self.GetUserCred(), false)
	rds.SetStatus(self.GetUserCred(), api.DBINSTANCE_SET_AUTO_RENEW_FAILED, err.Error())
	self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}

func (self *DBInstanceSetAutoRenewTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	rds := obj.(*models.SDBInstance)
	autoRenew, _ := self.GetParams().Bool("auto_renew")
	iRds, err := rds.GetIDBInstance()
	if err != nil {
		self.taskFailed(ctx, rds, errors.Wrapf(err, "GetIDBInstance"))
		return
	}
	err = iRds.SetAutoRenew(autoRenew)
	if err != nil {
		self.taskFailed(ctx, rds, errors.Wrapf(err, "iRds.SetAutoRenew"))
		return
	}
	self.SetStage("OnDBInstanceSyncComplete", nil)
	rds.StartDBInstanceSyncTask(ctx, self.GetUserCred(), nil, "")
}

func (self *DBInstanceSetAutoRenewTask) OnDBInstanceSyncComplete(ctx context.Context, rds *models.SDBInstance, data jsonutils.JSONObject) {
	self.SetStageComplete(ctx, nil)
}

func (self *DBInstanceSetAutoRenewTask) OnDBInstanceSyncCompleteFailed(ctx context.Context, rds *models.SDBInstance, data jsonutils.JSONObject) {
	self.SetStageFailed(ctx, data)
}


<|start_filename|>pkg/hostman/storageman/storage_base.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storageman

import (
	"context"
	"fmt"
	"io/ioutil"
	"path"
	"regexp"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
	"yunion.io/x/onecloud/pkg/hostman/hostutils"
	"yunion.io/x/onecloud/pkg/hostman/options"
	"yunion.io/x/onecloud/pkg/hostman/storageman/storageutils"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/mcclient/modules"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/procutils"
	"yunion.io/x/onecloud/pkg/util/qemuimg"
)

const (
	_RECYCLE_BIN_     = "recycle_bin"
	_IMGSAVE_BACKUPS_ = "imgsave_backups"
	_SNAPSHOT_PATH_   = "snapshots"

	ErrStorageTimeout = constError("storage accessible check timeout")
	TempBindMountPath = "/opt/cloud/workspace/temp-bind"
)

type constError string

func (e constError) Error() string { return string(e) }

var DELETEING_SNAPSHOTS = sync.Map{}

type IStorageFactory interface {
	NewStorage(manager *SStorageManager, mountPoint string) IStorage
	StorageType() string
}

var (
	storagesFactories = make([]IStorageFactory, 0)
)

// for shared storages
func registerStorageFactory(factory IStorageFactory) {
	storagesFactories = append(storagesFactories, factory)
}

func NewStorage(manager *SStorageManager, mountPoint, storageType string) IStorage {
	for i := range storagesFactories {
		if storageType == storagesFactories[i].StorageType() || strings.HasPrefix(mountPoint, storagesFactories[i].StorageType()) {
			return storagesFactories[i].NewStorage(manager, mountPoint)
		}
	}
	log.Errorf("no storage driver for %s found", storageType)
	return nil
}

type IStorage interface {
	GetId() string
	GetStorageName() string
	GetZoneName() string

	SetStorageInfo(storageId, storageName string, conf jsonutils.JSONObject) error
	SyncStorageInfo() (jsonutils.JSONObject, error)
	SyncStorageSize() error
	StorageType() string
	GetStorageConf() *jsonutils.JSONDict
	GetStoragecacheId() string
	SetStoragecacheId(storagecacheId string)

	SetPath(string)
	GetPath() string

	GetSnapshotDir() string
	GetSnapshotPathByIds(diskId, snapshotId string) string
	DeleteSnapshots(ctx context.Context, params interface{}) (jsonutils.JSONObject, error)
	IsSnapshotExist(diskId, snapshotId string) (bool, error)

	GetFreeSizeMb() int
	GetCapacity() int

	// Find owner disks first, if not found, call create disk
	GetDiskById(diskId string) (IDisk, error)
	CreateDisk(diskId string) IDisk
	RemoveDisk(IDisk)

	// DeleteDisk(ctx context.Context, params interface{}) (jsonutils.JSONObject, error)

	// *SDiskCreateByDiskinfo
	CreateDiskByDiskinfo(context.Context, interface{}) (jsonutils.JSONObject, error)
	SaveToGlance(context.Context, interface{}) (jsonutils.JSONObject, error)
	CreateDiskFromSnapshot(context.Context, IDisk, *SDiskCreateByDiskinfo) error

	CreateSnapshotFormUrl(ctx context.Context, snapshotUrl, diskId, snapshotPath string) error

	DeleteDiskfile(diskPath string) error
	GetFuseTmpPath() string
	GetFuseMountPath() string
	GetImgsaveBackupPath() string

	DestinationPrepareMigrate(ctx context.Context, liveMigrate bool, disksUri string, snapshotsUri string,
		disksBackingFile, srcSnapshots jsonutils.JSONObject, rebaseDisks bool, diskDesc jsonutils.JSONObject) error

	Accessible() error
	Detach() error
}

type SBaseStorage struct {
	Manager          *SStorageManager
	StorageId        string
	Path             string
	StorageName      string
	StorageConf      *jsonutils.JSONDict
	StoragecacheId   string
	isSetStorageInfo bool

	Disks    []IDisk
	DiskLock *sync.Mutex
}

func NewBaseStorage(manager *SStorageManager, path string) *SBaseStorage {
	var ret = new(SBaseStorage)
	ret.Disks = make([]IDisk, 0)
	ret.DiskLock = new(sync.Mutex)
	ret.Manager = manager
	ret.Path = path
	return ret
}

func (s *SBaseStorage) GetId() string {
	return s.StorageId
}

func (s *SBaseStorage) GetStorageName() string {
	return s.StorageName
}

func (s *SBaseStorage) GetStoragecacheId() string {
	return s.StoragecacheId
}

func (s *SBaseStorage) SetStoragecacheId(storagecacheId string) {
	s.StoragecacheId = storagecacheId
}

func (s *SBaseStorage) GetName(generateName func() string) string {
	if len(s.StorageName) > 0 {
		return s.StorageName
	} else {
		return generateName()
	}
}

func (s *SBaseStorage) GetPath() string {
	return s.Path
}

func (s *SBaseStorage) SetPath(p string) {
	s.Path = p
}

func (s *SBaseStorage) GetZoneName() string {
	return s.Manager.GetZoneName()
}

func (s *SBaseStorage) GetCapacity() int {
	return s.GetAvailSizeMb()
}

func (s *SBaseStorage) GetStorageConf() *jsonutils.JSONDict {
	return s.StorageConf
}

func (s *SBaseStorage) GetAvailSizeMb() int {
	return s.GetTotalSizeMb()
}

func (s *SBaseStorage) GetUsedSizeMb() int {
	size, err := storageutils.GetUsedSizeMb(s.Path)
	if err != nil {
		log.Errorf("failed get %s used size: %s", s.Path, err)
		return -1
	}
	return size
}

func (s *SBaseStorage) GetMediumType() string {
	return s.Manager.GetMediumType()
}

func (s *SBaseStorage) GetFreeSizeMb() int {
	size, err := storageutils.GetFreeSizeMb(s.Path)
	if err != nil {
		log.Errorf("failed get %s free size: %s", s.Path, err)
		return -1
	}
	return size
}

func (s *SBaseStorage) GetTotalSizeMb() int {
	size, err := storageutils.GetTotalSizeMb(s.Path)
	if err != nil {
		log.Errorf("failed get %s total size: %s", s.Path, err)
		return -1
	}
	return size
}

func (s *SBaseStorage) SetStorageInfo(storageId, storageName string, conf jsonutils.JSONObject) error {
	s.StorageId = storageId
	s.StorageName = storageName
	if dconf, ok := conf.(*jsonutils.JSONDict); ok {
		s.StorageConf = dconf
	}
	if strings.HasPrefix(s.Path, "/opt/cloud") {
		return nil
	}
	if !s.isSetStorageInfo && options.HostOptions.EnableRemoteExecutor {
		err := s.bindMountTo(s.Path)
		if err == nil {
			s.isSetStorageInfo = true
		}
		return err
	}
	return nil
}

func (s *SBaseStorage) SyncStorageSize() error {
	return fmt.Errorf("not ipmlement")
}

func (s *SBaseStorage) bindMountTo(sPath string) error {
	tempPath := path.Join(TempBindMountPath, sPath)
	out, err := procutils.NewCommand("mkdir", "-p", tempPath).Output()
	if err != nil {
		return errors.Errorf("mkdir temp mount path %s failed %s", tempPath, out)
	}
	out, err = procutils.NewCommand("mkdir", "-p", sPath).Output()
	if err != nil {
		return errors.Errorf("mkdir mount path %s failed %s", sPath, out)
	}
	if procutils.NewCommand("mountpoint", tempPath).Run() != nil {
		out, err = procutils.NewRemoteCommandAsFarAsPossible("mount", "--bind", sPath, tempPath).Output()
		if err != nil {
			return errors.Errorf("bind mount to temp path failed %s", out)
		}
	}
	if procutils.NewCommand("mountpoint", sPath).Run() != nil {
		out, err = procutils.NewCommand("mount", "--bind", tempPath, sPath).Output()
		if err != nil {
			return errors.Errorf("bind mount temp path to local image path failed %s", out)
		}
	}
	log.Infof("bind mount %s -> %s", tempPath, sPath)
	return nil
}

func (s *SBaseStorage) RemoveDisk(d IDisk) {
	s.DiskLock.Lock()
	defer s.DiskLock.Unlock()

	for i := 0; i < len(s.Disks); i++ {
		if s.Disks[i].GetId() == d.GetId() {
			s.Disks = append(s.Disks[:i], s.Disks[i+1:]...)
			break
		}
	}
}

func (s *SBaseStorage) DeleteDiskfile(diskpath string) error {
	return fmt.Errorf("Not Implement")
}

func (s *SBaseStorage) CreateDiskByDiskinfo(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
	createParams, ok := params.(*SDiskCreateByDiskinfo)
	if !ok {
		return nil, hostutils.ParamsError
	}

	if createParams.Disk != nil {
		if !jsonutils.QueryBoolean(createParams.DiskInfo, "rebuild", false) {
			return nil, fmt.Errorf("Disk exist")
		}
		if err := createParams.Disk.OnRebuildRoot(ctx, createParams.DiskInfo); err != nil {
			return nil, err
		}
	}

	disk := createParams.Storage.CreateDisk(createParams.DiskId)
	if disk == nil {
		return nil, fmt.Errorf("Fail to Create disk %s", createParams.DiskId)
	}

	switch {
	case createParams.DiskInfo.Contains("snapshot"):
		log.Infof("CreateDiskFromSnpashot %s", createParams)
		return s.CreateDiskFromSnpashot(ctx, disk, createParams)
	case createParams.DiskInfo.Contains("image_id"):
		log.Infof("CreateDiskFromTemplate %s", createParams)
		return s.CreateDiskFromTemplate(ctx, disk, createParams)
	case createParams.DiskInfo.Contains("size"):
		log.Infof("CreateRawDisk %s", createParams)
		return s.CreateRawDisk(ctx, disk, createParams)
	default:
		return nil, fmt.Errorf("Not fount")
	}
}

func (s *SBaseStorage) CreateRawDisk(ctx context.Context, disk IDisk, createParams *SDiskCreateByDiskinfo) (jsonutils.JSONObject, error) {
	size, _ := createParams.DiskInfo.Int("size")
	diskFromat, _ := createParams.DiskInfo.GetString("format")
	fsFormat, _ := createParams.DiskInfo.GetString("fs_format")
	encryption := jsonutils.QueryBoolean(createParams.DiskInfo, "encryption", false)

	return disk.CreateRaw(ctx, int(size), diskFromat, fsFormat, encryption, createParams.DiskId, "")
}

func (s *SBaseStorage) CreateDiskFromTemplate(ctx context.Context, disk IDisk, createParams *SDiskCreateByDiskinfo) (jsonutils.JSONObject, error) {
	var (
		imageId, _ = createParams.DiskInfo.GetString("image_id")
		format     = "qcow2" // force qcow2
		size, _    = createParams.DiskInfo.Int("size")
	)

	return disk.CreateFromTemplate(ctx, imageId, format, size)
}

func (s *SBaseStorage) CreateDiskFromSnpashot(ctx context.Context, disk IDisk, createParams *SDiskCreateByDiskinfo) (jsonutils.JSONObject, error) {
	var storage = createParams.Storage
	var snapshotUrl, _ = createParams.DiskInfo.GetString("snapshot_url")
	if len(snapshotUrl) == 0 {
		return nil, fmt.Errorf("Create disk from snapshot missing params snapshot url")
	}

	if err := storage.CreateDiskFromSnapshot(ctx, disk, createParams); err != nil {
		return nil, err
	}

	return disk.GetDiskDesc(), nil
}

func (s *SBaseStorage) DestinationPrepareMigrate(
	ctx context.Context, liveMigrate bool, disksUri string, snapshotsUri string,
	disksBackingFile, srcSnapshots jsonutils.JSONObject, rebaseDisks bool, diskinfo jsonutils.JSONObject,
) error {
	return nil
}

/*************************Background delete snapshot job****************************/

func StartSnapshotRecycle(storage IStorage) {
	log.Infof("Snapshot recyle job started")
	if !fileutils2.Exists(storage.GetSnapshotDir()) {
		procutils.NewCommand("mkdir", "-p", storage.GetSnapshotDir()).Run()
	}
	cronman.GetCronJobManager().AddJobAtIntervals(
		"SnapshotRecycle", time.Hour*6,
		func(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
			snapshotRecycle(ctx, userCred, isStart, storage)
		})
}

func StorageRequestSnapshotRecycle(ctx context.Context, userCred mcclient.TokenCredential, storage IStorage) {
	defer func() {
		if r := recover(); r != nil {
			log.Errorf("On storage request snapshot recycle %s \n %s", r, debug.Stack())
		}
	}()

	if !fileutils2.Exists(storage.GetSnapshotDir()) {
		procutils.NewCommand("mkdir", "-p", storage.GetSnapshotDir()).Run()
	}
	snapshotRecycle(ctx, userCred, false, storage)
}

func snapshotRecycle(ctx context.Context, userCred mcclient.TokenCredential, isStart bool, storage IStorage) {
	log.Infof("Snapshot Recycle Job Start, storage is  %s, ss dir is %s", storage.GetStorageName(), storage.GetSnapshotDir())
	res, err := modules.Snapshots.GetById(hostutils.GetComputeSession(ctx), "max-count", nil)
	if err != nil {
		log.Errorln(err)
		return
	}
	maxSnapshotCount, err := res.Int("max_count")
	if err != nil {
		log.Errorln("Request region get snapshot max count failed")
		return
	}
	files, err := ioutil.ReadDir(storage.GetSnapshotDir())
	if err != nil {
		log.Errorln(err)
		return
	}
	for _, file := range files {
		checkSnapshots(storage, file.Name(), int(maxSnapshotCount))
	}
}

func checkSnapshots(storage IStorage, snapshotDir string, maxSnapshotCount int) {
	re := regexp.MustCompile(`^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}_snap$`)
	if !re.MatchString(snapshotDir) {
		log.Warningf("snapshot_dir got unexcept file %s", snapshotDir)
		return
	}
	diskId := snapshotDir[:len(snapshotDir)-len(options.HostOptions.SnapshotDirSuffix)]
	snapshotPath := path.Join(storage.GetSnapshotDir(), snapshotDir)

	// If disk is Deleted, request delete this disk all snapshots
	if !fileutils2.Exists(path.Join(storage.GetPath(), diskId)) && fileutils2.Exists(snapshotPath) {
		params := jsonutils.NewDict()
		params.Set("disk_id", jsonutils.NewString(diskId))
		_, err := modules.Snapshots.PerformClassAction(
			hostutils.GetComputeSession(context.Background()),
			"delete-disk-snapshots", params)
		if err != nil {
			log.Infof("Request delele disk %s snapshots failed %s", diskId, err)
		}
		return
	}

	snapshots, err := ioutil.ReadDir(snapshotPath)
	if err != nil {
		log.Errorln(err)
		return
	}

	// if snapshot count greater than maxsnapshot count, do convert
	if len(snapshots) >= maxSnapshotCount {
		requestConvertSnapshot(storage, snapshotPath, diskId)
	}
}

func requestConvertSnapshot(storage IStorage, snapshotPath, diskId string) {
	log.Infof("SNPASHOT path %s", snapshotPath)
	res, err := modules.Disks.GetSpecific(
		hostutils.GetComputeSession(context.Background()), diskId, "convert-snapshot", nil)
	if err != nil {
		log.Errorln(err)
		return
	}

	var (
		deleteSnapshot, _  = res.GetString("delete_snapshot")
		convertSnapshot, _ = res.GetString("convert_snapshot")
		pendingDelete, _   = res.Bool("pending_delete")
	)
	log.Infof("start convert disk(%s) snapshot(%s), delete_snapshot is %s",
		diskId, convertSnapshot, deleteSnapshot)
	convertSnapshotPath := path.Join(snapshotPath, convertSnapshot)
	outfile := convertSnapshotPath + ".tmp"
	img, err := qemuimg.NewQemuImage(convertSnapshotPath)
	if err != nil {
		log.Errorln(err)
		return
	}
	log.Infof("convertSnapshot path %s", convertSnapshotPath)
	err = img.Convert2Qcow2To(outfile, true)
	if err != nil {
		log.Errorln(err)
		return
	}
	requestDeleteSnapshot(
		storage, diskId, snapshotPath, deleteSnapshot,
		convertSnapshotPath, outfile, pendingDelete,
	)
}

func requestDeleteSnapshot(
	storage IStorage, diskId, snapshotPath, deleteSnapshot, convertSnapshotPath,
	outfile string, pendingDelete bool,
) {
	deleteSnapshotPath := path.Join(snapshotPath, deleteSnapshot)
	DELETEING_SNAPSHOTS.Store(diskId, true)
	defer DELETEING_SNAPSHOTS.Delete(diskId)
	_, err := modules.Snapshots.PerformAction(hostutils.GetComputeSession(context.Background()),
		deleteSnapshot, "deleted", nil)
	if err != nil {
		log.Errorln(err)
		return
	}
	if out, err := procutils.NewCommand("rm", "-f", convertSnapshotPath).Output(); err != nil {
		log.Errorf("%s", out)
		return
	}
	if out, err := procutils.NewCommand("mv", "-f", outfile, convertSnapshotPath).Output(); err != nil {
		log.Errorf("%s", out)
		return
	}
	if !pendingDelete {
		if err := storage.DeleteDiskfile(deleteSnapshotPath); err != nil {
			log.Errorln(err)
			return
		}
	}
}


<|start_filename|>pkg/multicloud/aliyun/dns_domain_record.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aliyun

import (
	"strconv"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudprovider"
)

type SDomainRecords struct {
	//RequestID     string         `json:"RequestId"`
	TotalCount    int            `json:"TotalCount"`
	PageNumber    int            `json:"PageNumber"`
	PageSize      int            `json:"PageSize"`
	DomainRecords sDomainRecords `json:"DomainRecords"`
}

// https://help.aliyun.com/document_detail/29777.html?spm=a2c4g.11186623.6.666.aa4832307YdopF
type SDomainRecord struct {
	DomainId   string `json:"DomainId"`
	GroupId    string `json:"GroupId"`
	GroupName  string `json:"GroupName"`
	PunyCode   string `json:"PunyCode"`
	RR         string `json:"RR"`
	Status     string `json:"Status"`
	Value      string `json:"Value"`
	RecordID   string `json:"RecordId"`
	Type       string `json:"Type"`
	RequestID  string `json:"RequestId"`
	DomainName string `json:"DomainName"`
	Locked     bool   `json:"Locked"`
	Line       string `json:"Line"`
	TTL        int64  `json:"TTL"`
	Priority   int64  `json:"Priority"`
}

type sDomainRecords struct {
	Record []SDomainRecord `json:"Record"`
}

func (client *SAliyunClient) DescribeDomainRecords(domainName string, pageNumber int, pageSize int) (SDomainRecords, error) {
	srecords := SDomainRecords{}
	params := map[string]string{}
	params["Action"] = "DescribeDomainRecords"
	params["DomainName"] = domainName
	params["PageNumber"] = strconv.Itoa(pageNumber)
	params["PageSize"] = strconv.Itoa(pageSize)
	resp, err := client.alidnsRequest("DescribeDomainRecords", params)
	if err != nil {
		return srecords, errors.Wrap(err, "DescribeDomainRecords")
	}
	err = resp.Unmarshal(&srecords)
	if err != nil {
		return srecords, errors.Wrap(err, "resp.Unmarshal")
	}
	return srecords, nil
}

func (client *SAliyunClient) GetAllDomainRecords(domainName string) ([]SDomainRecord, error) {
	srecords := []SDomainRecord{}
	pageNumber := 0
	for {
		pageNumber++
		records, err := client.DescribeDomainRecords(domainName, pageNumber, 2)
		if err != nil {
			return nil, errors.Wrapf(err, "client.DescribeDomainRecords(%d, 20)", len(srecords))
		}
		srecords = append(srecords, records.DomainRecords.Record...)
		if len(srecords) >= records.TotalCount {
			break
		}
	}
	return srecords, nil
}

func (client *SAliyunClient) DescribeDomainRecordInfo(recordId string) (*SDomainRecord, error) {
	srecord := SDomainRecord{}
	params := map[string]string{}
	params["Action"] = "DescribeDomainRecordInfo"
	params["RecordId"] = recordId

	resp, err := client.alidnsRequest("DescribeDomainRecordInfo", params)
	if err != nil {
		return nil, errors.Wrap(err, "DescribeDomainRecordInfo")
	}
	err = resp.Unmarshal(&srecord)
	if err != nil {
		return nil, errors.Wrap(err, "resp.Unmarshal")
	}
	return &srecord, nil
}

func GetRecordLineLineType(policyinfo cloudprovider.TDnsPolicyValue) string {
	switch policyinfo {
	case cloudprovider.DnsPolicyValueOversea:
		return "oversea"
	case cloudprovider.DnsPolicyValueTelecom:
		return "telecom"
	case cloudprovider.DnsPolicyValueUnicom:
		return "unicom"
	case cloudprovider.DnsPolicyValueChinaMobile:
		return "mobile"
	case cloudprovider.DnsPolicyValueCernet:
		return "edu"
	case cloudprovider.DnsPolicyValueDrPeng:
		return "drpeng"
	case cloudprovider.DnsPolicyValueBtvn:
		return "btvn"

	case cloudprovider.DnsPolicyValueBaidu:
		return "baidu"
	case cloudprovider.DnsPolicyValueGoogle:
		return "google"
	case cloudprovider.DnsPolicyValueYoudao:
		return "youdao"
	case cloudprovider.DnsPolicyValueBing:
		return "biying"
	default:
		return "default"
	}
}

func (client *SAliyunClient) AddDomainRecord(domainName string, opts cloudprovider.DnsRecordSet) (string, error) {
	line := GetRecordLineLineType(opts.PolicyValue)
	params := map[string]string{}
	params["Action"] = "AddDomainRecord"
	params["RR"] = opts.DnsName
	params["Type"] = string(opts.DnsType)
	params["Value"] = opts.DnsValue
	params["DomainName"] = domainName
	params["TTL"] = strconv.FormatInt(opts.Ttl, 10)
	params["Line"] = line
	if opts.DnsType == cloudprovider.DnsTypeMX {
		params["Priority"] = strconv.FormatInt(opts.MxPriority, 10)
	}
	ret, err := client.alidnsRequest("AddDomainRecord", params)
	if err != nil {
		return "", errors.Wrap(err, "AddDomainRecord")
	}
	recordId := ""
	return recordId, ret.Unmarshal(&recordId, "RecordId")
}

// line
func (client *SAliyunClient) UpdateDomainRecord(opts cloudprovider.DnsRecordSet) error {
	line := GetRecordLineLineType(opts.PolicyValue)
	params := map[string]string{}
	params["Action"] = "UpdateDomainRecord"
	params["RR"] = opts.DnsName
	params["RecordId"] = opts.ExternalId
	params["Type"] = string(opts.DnsType)
	params["Value"] = opts.DnsValue
	params["TTL"] = strconv.FormatInt(opts.Ttl, 10)
	params["Line"] = line
	if opts.DnsType == cloudprovider.DnsTypeMX {
		params["Priority"] = strconv.FormatInt(opts.MxPriority, 10)
	}
	_, err := client.alidnsRequest("UpdateDomainRecord", params)
	if err != nil {
		return errors.Wrap(err, "UpdateDomainRecord")
	}
	return nil
}

// Enable: 启用解析 Disable: 暂停解析
func (client *SAliyunClient) SetDomainRecordStatus(recordId, status string) error {
	params := map[string]string{}
	params["Action"] = "SetDomainRecordStatus"
	params["RecordId"] = recordId
	params["Status"] = status
	_, err := client.alidnsRequest("SetDomainRecordStatus", params)
	if err != nil {
		return errors.Wrap(err, "SetDomainRecordStatus")
	}
	return nil
}

func (client *SAliyunClient) DeleteDomainRecord(recordId string) error {
	params := map[string]string{}
	params["Action"] = "DeleteDomainRecord"
	params["RecordId"] = recordId
	_, err := client.alidnsRequest("DeleteDomainRecord", params)
	if err != nil {
		return errors.Wrap(err, "DeleteDomainRecord")
	}
	return nil
}

func (self *SDomainRecord) GetGlobalId() string {
	return self.RecordID
}

func (self *SDomainRecord) GetDnsName() string {
	return self.RR
}

func (self *SDomainRecord) GetStatus() string {
	return api.DNS_RECORDSET_STATUS_AVAILABLE
}

func (self *SDomainRecord) GetEnabled() bool {
	return self.Status == "ENABLE"
}

func (self *SDomainRecord) GetDnsType() cloudprovider.TDnsType {
	return cloudprovider.TDnsType(self.Type)
}

func (self *SDomainRecord) GetDnsValue() string {
	return self.Value
}

func (self *SDomainRecord) GetTTL() int64 {
	return self.TTL
}

func (self *SDomainRecord) GetMxPriority() int64 {
	if self.GetDnsType() == cloudprovider.DnsTypeMX {
		return self.Priority
	}
	return 0
}

func (self *SDomainRecord) GetPolicyType() cloudprovider.TDnsPolicyType {
	switch self.Line {
	case "telecom", "unicom", "mobile", "edu", "drpeng", "btvn":
		return cloudprovider.DnsPolicyTypeByCarrier
	case "google", "baidu", "biying", "youdao":
		return cloudprovider.DnsPolicyTypeBySearchEngine
	case "oversea":
		return cloudprovider.DnsPolicyTypeByGeoLocation
	default:
		return cloudprovider.DnsPolicyTypeSimple
	}
}

func (self *SDomainRecord) GetPolicyValue() cloudprovider.TDnsPolicyValue {
	switch self.Line {
	case "telecom":
		return cloudprovider.DnsPolicyValueTelecom
	case "unicom":
		return cloudprovider.DnsPolicyValueUnicom
	case "mobile":
		return cloudprovider.DnsPolicyValueChinaMobile
	case "oversea":
		return cloudprovider.DnsPolicyValueOversea
	case "edu":
		return cloudprovider.DnsPolicyValueCernet

	case "drpeng":
		return cloudprovider.DnsPolicyValueDrPeng
	case "btvn":
		return cloudprovider.DnsPolicyValueBtvn
	case "google":
		return cloudprovider.DnsPolicyValueGoogle
	case "baidu":
		return cloudprovider.DnsPolicyValueBaidu
	case "biying":
		return cloudprovider.DnsPolicyValueBing
	case "youdao":
		return cloudprovider.DnsPolicyValueYoudao

	}
	return ""
}

func (self *SDomainRecord) GetPolicyOptions() *jsonutils.JSONDict {
	return nil
}

func (self *SDomainRecord) match(update cloudprovider.DnsRecordSet) bool {
	if update.DnsName != self.GetDnsName() {
		return false
	}
	if update.DnsType != self.GetDnsType() {
		return false
	}
	if update.DnsValue != self.GetDnsValue() {
		return false
	}
	if update.PolicyType != self.GetPolicyType() {
		return false
	}
	if update.PolicyValue != self.GetPolicyValue() {
		return false
	}
	if update.Ttl != self.GetTTL() {
		return false
	}
	return true
}


<|start_filename|>pkg/logger/service/handlers.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"yunion.io/x/onecloud/pkg/appsrv"
	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/logger/models"
)

func initHandlers(app *appsrv.Application) {
	db.InitAllManagers()

	for _, manager := range []db.IModelManager{
		db.UserCacheManager,
		db.TenantCacheManager,
		db.DistinctFieldManager,
	} {
		db.RegisterModelManager(manager)
	}

	for _, manager := range []db.IModelManager{
		models.ActionLog,
		models.BaremetalEventManager,
	} {
		db.RegisterModelManager(manager)
		handler := db.NewModelHandler(manager)
		dispatcher.AddModelDispatcher("", app, handler)
	}
}


<|start_filename|>pkg/multicloud/azure/shell/storageaccount.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shell

import (
	"fmt"

	"yunion.io/x/onecloud/pkg/multicloud/azure"
	"yunion.io/x/onecloud/pkg/util/shellutils"
)

func init() {
	type StorageAccountListOptions struct {
	}
	shellutils.R(&StorageAccountListOptions{}, "storage-account-list", "List storage account", func(cli *azure.SRegion, args *StorageAccountListOptions) error {
		accounts, err := cli.ListStorageAccounts()
		if err != nil {
			return err
		}
		printList(accounts, len(accounts), 0, 0, []string{})
		return nil
	})

	shellutils.R(&StorageAccountListOptions{}, "classic-storage-account-list", "List classic storage account", func(cli *azure.SRegion, args *StorageAccountListOptions) error {
		accounts, err := cli.ListClassicStorageAccounts()
		if err != nil {
			return err
		}
		printList(accounts, len(accounts), 0, 0, []string{})
		return nil
	})

	type StorageAccountOptions struct {
		ID string `help:"StorageAccount ID"`
	}

	shellutils.R(&StorageAccountOptions{}, "classic-storage-account-show", "Show storage account detail", func(cli *azure.SRegion, args *StorageAccountOptions) error {
		account, err := cli.GetClassicStorageAccount(args.ID)
		if err != nil {
			return err
		}
		printObject(account)
		return nil
	})

	shellutils.R(&StorageAccountOptions{}, "storage-account-delete", "Delete storage account", func(cli *azure.SRegion, args *StorageAccountOptions) error {
		return cli.DeleteStorageAccount(args.ID)
	})

	shellutils.R(&StorageAccountOptions{}, "storage-account-show", "Show storage account detail", func(cli *azure.SRegion, args *StorageAccountOptions) error {
		account, err := cli.GetStorageAccountDetail(args.ID)
		if err != nil {
			return err
		}
		printObject(account)
		return nil
	})

	shellutils.R(&StorageAccountOptions{}, "storage-account-key", "Get storage account key", func(cli *azure.SRegion, args *StorageAccountOptions) error {
		if key, err := cli.GetStorageAccountKey(args.ID); err != nil {
			return err
		} else {
			fmt.Printf("Key: %s", key)
			return nil
		}
	})

	shellutils.R(&StorageAccountOptions{}, "storage-container-list", "Get list of containers of a storage account", func(cli *azure.SRegion, args *StorageAccountOptions) error {
		account, err := cli.GetStorageAccountDetail(args.ID)
		if err != nil {
			return err
		}
		containers, err := account.GetContainers()
		if err != nil {
			return err
		}
		printList(containers, len(containers), 0, 0, nil)
		return nil
	})

	type StorageAccountCreateContainerOptions struct {
		ACCOUNT   string `help:"storage account ID"`
		CONTAINER string `help:"name of container to create"`
	}
	shellutils.R(&StorageAccountCreateContainerOptions{}, "storage-container-create", "Create a container in a storage account", func(cli *azure.SRegion, args *StorageAccountCreateContainerOptions) error {
		account, err := cli.GetStorageAccountDetail(args.ACCOUNT)
		if err != nil {
			return err
		}
		container, err := account.CreateContainer(args.CONTAINER)
		if err != nil {
			return err
		}
		printObject(container)
		return nil
	})

	shellutils.R(&StorageAccountCreateContainerOptions{}, "storage-container-list-objects", "Create a container in a storage account", func(cli *azure.SRegion, args *StorageAccountCreateContainerOptions) error {
		account, err := cli.GetStorageAccountDetail(args.ACCOUNT)
		if err != nil {
			return err
		}
		container, err := account.GetContainer(args.CONTAINER)
		if err != nil {
			return err
		}
		blobs, err := container.ListAllFiles(nil)
		if err != nil {
			return err
		}
		printList(blobs, len(blobs), 0, 0, nil)
		return nil
	})

	type StorageAccountUploadOptions struct {
		ACCOUNT   string `help:"storage account ID"`
		CONTAINER string `help:"name of container to create"`
		FILE      string `help:"local file to upload"`
	}
	shellutils.R(&StorageAccountUploadOptions{}, "storage-container-upload", "Upload a container in a storage account", func(cli *azure.SRegion, args *StorageAccountUploadOptions) error {
		account, err := cli.GetStorageAccountDetail(args.ACCOUNT)
		if err != nil {
			return err
		}
		container, err := account.GetContainer(args.CONTAINER)
		if err != nil {
			return err
		}
		url, err := container.UploadFile(args.FILE)
		if err != nil {
			return err
		}
		fmt.Println(url)
		return nil
	})

	type StorageAccountCreateOptions struct {
		NAME string `help:"StorageAccount NAME"`
	}

	shellutils.R(&StorageAccountCreateOptions{}, "storage-account-create", "Create a storage account", func(cli *azure.SRegion, args *StorageAccountCreateOptions) error {
		if account, err := cli.CreateStorageAccount(args.NAME); err != nil {
			return err
		} else {
			printObject(account)
			return nil
		}
	})

	type StorageAccountCheckeOptions struct {
	}

	shellutils.R(&StorageAccountCheckeOptions{}, "storage-uniq-name", "Get a uniqel storage account name", func(cli *azure.SRegion, args *StorageAccountCheckeOptions) error {
		uniqName, err := cli.GetUniqStorageAccountName()
		if err != nil {
			return err
		}
		fmt.Println(uniqName)
		return nil
	})

	type SStorageAccountSkuOptions struct {
	}
	shellutils.R(&SStorageAccountSkuOptions{}, "storage-account-skus", "List skus of storage account", func(cli *azure.SRegion, args *SStorageAccountSkuOptions) error {
		skus, err := cli.GetStorageAccountSkus()
		if err != nil {
			return err
		}
		printList(skus, 0, 0, 0, nil)
		return nil
	})
}


<|start_filename|>pkg/compute/tasks/route_table_update_task.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
	"context"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type RouteTableUpdateTask struct {
	taskman.STask
}

func (self *RouteTableUpdateTask) taskFailed(ctx context.Context, routeTable *models.SRouteTable, err error) {
	routeTable.SetStatus(self.GetUserCred(), api.ROUTE_TABLE_UPDATEFAILED, err.Error())
	db.OpsLog.LogEvent(routeTable, db.ACT_UPDATE, err, self.GetUserCred())
	logclient.AddActionLogWithContext(ctx, routeTable, logclient.ACT_UPDATE, err, self.UserCred, false)
	self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}

func init() {
	taskman.RegisterTask(RouteTableUpdateTask{})
}

func (self *RouteTableUpdateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	routeTable := obj.(*models.SRouteTable)
	action, err := self.Params.GetString("action")
	if err != nil {
		self.taskFailed(ctx, routeTable, errors.Wrapf(err, "self.Params.GetString(action)"))
		return
	}

	routeSetId, err := self.Params.GetString("route_table_route_set_id")
	if err != nil {
		self.taskFailed(ctx, routeTable, errors.Wrapf(err, "self.Params.GetString(route_table_route_set_id)"))
		return
	}

	_routeSet, err := models.RouteTableRouteSetManager.FetchById(routeSetId)
	if err != nil {
		self.taskFailed(ctx, routeTable, errors.Wrapf(err, "RouteTableRouteSetManager.FetchById(routeSetId)"))
		return
	}
	routeSet := _routeSet.(*models.SRouteTableRouteSet)

	iRouteTable, err := routeTable.GetICloudRouteTable()
	if err != nil {
		self.taskFailed(ctx, routeTable, errors.Wrapf(err, "routeTable.GetICloudRouteTable()"))
		return
	}
	cloudRouteSet := cloudprovider.RouteSet{
		RouteId:     routeSet.ExternalId,
		Destination: routeSet.Cidr,
		NextHopType: routeSet.NextHopType,
		NextHop:     routeSet.ExtNextHopId,
	}
	err = nil
	switch action {
	case "create":
		err = iRouteTable.CreateRoute(cloudRouteSet)
		if err != nil {
			self.taskFailed(ctx, routeTable, errors.Wrapf(err, "iRouteTable.CreateRoute(%s)", jsonutils.Marshal(cloudRouteSet).String()))
			return
		}
	case "update":
		err = iRouteTable.UpdateRoute(cloudRouteSet)
		if err != nil {
			self.taskFailed(ctx, routeTable, errors.Wrapf(err, "iRouteTable.UpdateRoute(%s)", jsonutils.Marshal(cloudRouteSet).String()))
			return
		}
	case "delete":
		err = iRouteTable.RemoveRoute(cloudRouteSet)
		if err != nil {
			self.taskFailed(ctx, routeTable, errors.Wrapf(err, "iRouteTable.RemoveRoute(%s)", jsonutils.Marshal(cloudRouteSet).String()))
			return
		}
	default:
		self.taskFailed(ctx, routeTable, errors.Wrapf(err, "invalid routetable update action"))
		return
	}
	logclient.AddActionLogWithContext(ctx, routeTable, logclient.ACT_UPDATE, nil, self.UserCred, true)

	self.SetStage("OnSyncRouteTableComplete", nil)
	models.StartResourceSyncStatusTask(ctx, self.GetUserCred(), routeTable, "RouteTableSyncStatusTask", self.GetTaskId())
}

func (self *RouteTableUpdateTask) OnSyncRouteTableComplete(ctx context.Context, routeTable *models.SRouteTable, data jsonutils.JSONObject) {
	self.SetStageComplete(ctx, nil)
}

func (self *RouteTableUpdateTask) OnSyncRouteTableCompleteFailed(ctx context.Context, routeTable *models.SRouteTable, data jsonutils.JSONObject) {
	self.SetStageFailed(ctx, data)
}


<|start_filename|>pkg/multicloud/qcloud/shell/vpc.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shell

import (
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/multicloud/qcloud"
	"yunion.io/x/onecloud/pkg/util/shellutils"
)

func init() {
	type VpcListOptions struct {
		Limit  int `help:"page size"`
		Offset int `help:"page offset"`
	}
	shellutils.R(&VpcListOptions{}, "vpc-list", "List vpcs", func(cli *qcloud.SRegion, args *VpcListOptions) error {
		vpcs, total, err := cli.GetVpcs(nil, args.Offset, args.Limit)
		if err != nil {
			return err
		}
		printList(vpcs, total, args.Offset, args.Limit, []string{})
		return nil
	})

	type VpcCreateOptions struct {
		NAME string `help:"Name for vpc"`
		CIDR string `help:"Cidr for vpc" choices:"10.0.0.0/16|172.16.0.0/12|192.168.0.0/16"`
	}
	shellutils.R(&VpcCreateOptions{}, "vpc-create", "Create vpc", func(cli *qcloud.SRegion, args *VpcCreateOptions) error {
		vpc, err := cli.CreateIVpc(args.NAME, "", args.CIDR)
		if err != nil {
			return err
		}
		printObject(vpc)
		return nil
	})

	type VpcDeleteOptions struct {
		ID string `help:"VPC ID or Name"`
	}
	shellutils.R(&VpcDeleteOptions{}, "vpc-delete", "Delete vpc", func(cli *qcloud.SRegion, args *VpcDeleteOptions) error {
		return cli.DeleteVpc(args.ID)
	})

	type VpcPCListOption struct {
		VPCID  string
		Limit  int `help:"page size"`
		Offset int `help:"page offset"`
	}
	shellutils.R(&VpcPCListOption{}, "vpcPC-list", "List vpc peering connections", func(cli *qcloud.SRegion, args *VpcPCListOption) error {
		vpcs, total, err := cli.DescribeVpcPeeringConnections(args.VPCID, "", args.Offset, args.Limit)
		if err != nil {
			return err
		}
		printList(vpcs, total, args.Offset, args.Limit, []string{})
		return nil
	})

	type VpcPCShowOption struct {
		VPCPCID string
	}
	shellutils.R(&VpcPCShowOption{}, "vpcPC-show", "show vpc peering connection", func(cli *qcloud.SRegion, args *VpcPCShowOption) error {
		vpcPC, err := cli.GetVpcPeeringConnectionbyId(args.VPCPCID)
		if err != nil {
			return err
		}
		printObject(vpcPC)
		return nil
	})

	type VpcPCCreateOPtion struct {
		NAME         string
		VPCID        string
		PEERVPCID    string
		PEERREGIONID string
		PEEROWNERID  string
		Bandwidth    int
	}
	shellutils.R(&VpcPCCreateOPtion{}, "vpcPC-create", "create vpc peering connection", func(cli *qcloud.SRegion, args *VpcPCCreateOPtion) error {
		opts := cloudprovider.VpcPeeringConnectionCreateOptions{}
		opts.Name = args.NAME
		opts.PeerVpcId = args.PEERVPCID
		opts.PeerAccountId = args.PEEROWNERID
		opts.PeerRegionId = args.PEERREGIONID
		vpcPCId, err := cli.CreateVpcPeeringConnection(args.VPCID, &opts)
		if err != nil {
			return err
		}
		printObject(vpcPCId)
		return nil
	})

	shellutils.R(&VpcPCCreateOPtion{}, "vpcPC-createEx", "create Ex vpc peering connection", func(cli *qcloud.SRegion, args *VpcPCCreateOPtion) error {
		opts := cloudprovider.VpcPeeringConnectionCreateOptions{}
		opts.Name = args.NAME
		opts.PeerVpcId = args.PEERVPCID
		opts.PeerAccountId = args.PEEROWNERID
		opts.PeerRegionId = args.PEERREGIONID
		opts.Bandwidth = args.Bandwidth
		taskId, err := cli.CreateVpcPeeringConnectionEx(args.VPCID, &opts)
		if err != nil {
			return err
		}
		printObject(taskId)
		return nil
	})

	type VpcPeeringAcceptOPtion struct {
		VPCPEERINGID string
	}
	shellutils.R(&VpcPeeringAcceptOPtion{}, "vpcPC-accept", "Accept vpcPeering", func(cli *qcloud.SRegion, args *VpcPeeringAcceptOPtion) error {
		err := cli.AcceptVpcPeeringConnection(args.VPCPEERINGID)
		if err != nil {
			return err
		}
		return nil
	})

	shellutils.R(&VpcPeeringAcceptOPtion{}, "vpcPC-acceptEx", "Accept Ex vpcPeering", func(cli *qcloud.SRegion, args *VpcPeeringAcceptOPtion) error {
		_, err := cli.AcceptVpcPeeringConnectionEx(args.VPCPEERINGID)
		if err != nil {
			return err
		}
		return nil
	})

	type VpcPeeringDeleteOPtion struct {
		VPCPEERINGID string
	}
	shellutils.R(&VpcPeeringAcceptOPtion{}, "vpcPC-delete", "Delete vpcPeering", func(cli *qcloud.SRegion, args *VpcPeeringAcceptOPtion) error {
		err := cli.DeleteVpcPeeringConnection(args.VPCPEERINGID)
		if err != nil {
			return err
		}
		return nil
	})

	shellutils.R(&VpcPeeringAcceptOPtion{}, "vpcPC-deleteEx", "Delete Ex vpcPeering", func(cli *qcloud.SRegion, args *VpcPeeringAcceptOPtion) error {
		status, err := cli.DeleteVpcPeeringConnectionEx(args.VPCPEERINGID)
		if err != nil {
			return err
		}
		println(status)
		return nil
	})

	type VpcTaskShowOption struct {
		TASKID string
	}
	shellutils.R(&VpcTaskShowOption{}, "vpcTask-show", "show vpc task", func(cli *qcloud.SRegion, args *VpcTaskShowOption) error {
		status, err := cli.DescribeVpcTaskResult(args.TASKID)
		if err != nil {
			return err
		}
		println(status)
		return nil
	})

}


<|start_filename|>pkg/multicloud/qcloud/shell/cdn.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shell

import (
	"yunion.io/x/onecloud/pkg/multicloud/qcloud"
	"yunion.io/x/onecloud/pkg/util/shellutils"
)

func init() {
	type CdnDomainCreateOption struct {
		Domain           string
		OriginType       string
		Origins          []string
		CosPrivateAccess string
	}

	shellutils.R(&CdnDomainCreateOption{}, "cdn-domain-create", "create cdn domain", func(cli *qcloud.SRegion, args *CdnDomainCreateOption) error {
		err := cli.GetClient().AddCdnDomain(args.Domain, args.OriginType, args.Origins, args.CosPrivateAccess)
		if err != nil {
			return err
		}
		return nil
	})

	type CdnDomainListOption struct {
		Domains    []string
		Origins    []string
		DomainType string `help:"origin domain type" choices:"cos|cname"`
	}
	shellutils.R(&CdnDomainListOption{}, "cdn-domain-list", "list cdn domain", func(cli *qcloud.SRegion, args *CdnDomainListOption) error {
		domains, err := cli.GetClient().DescribeAllCdnDomains(args.Domains, args.Origins, args.DomainType)
		if err != nil {
			return err
		}
		printList(domains, len(domains), 0, len(domains), nil)
		return nil
	})

}


<|start_filename|>pkg/multicloud/esxi/vswitch.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package esxi

import (
	"regexp"

	"github.com/vmware/govmomi/property"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/types"

	"yunion.io/x/pkg/errors"
)

var DVS_PROPS = []string{"name"}

type IVirtualSwitch interface {
	FindNetworkByVlanID(vlanID int32) (IVMNetwork, error)
}

type SVirtualSwitch struct {
	Host              *SHost
	HostVirtualSwitch types.HostVirtualSwitch
}

func (vs *SVirtualSwitch) FindNetworkByVlanID(vlanID int32) (IVMNetwork, error) {
	vsKey := vs.HostVirtualSwitch.Key
	pgs := vs.Host.getHostSystem().Config.Network.Portgroup
	var networkName string
	for i := range pgs {
		if pgs[i].Vswitch != vsKey {
			continue
		}
		if vlanEqual(pgs[i].Spec.VlanId, vlanID) {
			networkName = pgs[i].Spec.Name
			break
		}
	}
	if len(networkName) == 0 {
		return nil, nil
	}
	networks, err := vs.Host.GetNetworks()
	if err != nil {
		return nil, errors.Wrapf(err, "can't get networks of host %q", vs.Host.GetGlobalId())
	}
	for i := range networks {
		if networks[i].GetName() == networkName {
			return networks[i], nil
		}
	}
	return nil, nil
}

type SDistributedVirtualSwitch struct {
	Host                     *SHost
	DistributedVirtualSwitch mo.DistributedVirtualSwitch
}

func (vs *SDistributedVirtualSwitch) FindNetworkByVlanID(vlanID int32) (IVMNetwork, error) {
	var modvpgs []mo.DistributedVirtualPortgroup
	filter := property.Filter{}
	filter["config.distributedVirtualSwitch"] = vs.DistributedVirtualSwitch.Self
	err := vs.Host.manager.scanMObjectsWithFilter(vs.Host.datacenter.object.Entity().Self, DVPORTGROUP_PROPS, &modvpgs, filter)
	if err != nil {
		return nil, errors.Wrapf(err, "can't fetch portgroup of DistributedVirtualSwitch %q", vs.DistributedVirtualSwitch.Name)
	}
	dvpgs := make([]*SDistributedVirtualPortgroup, 0, len(modvpgs))
	for i := range modvpgs {
		dvpgs = append(dvpgs, NewDistributedVirtualPortgroup(vs.Host.manager, &modvpgs[i], nil))
	}
	for i := range dvpgs {
		if vlanEqual(dvpgs[i].GetVlanId(), vlanID) {
			return dvpgs[i], nil
		}
	}
	return nil, nil
}

func vlanEqual(v1, v2 int32) bool {
	if v1 <= 1 && v2 <= 1 {
		return true
	}
	return v1 == v2
}

var (
	vsBridgeRegex  = regexp.MustCompile(`^(host-\d+)/(.*)`)
	dvsBridgeRegex = regexp.MustCompile(`^dvs-\d+$`)
)

// config.distributedVirtualSwitch

func findVirtualSwitch(host *SHost, bridge string) (IVirtualSwitch, error) {
	group := vsBridgeRegex.FindStringSubmatch(bridge)
	oHost := host.getHostSystem()
	if len(group) > 0 {
		// vswitch
		vsName := group[2]
		for _, vs := range oHost.Config.Network.Vswitch {
			if vs.Name != vsName {
				continue
			}
			return &SVirtualSwitch{
				Host:              host,
				HostVirtualSwitch: vs,
			}, nil
		}
		return nil, nil
	}

	// distributed vswitch
	if !dvsBridgeRegex.MatchString(bridge) {
		return nil, nil
	}
	objRef := types.ManagedObjectReference{
		Type:  "VmwareDistributedVirtualSwitch",
		Value: bridge,
	}
	var dvs mo.DistributedVirtualSwitch
	err := host.manager.reference2Object(objRef, DVS_PROPS, &dvs)
	if err != nil {
		return nil, errors.Wrapf(err, "can't fetch DistributedVirtualSwitch %q", objRef.String())
	}
	return &SDistributedVirtualSwitch{
		Host:                     host,
		DistributedVirtualSwitch: dvs,
	}, nil
}


<|start_filename|>pkg/multicloud/aws/networkinterface.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"time"

	"yunion.io/x/pkg/errors"
)

type SGroupSet struct {
	GroupId   string `xml:"groupId"`
	GroupName string `xml:"groupName"`
}

type SPrivateIpAddress struct {
	PrivateIpAddress string `xml:"privateIpAddress"`
	PrivateDnsName   string `xml:"privateDnsName"`
	Primary          bool   `xml:"primary"`
}

type SAttachment struct {
	AttachmentId        string    `xml:"attachmentId"`
	InstanceOwnerId     string    `xml:"instanceOwnerId"`
	DeviceIndex         int       `xml:"deviceIndex"`
	Status              string    `xml:"status"`
	AttachTime          time.Time `xml:"attachTime"`
	DeleteOnTermination bool      `xml:"deleteOnTermination"`
}

type SNetworkInterface struct {
	NetworkInterfaceId    string              `xml:"networkInterfaceId"`
	SubnetId              string              `xml:"subnetId"`
	VpcId                 string              `xml:"vpcId"`
	AvailabilityZone      string              `xml:"availabilityZone"`
	Description           string              `xml:"description"`
	OwnerId               string              `xml:"ownerId"`
	RequesterId           string              `xml:"requesterId"`
	RequesterManaged      bool                `xml:"requesterManaged"`
	Status                string              `xml:"status"`
	MacAddress            string              `xml:"macAddress"`
	PrivateIpAddress      string              `xml:"privateIpAddress"`
	PrivateDnsName        string              `xml:"privateDnsName"`
	SourceDestCheck       bool                `xml:"sourceDestCheck"`
	GroupSet              []SGroupSet         `xml:"groupSet>item"`
	Attachment            SAttachment         `xml:"attachment"`
	PrivateIpAddressesSet []SPrivateIpAddress `xml:"privateIpAddressesSet>item"`
	InterfaceType         string              `xml:"interfaceType"`
}

type SNetworkInterfaces struct {
	NetworkInterface []SNetworkInterface `xml:"networkInterfaceSet>item"`
}

func (region *SRegion) GetNetworkInterfaces() ([]SNetworkInterface, error) {
	params := map[string]string{}
	interfaces := SNetworkInterfaces{}
	err := region.ec2Request("DescribeNetworkInterfaces", params, &interfaces)
	if err != nil {
		return nil, errors.Wrap(err, "DescribeNetworkInterfaces")
	}
	return interfaces.NetworkInterface, nil
}


<|start_filename|>cmd/climc/shell/identity/domains.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package identity

import (
	"yunion.io/x/jsonutils"

	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/mcclient/modules"
	"yunion.io/x/onecloud/pkg/mcclient/options"
)

func init() {
	type DomainListOptions struct {
		options.BaseListOptions
		IdpId       string `help:"filter by idp_id"`
		IdpEntityId string `help:"filter by idp_entity_id"`
	}
	R(&DomainListOptions{}, "domain-list", "List domains", func(s *mcclient.ClientSession, args *DomainListOptions) error {
		params, err := options.ListStructToParams(args)
		if err != nil {
			return err
		}
		result, err := modules.Domains.List(s, params)
		if err != nil {
			return err
		}
		printList(result, modules.Domains.GetColumns(s))
		return nil
	})

	type DomainDetailOptions struct {
		ID string `help:"ID or domain"`
	}
	R(&DomainDetailOptions{}, "domain-show", "Show detail of domain", func(s *mcclient.ClientSession, args *DomainDetailOptions) error {
		result, err := modules.Domains.Get(s, args.ID, nil)
		if err != nil {
			return err
		}
		printObject(result)
		return nil
	})
	R(&DomainDetailOptions{}, "domain-delete", "Delete a domain", func(s *mcclient.ClientSession, args *DomainDetailOptions) error {
		objId, err := modules.Domains.GetId(s, args.ID, nil)
		if err != nil {
			return err
		}
		result, err := modules.Domains.Delete(s, objId, nil)
		if err != nil {
			return err
		}
		printObject(result)
		return nil
	})

	/* R(&DomainDetailOptions{}, "domain-config-sql", "Config a domain with SQL driver", func(s *mcclient.ClientSession, args *DomainDetailOptions) error {
	    config := jsonutils.NewDict()
	    config.Add(jsonutils.NewString("sql"), "config", "identity", "driver")
	    objId, err := modules.Domains.GetId(s, args.ID, nil)
	    if err != nil {
	        return err
	    }
	    nconf, err := modules.Domains.UpdateConfig(s, objId, config)
	    if err != nil {
	        return err
	    }
	    fmt.Println(nconf.PrettyString())
	    return nil
	}) */

	type DomainCreateOptions struct {
		NAME     string `help:"Name of domain"`
		Desc     string `help:"Description"`
		Enabled  bool   `help:"Set the domain enabled"`
		Disabled bool   `help:"Set the domain disabled"`

		Displayname string `help:"display name"`
	}
	R(&DomainCreateOptions{}, "domain-create", "Create a new domain", func(s *mcclient.ClientSession, args *DomainCreateOptions) error {
		params := jsonutils.NewDict()
		params.Add(jsonutils.NewString(args.NAME), "name")
		if len(args.Desc) > 0 {
			params.Add(jsonutils.NewString(args.Desc), "description")
		}
		if args.Enabled && !args.Disabled {
			params.Add(jsonutils.JSONTrue, "enabled")
		} else if !args.Enabled && args.Disabled {
			params.Add(jsonutils.JSONFalse, "enabled")
		}
		if len(args.Displayname) > 0 {
			params.Add(jsonutils.NewString(args.Displayname), "displayname")
		}
		result, err := modules.Domains.Create(s, params)
		if err != nil {
			return err
		}
		printObject(result)
		return nil
	})

	type DomainUpdateOptions struct {
		ID       string `help:"ID of domain to update"`
		Name     string `help:"Name of domain"`
		Desc     string `help:"Description"`
		Enabled  bool   `help:"Set the domain enabled"`
		Disabled bool   `help:"Set the domain disabled"`
		Driver   string `help:"Set the domain Driver"`

		Displayname string `help:"display name"`
	}
	R(&DomainUpdateOptions{}, "domain-update", "Update a domain", func(s *mcclient.ClientSession, args *DomainUpdateOptions) error {
		obj, err := modules.Domains.Get(s, args.ID, nil)
		if err != nil {
			return err
		}
		objId, err := obj.GetString("id")
		if err != nil {
			return err
		}
		params := jsonutils.NewDict()
		if len(args.Name) > 0 {
			params.Add(jsonutils.NewString(args.Name), "name")
		}
		if len(args.Desc) > 0 {
			params.Add(jsonutils.NewString(args.Desc), "description")
		}
		if len(args.Driver) > 0 {
			params.Add(jsonutils.NewString(args.Driver), "driver")
		}

		if args.Enabled && !args.Disabled {
			params.Add(jsonutils.JSONTrue, "enabled")
		} else if !args.Enabled && args.Disabled {
			params.Add(jsonutils.JSONFalse, "enabled")
		}
		if len(args.Displayname) > 0 {
			params.Add(jsonutils.NewString(args.Displayname), "displayname")
		}
		result, err := modules.Domains.Patch(s, objId, params)
		if err != nil {
			return err
		}
		printObject(result)
		return nil
	})

	type DomainUnlinkIdpOptions struct {
		DOMAIN string `help:"ID or name of domain to operate" json:"-"`
	}
	R(&DomainUnlinkIdpOptions{}, "domain-unlink-idp", "Unlink domain from an entity in the speicified identity provider", func(s *mcclient.ClientSession, args *DomainUnlinkIdpOptions) error {
		result, err := modules.Domains.PerformAction(s, args.DOMAIN, "unlink-idp", nil)
		if err != nil {
			return err
		}
		printObject(result)
		return nil
	})

}


<|start_filename|>pkg/compute/tasks/dbinstance_recovery_task.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
	"context"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type DBInstanceRecoveryTask struct {
	taskman.STask
}

func init() {
	taskman.RegisterTask(DBInstanceRecoveryTask{})
}

func (self *DBInstanceRecoveryTask) taskFailed(ctx context.Context, instance *models.SDBInstance, err error) {
	instance.SetStatus(self.UserCred, api.DBINSTANCE_RESTORE_FAILED, err.Error())
	db.OpsLog.LogEvent(instance, db.ACT_RESTORE, err, self.GetUserCred())
	logclient.AddActionLogWithStartable(self, instance, logclient.ACT_RESTORE, err, self.UserCred, false)
	self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}

func (self *DBInstanceRecoveryTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	instance := obj.(*models.SDBInstance)

	iRds, err := instance.GetIDBInstance()
	if err != nil {
		self.taskFailed(ctx, instance, errors.Wrap(err, "instance.GetIDBInstance"))
		return
	}

	input := &api.SDBInstanceRecoveryConfigInput{}
	err = self.GetParams().Unmarshal(input)
	if err != nil {
		self.taskFailed(ctx, instance, errors.Wrap(err, "self.GetParams().Unmarshal(input)"))
		return
	}

	_backup, err := models.DBInstanceBackupManager.FetchById(input.DBInstancebackupId)
	if err != nil {
		self.taskFailed(ctx, instance, errors.Wrapf(err, "instance.GetDBInstanceBackup(%s)", input.DBInstancebackupId))
		return
	}

	backup := _backup.(*models.SDBInstanceBackup)
	origin, _ := backup.GetDBInstance()

	conf := &cloudprovider.SDBInstanceRecoveryConfig{
		BackupId:  backup.ExternalId,
		Databases: input.Databases,
	}
	if origin != nil {
		conf.OriginDBInstanceExternalId = origin.ExternalId
	}

	err = iRds.RecoveryFromBackup(conf)
	if err != nil {
		self.taskFailed(ctx, instance, errors.Wrap(err, "iRds.RecoveryFromBackup"))
		return
	}

	err = cloudprovider.WaitStatus(iRds, api.DBINSTANCE_RUNNING, time.Second*10, time.Minute*40)
	if err != nil {
		self.taskFailed(ctx, instance, errors.Wrap(err, "cloudprovider.WaitStatus(running)"))
		return
	}

	databases, err := iRds.GetIDBInstanceDatabases()
	if err != nil {
		log.Errorf("failed to get dbinstance %s database error: %v", instance.Name, err)
	} else {
		models.DBInstanceDatabaseManager.SyncDBInstanceDatabases(ctx, self.UserCred, instance, databases)
	}

	db.OpsLog.LogEvent(instance, db.ACT_RESTORE, nil, self.GetUserCred())
	logclient.AddActionLogWithStartable(self, instance, logclient.ACT_RESTORE, backup, self.UserCred, true)
	instance.SetStatus(self.UserCred, api.DBINSTANCE_RUNNING, "")
	self.SetStageComplete(ctx, nil)
}


<|start_filename|>pkg/baremetal/service/service.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"os"
	"path/filepath"
	"time"

	"yunion.io/x/log"

	api "yunion.io/x/onecloud/pkg/apis/baremetal"
	"yunion.io/x/onecloud/pkg/appsrv"
	"yunion.io/x/onecloud/pkg/baremetal"
	"yunion.io/x/onecloud/pkg/baremetal/handler"
	o "yunion.io/x/onecloud/pkg/baremetal/options"
	"yunion.io/x/onecloud/pkg/baremetal/tasks"
	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
	"yunion.io/x/onecloud/pkg/cloudcommon/service"
	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
	_ "yunion.io/x/onecloud/pkg/util/redfish/loader"
)

type BaremetalService struct {
	service.SServiceBase
}

func New() *BaremetalService {
	return &BaremetalService{}
}

func (s *BaremetalService) StartService() {
	common_options.ParseOptions(&o.Options, os.Args, "baremetal.conf", "baremetal")

	if len(o.Options.CachePath) == 0 {
		o.Options.CachePath = filepath.Join(filepath.Dir(o.Options.BaremetalsPath), "bm_image_cache")
		log.Infof("No cachepath, use default %s", o.Options.CachePath)
	}
	if len(o.Options.BootIsoPath) == 0 {
		o.Options.BootIsoPath = filepath.Join(filepath.Dir(o.Options.BaremetalsPath), "bm_boot_iso")
		log.Infof("No BootIsoPath, use default %s", o.Options.BootIsoPath)
		err := os.MkdirAll(o.Options.BootIsoPath, os.FileMode(0760))
		if err != nil {
			log.Fatalf("fail to create BootIsoPath %s", o.Options.BootIsoPath)
		}
	}

	app_common.InitAuth(&o.Options.CommonOptions, func() {
		log.Infof("auth complete")
	})

	fsdriver.Init(nil, "")
	app := app_common.InitApp(&o.Options.BaseOptions, false)

	common_options.StartOptionManager(&o.Options, o.Options.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, o.OnOptionsChange)

	handler.InitHandlers(app)

	s.startAgent(app)

	cron := cronman.InitCronJobManager(false, o.Options.CronJobWorkerCount)
	cron.AddJobAtIntervals("BaremetalCronJobs", 10*time.Second, baremetal.DoCronJobs)
	cron.Start()
	defer cron.Stop()

	app_common.ServeForeverWithCleanup(app, &o.Options.BaseOptions, func() {
		tasks.OnStop()
		baremetal.Stop()
	})
}

func (s *BaremetalService) startAgent(app *appsrv.Application) {
	// init lockman
	lm := lockman.NewInMemoryLockManager()
	lockman.Init(lm)

	err := baremetal.Start(app)
	if err != nil {
		log.Fatalf("Start agent error: %v", err)
	}
}


<|start_filename|>pkg/multicloud/qcloud/dnspod_record.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package qcloud

import (
	"strconv"
	"strings"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudprovider"
)

type SRecordCreateRet struct {
	ID     string `json:"id"`
	Name   string `json:"name"`
	Status string `json:"status"`
	Weight int    `json:"weight"`
}

type SRecordCountInfo struct {
	RecordTotal string `json:"record_total"`
	RecordsNum  string `json:"records_num"`
	SubDomains  string `json:"sub_domains"`
}

type SDnsRecord struct {
	domain     *SDomian
	ID         int    `json:"id"`
	TTL        int    `json:"ttl"`
	Value      string `json:"value"`
	Enabled    int    `json:"enabled"`
	Status     string `json:"status"`
	UpdatedOn  string `json:"updated_on"`
	QProjectID int    `json:"q_project_id"`
	Name       string `json:"name"`
	Line       string `json:"line"`
	LineID     string `json:"line_id"`
	Type       string `json:"type"`
	Remark     string `json:"remark"`
	Mx         int64  `json:"mx"`
	Hold       string `json:"hold"`
}

// https://cloud.tencent.com/document/product/302/8517
func (client *SQcloudClient) GetDnsRecords(projectId string, sDomainName string, offset int, limit int) ([]SDnsRecord, int, error) {

	params := map[string]string{}
	params["offset"] = strconv.Itoa(offset)
	params["length"] = strconv.Itoa(limit)
	params["domain"] = sDomainName
	if len(projectId) > 0 {
		params["qProjectId"] = projectId
	}
	resp, err := client.cnsRequest("RecordList", params)
	if err != nil {
		return nil, 0, errors.Wrapf(err, "client.cnsRequest(RecordList, %s)", jsonutils.Marshal(params).String())
	}
	count := SRecordCountInfo{}
	err = resp.Unmarshal(&count, "info")
	if err != nil {
		return nil, 0, errors.Wrapf(err, "%s.Unmarshal(info)", jsonutils.Marshal(resp).String())
	}
	records := []SDnsRecord{}
	err = resp.Unmarshal(&records, "records")
	if err != nil {
		return nil, 0, errors.Wrapf(err, "%s.Unmarshal(records)", jsonutils.Marshal(resp).String())
	}
	RecordTotal, err := strconv.Atoi(count.RecordTotal)
	if err != nil {
		return nil, 0, errors.Wrapf(err, "strconv.Atoi(%s)", count.RecordTotal)
	}
	return records, RecordTotal, nil
}

func (client *SQcloudClient) GetAllDnsRecords(sDomainName string) ([]SDnsRecord, error) {
	count := 0
	result := []SDnsRecord{}
	for true {
		// -1 所有项目; 0,default默认项目
		records, total, err := client.GetDnsRecords("-1", sDomainName, count, 100)
		if err != nil {
			return nil, errors.Wrapf(err, "client.GetDnsRecords(%s,%d,%d)", sDomainName, count, 100)
		}

		result = append(result, records...)
		count += len(records)
		if total <= count {
			break
		}
	}
	return result, nil
}

func GetRecordLineLineType(policyinfo cloudprovider.TDnsPolicyValue) string {
	switch policyinfo {
	case cloudprovider.DnsPolicyValueMainland:
		return "境内"
	case cloudprovider.DnsPolicyValueOversea:
		return "境外"
	case cloudprovider.DnsPolicyValueTelecom:
		return "电信"
	case cloudprovider.DnsPolicyValueUnicom:
		return "联通"
	case cloudprovider.DnsPolicyValueChinaMobile:
		return "移动"
	case cloudprovider.DnsPolicyValueCernet:
		return "教育网"

	case cloudprovider.DnsPolicyValueBaidu:
		return "百度"
	case cloudprovider.DnsPolicyValueGoogle:
		return "谷歌"
	case cloudprovider.DnsPolicyValueYoudao:
		return "有道"
	case cloudprovider.DnsPolicyValueBing:
		return "必应"
	case cloudprovider.DnsPolicyValueSousou:
		return "搜搜"
	case cloudprovider.DnsPolicyValueSougou:
		return "搜狗"
	case cloudprovider.DnsPolicyValueQihu360:
		return "奇虎"
	default:
		return "默认"
	}
}

// https://cloud.tencent.com/document/api/302/8516
func (client *SQcloudClient) CreateDnsRecord(opts *cloudprovider.DnsRecordSet, domainName string) (string, error) {
	params := map[string]string{}
	recordline := GetRecordLineLineType(opts.PolicyValue)
	if opts.Ttl < 600 {
		opts.Ttl = 600
	}
	if opts.Ttl > 604800 {
		opts.Ttl = 604800
	}
	if len(opts.DnsName) < 1 {
		opts.DnsName = "@"
	}
	params["domain"] = domainName
	params["subDomain"] = opts.DnsName
	params["recordType"] = string(opts.DnsType)
	params["ttl"] = strconv.FormatInt(opts.Ttl, 10)
	params["value"] = opts.DnsValue
	params["recordLine"] = recordline
	if opts.DnsType == cloudprovider.DnsTypeMX {
		params["mx"] = strconv.FormatInt(opts.MxPriority, 10)
	}
	resp, err := client.cnsRequest("RecordCreate", params)
	if err != nil {
		return "", errors.Wrapf(err, "client.cnsRequest(RecordCreate, %s)", jsonutils.Marshal(params).String())
	}
	SRecordCreateRet := SRecordCreateRet{}
	err = resp.Unmarshal(&SRecordCreateRet, "record")
	if err != nil {
		return "", errors.Wrapf(err, "%s.Unmarshal(records)", jsonutils.Marshal(resp).String())
	}
	return SRecordCreateRet.ID, nil
}

// https://cloud.tencent.com/document/product/302/8511
func (client *SQcloudClient) ModifyDnsRecord(opts *cloudprovider.DnsRecordSet, domainName string) error {
	params := map[string]string{}
	recordline := GetRecordLineLineType(opts.PolicyValue)
	if opts.Ttl < 600 {
		opts.Ttl = 600
	}
	if opts.Ttl > 604800 {
		opts.Ttl = 604800
	}
	subDomain := strings.TrimSuffix(opts.DnsName, "."+domainName)
	if len(subDomain) < 1 {
		subDomain = "@"
	}
	params["domain"] = domainName
	params["recordId"] = opts.ExternalId
	params["subDomain"] = subDomain
	params["recordType"] = string(opts.DnsType)
	params["ttl"] = strconv.FormatInt(opts.Ttl, 10)
	params["value"] = opts.DnsValue
	params["recordLine"] = recordline
	if opts.DnsType == cloudprovider.DnsTypeMX {
		params["mx"] = strconv.FormatInt(opts.MxPriority, 10)
	}
	_, err := client.cnsRequest("RecordModify", params)
	if err != nil {
		return errors.Wrapf(err, "client.cnsRequest(RecordModify, %s)", jsonutils.Marshal(params).String())
	}
	return nil
}

// https://cloud.tencent.com/document/product/302/8519
func (client *SQcloudClient) ModifyRecordStatus(status, recordId, domain string) error {
	params := map[string]string{}
	params["domain"] = domain
	params["recordId"] = recordId
	params["status"] = status // “disable” 和 “enable”
	_, err := client.cnsRequest("RecordStatus", params)
	if err != nil {
		return errors.Wrapf(err, "client.cnsRequest(RecordModify, %s)", jsonutils.Marshal(params).String())
	}
	return nil
}

// https://cloud.tencent.com/document/api/302/8514
func (client *SQcloudClient) DeleteDnsRecord(recordId int, domainName string) error {
	params := map[string]string{}
	params["domain"] = domainName
	params["recordId"] = strconv.Itoa(recordId)
	_, err := client.cnsRequest("RecordDelete", params)
	if err != nil {
		return errors.Wrapf(err, "client.cnsRequest(RecordDelete, %s)", jsonutils.Marshal(params).String())
	}
	return nil
}

func (self *SDnsRecord) GetGlobalId() string {
	return strconv.Itoa(self.ID)
}

func (self *SDnsRecord) GetDnsName() string {
	return self.Name
}

func (self *SDnsRecord) GetStatus() string {
	if self.Status != "spam" {
		return api.DNS_RECORDSET_STATUS_AVAILABLE
	}
	return api.DNS_ZONE_STATUS_UNKNOWN
}

func (self *SDnsRecord) GetEnabled() bool {
	return self.Enabled == 1
}

func (self *SDnsRecord) GetDnsType() cloudprovider.TDnsType {
	return cloudprovider.TDnsType(self.Type)
}

func (self *SDnsRecord) GetDnsValue() string {
	if self.GetDnsType() == cloudprovider.DnsTypeMX || self.GetDnsType() == cloudprovider.DnsTypeCNAME || self.GetDnsType() == cloudprovider.DnsTypeSRV {
		return self.Value[:len(self.Value)-1]
	}
	return self.Value
}

func (self *SDnsRecord) GetTTL() int64 {
	return int64(self.TTL)
}

func (self *SDnsRecord) GetMxPriority() int64 {
	if self.GetDnsType() == cloudprovider.DnsTypeMX {
		return self.Mx
	}
	return 0
}

func (self *SDnsRecord) GetPolicyType() cloudprovider.TDnsPolicyType {
	switch self.Line {
	case "境内", "境外":
		return cloudprovider.DnsPolicyTypeByGeoLocation
	case "电信", "联通", "移动", "教育网":
		return cloudprovider.DnsPolicyTypeByCarrier
	case "百度", "谷歌", "有道", "必应", "搜搜", "搜狗", "奇虎":
		return cloudprovider.DnsPolicyTypeBySearchEngine
	default:
		return cloudprovider.DnsPolicyTypeSimple
	}
}

func (self *SDnsRecord) GetPolicyOptions() *jsonutils.JSONDict {
	return nil
}

func (self *SDnsRecord) GetPolicyValue() cloudprovider.TDnsPolicyValue {
	switch self.Line {
	case "境内":
		return cloudprovider.DnsPolicyValueMainland
	case "境外":
		return cloudprovider.DnsPolicyValueOversea

	case "电信":
		return cloudprovider.DnsPolicyValueTelecom
	case "联通":
		return cloudprovider.DnsPolicyValueUnicom
	case "移动":
		return cloudprovider.DnsPolicyValueChinaMobile
	case "教育网":
		return cloudprovider.DnsPolicyValueCernet

	case "百度":
		return cloudprovider.DnsPolicyValueBaidu
	case "谷歌":
		return cloudprovider.DnsPolicyValueGoogle
	case "有道":
		return cloudprovider.DnsPolicyValueYoudao
	case "必应":
		return cloudprovider.DnsPolicyValueBing
	case "搜搜":
		return cloudprovider.DnsPolicyValueSousou
	case "搜狗":
		return cloudprovider.DnsPolicyValueSougou
	case "奇虎":
		return cloudprovider.DnsPolicyValueQihu360
	default:
		return cloudprovider.DnsPolicyValueEmpty
	}
}


<|start_filename|>pkg/multicloud/aliyun/cdn.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aliyun

import (
	"time"

	"github.com/pkg/errors"
)

type SCdnDomainNames struct {
	DomainNames []string `json:"domainNames"`
}
type SDomainInfo struct {
	DomainCname string    `json:"DomainCname"`
	Status      string    `json:"Status"`
	CreateTime  time.Time `json:"CreateTime"`
	UpdateTime  time.Time `json:"UpdateTime"`
	DomainName  string    `json:"DomainName"`
}
type SCdnDomainInfos struct {
	DomainInfo []SDomainInfo `json:"domainInfo"`
}
type SCdnDomainsData struct {
	Source      string          `json:"Source"`
	Domains     SCdnDomainNames `json:"Domains"`
	DomainInfos SCdnDomainInfos `json:"DomainInfos"`
}
type SCdnDomainsList struct {
	DomainsData []SCdnDomainsData `json:"DomainsData"`
}

type SCdnSource struct {
	Port     int    `json:"Port"`
	Weight   string `json:"Weight"`
	Type     string `json:"Type"`
	Content  string `json:"Content"`
	Priority string `json:"Priority"`
}
type SCdnSources struct {
	Source []SCdnSource `json:"Source"`
}
type SCdnPageData struct {
	Cname           string     `json:"Cname"`
	Description     string     `json:"Description"`
	CdnType         string     `json:"CdnType"`
	ResourceGroupID string     `json:"ResourceGroupId"`
	DomainStatus    string     `json:"DomainStatus"`
	SslProtocol     string     `json:"SslProtocol"`
	DomainName      string     `json:"DomainName"`
	Coverage        string     `json:"Coverage"`
	Sources         SCdnSource `json:"Sources"`
	GmtModified     string     `json:"GmtModified"`
	Sandbox         string     `json:"Sandbox"`
	GmtCreated      time.Time  `json:"GmtCreated"`
}
type SCdnDomains struct {
	PageData []SCdnPageData `json:"PageData"`
}

func (client *SAliyunClient) DescribeDomainsBySource(origin string) (SCdnDomainsList, error) {
	sproducts := SCdnDomainsList{}
	params := map[string]string{}
	params["Action"] = "DescribeDomainsBySource"
	params["Sources"] = origin
	resp, err := client.cdnRequest("DescribeDomainsBySource", params)
	if err != nil {
		return sproducts, errors.Wrap(err, "DescribeDomainsBySource")
	}
	err = resp.Unmarshal(&sproducts, "DomainsList")
	if err != nil {
		return sproducts, errors.Wrap(err, "resp.Unmarshal")
	}
	return sproducts, nil
}

func (client *SAliyunClient) DescribeUserDomains(domain string) (SCdnDomains, error) {
	sproducts := SCdnDomains{}
	params := map[string]string{}
	params["Action"] = "DescribeUserDomains"
	params["DomainName"] = domain
	params["DomainSearchType"] = "full_match"
	resp, err := client.cdnRequest("DescribeUserDomains", params)
	if err != nil {
		return sproducts, errors.Wrap(err, "DescribeUserDomains")
	}
	err = resp.Unmarshal(&sproducts, "Domains")
	if err != nil {
		return sproducts, errors.Wrap(err, "resp.Unmarshal")
	}
	return sproducts, nil
}


<|start_filename|>pkg/cloudcommon/db/rolecache.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package db

import (
	"context"
	"database/sql"
	"fmt"
	"runtime/debug"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/sqlchemy"

	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
	"yunion.io/x/onecloud/pkg/mcclient/auth"
	"yunion.io/x/onecloud/pkg/mcclient/informer"
	"yunion.io/x/onecloud/pkg/mcclient/modules"
	"yunion.io/x/onecloud/pkg/util/httputils"
	"yunion.io/x/onecloud/pkg/util/stringutils2"
)

// TODO: 1. Access to a non-existent role will still trigger synchronization.
//       2. The failure of the watch mechanism makes the cache miss an event,
//          then there is outdated data in the cache before the expiration
//          time comes or the event about this data occurs again.

var (
	DefaultRoleFetcher func(ctx context.Context, id string) (*SRole, error)
)

type SRoleCacheManager struct {
	SKeystoneCacheObjectManager
	watching bool
}

type SRole struct {
	SKeystoneCacheObject
}

func (role *SRole) GetModelManager() IModelManager {
	return RoleCacheManager
}

var RoleCacheManager *SRoleCacheManager

func init() {
	RoleCacheManager = &SRoleCacheManager{
		NewKeystoneCacheObjectManager(SRole{}, "roles_cache_tbl", "role", "roles"), false}
	// log.Debugf("initialize role cache manager %s", RoleCacheManager.KeywordPlural())
	RoleCacheManager.SetVirtualObject(RoleCacheManager)

	DefaultRoleFetcher = RoleCacheManager.FetchRoleByIdOrName
}

func (manager *SRoleCacheManager) FetchRoleByIdOrName(ctx context.Context, idStr string) (*SRole, error) {
	return manager.fetchRole(ctx, idStr, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
		if stringutils2.IsUtf8(idStr) {
			return q.Equals("name", idStr)
		} else {
			return q.Filter(sqlchemy.OR(
				sqlchemy.Equals(q.Field("id"), idStr),
				sqlchemy.Equals(q.Field("name"), idStr),
			))
		}
	})
}

func (manager *SRoleCacheManager) FetchRoleById(ctx context.Context, idStr string) (*SRole, error) {
	return manager.fetchRole(ctx, idStr, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
		return q.Filter(sqlchemy.Equals(q.Field("id"), idStr))
	})
}

func (manager *SRoleCacheManager) FetchRoleByName(ctx context.Context, idStr string) (*SRole, error) {
	return manager.fetchRole(ctx, idStr, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
		return q.Filter(sqlchemy.Equals(q.Field("name"), idStr))
	})
}

func (manager *SRoleCacheManager) fetchRole(ctx context.Context, idStr string, noExpireCheck bool, filter func(*sqlchemy.SQuery) *sqlchemy.SQuery) (*SRole, error) {
	q := manager.Query()
	q = filter(q)
	tobj, err := NewModelObject(manager)
	if err != nil {
		return nil, errors.Wrap(err, "NewModelObject")
	}
	err = q.First(tobj)
	if err != nil && err != sql.ErrNoRows {
		return nil, errors.Wrap(err, "query")
	} else if tobj != nil {
		role := tobj.(*SRole)
		if noExpireCheck || !role.IsExpired() {
			return role, nil
		}
	}
	return manager.FetchRoleFromKeystone(ctx, idStr)
}

func (manager *SRoleCacheManager) FetchRoleFromKeystone(ctx context.Context, idStr string) (*SRole, error) {
	if len(idStr) == 0 {
		log.Debugf("fetch empty role!!!!\n%s", debug.Stack())
		return nil, fmt.Errorf("Empty idStr")
	}

	// It's to query the full list of roles(contains other domain's ones and system ones)
	query := jsonutils.NewDict()
	query.Set("scope", jsonutils.NewString("system"))
	query.Set("system", jsonutils.JSONTrue)

	s := auth.GetAdminSession(ctx, consts.GetRegion(), "")
	role, err := modules.RolesV3.GetById(s, idStr, query)
	if err != nil {
		if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 {
			role, err = modules.RolesV3.GetByName(s, idStr, query)
			if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 {
				return nil, sql.ErrNoRows
			}
		}
		if err != nil {
			log.Errorf("fetch role %s fail %s", idStr, err)
			return nil, errors.Wrap(err, "modules.RolesV3.Get")
		}
	}
	id, _ := role.GetString("id")
	name, _ := role.GetString("name")
	domainId, _ := role.GetString("domain_id")
	domainName, _ := role.GetString("project_domain")
	return manager.Save(ctx, id, name, domainId, domainName)
}

func (manager *SRoleCacheManager) Save(ctx context.Context, idStr string, name string, domainId string, domain string) (*SRole, error) {
	lockman.LockRawObject(ctx, manager.KeywordPlural(), idStr)
	defer lockman.ReleaseRawObject(ctx, manager.KeywordPlural(), idStr)

	objo, err := manager.FetchById(idStr)
	if err != nil && err != sql.ErrNoRows {
		log.Errorf("FetchRolebyId fail %s", err)
		return nil, err
	}
	if err == nil {
		obj := objo.(*SRole)
		_, err = Update(obj, func() error {
			obj.Id = idStr
			obj.Name = name
			obj.Domain = domain
			obj.DomainId = domainId
			obj.LastCheck = time.Now().UTC()
			return nil
		})
		if err != nil {
			return nil, err
		} else {
			return obj, nil
		}
	} else {
		objm, err := NewModelObject(manager)
		obj := objm.(*SRole)
		obj.Id = idStr
		obj.Name = name
		obj.Domain = domain
		obj.DomainId = domainId
		obj.LastCheck = time.Now().UTC()
		err = manager.TableSpec().InsertOrUpdate(ctx, obj)
		if err != nil {
			return nil, err
		} else {
			return obj, nil
		}
	}
}

func (manager *SRoleCacheManager) OnAdd(obj *jsonutils.JSONDict) {
	id, err := obj.GetString("id")
	if err != nil {
		log.Errorf("unable to get Id: %v", err)
		return
	}
	ctx := context.Background()
	lockman.LockRawObject(ctx, manager.KeywordPlural(), id)
	defer lockman.ReleaseRawObject(ctx, manager.KeywordPlural(), id)
	role := new(SRole)
	role.Id = id
	role.Name, _ = obj.GetString("name")
	role.DomainId, _ = obj.GetString("domain_id")
	role.LastCheck = time.Now()
	err = manager.TableSpec().Insert(ctx, role)
	if err != nil {
		log.Errorf("unable to insert: %v", err)
	}
}

func (manager *SRoleCacheManager) OnUpdate(oldObj, newObj *jsonutils.JSONDict) {
	id, err := oldObj.GetString("id")
	if err != nil {
		log.Errorf("unable to get Id: %v", err)
		return
	}
	ctx := context.Background()
	role, err := manager.fetchRole(ctx, id, true, nil)
	if err != nil {
		log.Errorf("unable to fetch Role from db: %v", err)
		return
	}
	name, _ := newObj.GetString("name")
	domainId, _ := newObj.GetString("domain_id")
	if role.Name == name && role.DomainId == domainId {
		return
	}
	lockman.LockRawObject(ctx, manager.KeywordPlural(), id)
	defer lockman.ReleaseRawObject(ctx, manager.KeywordPlural(), id)
	_, err = Update(role, func() error {
		role.Name = name
		role.DomainId = domainId
		role.LastCheck = time.Now()
		return nil
	})
	if err != nil {
		log.Errorf("unable to update: %v", err)
	}
}

func (manager *SRoleCacheManager) OnDelete(obj *jsonutils.JSONDict) {
	id, err := obj.GetString("id")
	if err != nil {
		log.Errorf("unable to get Id: %v", err)
		return
	}
	ctx := context.Background()
	role, err := manager.fetchRole(ctx, id, true, nil)
	if err != nil {
		log.Errorf("unable to fetch Role from db: %v", err)
		return
	}
	lockman.LockRawObject(ctx, manager.KeywordPlural(), id)
	defer lockman.ReleaseRawObject(ctx, manager.KeywordPlural(), id)
	_, err = Update(role, func() error {
		role.Deleted = false
		return nil
	})
	if err != nil {
		log.Errorf("unable to delete role: %v", err)
	}
}

func (manager *SRoleCacheManager) StartWatchRoleInKeystone() error {
	if manager.watching {
		return nil
	}
	ctx := context.Background()
	s := auth.GetAdminSession(ctx, "", "")
	watchMan, err := informer.NewWatchManagerBySession(s)
	if err != nil {
		return err
	}
	resMan := &modules.RolesV3
	return watchMan.For(resMan).AddEventHandler(ctx, manager)
}

func (r *SRole) IsExpired() bool {
	if r.LastCheck.IsZero() {
		return true
	}
	now := time.Now().UTC()
	if r.LastCheck.Add(consts.GetRoleCacheExpireHours()).Before(now) {
		return true
	}
	return false
}


<|start_filename|>vendor/github.com/tencentyun/cos-go-sdk-v5/ci.go<|end_filename|>
package cos

import (
	"encoding/json"
)

type PicOperations struct {
	IsPicInfo int                  `json:"is_pic_info,omitempty"`
	Rules     []PicOperationsRules `json:"rules,omitemtpy"`
}

type PicOperationsRules struct {
	Bucket string `json:"bucket,omitempty"`
	FileId string `json:"fileid"`
	Rule   string `json:"rule"`
}

func EncodePicOperations(pic *PicOperations) string {
	bs, err := json.Marshal(pic)
	if err != nil {
		return ""
	}
	return string(bs)
}


<|start_filename|>pkg/scheduler/algorithm/predicates/domain_predicate.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package predicates

import (
	"yunion.io/x/pkg/utils"

	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/scheduler/core"
	"yunion.io/x/onecloud/pkg/util/rbacutils"
)

type DomainPredicate struct {
	BasePredicate
}

func (p *DomainPredicate) Name() string {
	return "host_domain"
}

func (p *DomainPredicate) Clone() core.FitPredicate {
	return &DomainPredicate{}
}

func (p *DomainPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
	h := NewPredicateHelper(p, u, c)
	getter := c.Getter()
	if getter.DomainId() == u.SchedInfo.Domain {
	} else if getter.IsPublic() && getter.PublicScope() == string(rbacutils.ScopeSystem) {
	} else if getter.IsPublic() && getter.PublicScope() == string(rbacutils.ScopeDomain) && utils.IsInStringArray(u.SchedInfo.Domain, getter.SharedDomains()) {
	} else if db.IsAdminAllowGet(u.SchedInfo.UserCred, getter) {
	} else {
		h.Exclude("domain_ownership")
	}
	return h.GetResult()
}


<|start_filename|>pkg/mcclient/options/yunionagent/notices.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package yunionagent

import (
	"yunion.io/x/jsonutils"

	"yunion.io/x/onecloud/pkg/mcclient/options"
)

type NoticeListOptions struct {
	options.BaseListOptions

	VisibleScope string `help:"visiable scope" choices:"system|domain"`
}

func (n *NoticeListOptions) Params() (jsonutils.JSONObject, error) {
	return options.ListStructToParams(n)
}

type NoticeCreateOptions struct {
	TITLE   string `help:"The notice title" json:"title"`
	CONTENT string `help:"The notice content" json:"content"`
}

func (n *NoticeCreateOptions) Params() (jsonutils.JSONObject, error) {
	return jsonutils.Marshal(n), nil
}

type NoticeOptions struct {
	ID string `help:"ID of notice to update"`
}

func (n *NoticeOptions) GetId() string {
	return n.ID
}

func (n *NoticeOptions) Params() (jsonutils.JSONObject, error) {
	return nil, nil
}

type SnoticeUpdateOptions struct {
	Title   string `help:"The notice title"`
	Content string `help:"The notice content"`
}

type NoticesUpdateOptions struct {
	NoticeOptions
	SnoticeUpdateOptions
}

func (n *NoticesUpdateOptions) Params() (jsonutils.JSONObject, error) {
	return jsonutils.Marshal(n.SnoticeUpdateOptions), nil
}


<|start_filename|>pkg/compute/tasks/dbinstance_database_create_task.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
	"context"
	"fmt"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type DBInstanceDatabaseCreateTask struct {
	taskman.STask
}

func init() {
	taskman.RegisterTask(DBInstanceDatabaseCreateTask{})
}

func (self *DBInstanceDatabaseCreateTask) taskFailed(ctx context.Context, database *models.SDBInstanceDatabase, err error) {
	database.SetStatus(self.UserCred, api.DBINSTANCE_DATABASE_CREATE_FAILE, err.Error())
	db.OpsLog.LogEvent(database, db.ACT_CREATE, err, self.GetUserCred())
	logclient.AddActionLogWithStartable(self, database, logclient.ACT_CREATE, err, self.UserCred, false)
	self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}

func (self *DBInstanceDatabaseCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	database := obj.(*models.SDBInstanceDatabase)
	self.CreateDBInstanceDatabase(ctx, database)
}

func (self *DBInstanceDatabaseCreateTask) CreateDBInstanceDatabase(ctx context.Context, database *models.SDBInstanceDatabase) {
	instance, err := database.GetDBInstance()
	if err != nil {
		self.taskFailed(ctx, database, errors.Wrap(err, "database.GetDBInstance"))
		return
	}

	iRds, err := instance.GetIDBInstance()
	if err != nil {
		self.taskFailed(ctx, database, errors.Wrap(err, "instance.GetIDBInstance"))
		return
	}

	desc := &cloudprovider.SDBInstanceDatabaseCreateConfig{
		CharacterSet: database.CharacterSet,
		Name:         database.Name,
		Description:  database.Description,
	}

	err = iRds.CreateDatabase(desc)
	if err != nil {
		self.taskFailed(ctx, database, errors.Wrap(err, "iRds.CreateDatabase"))
		return
	}

	iDatabases, err := iRds.GetIDBInstanceDatabases()
	if err != nil {
		msg := fmt.Sprintf("failed to found databases from cloud dbinstance error: %v", err)
		db.OpsLog.LogEvent(database, db.ACT_CREATE, msg, self.GetUserCred())
		logclient.AddActionLogWithStartable(self, database, logclient.ACT_CREATE, msg, self.UserCred, false)
		database.SetStatus(self.UserCred, api.DBINSTANCE_UNKNOWN, "")
		self.SetStageComplete(ctx, nil)
		return
	}

	var iDatabase cloudprovider.ICloudDBInstanceDatabase = nil
	for i := range iDatabases {
		if iDatabases[i].GetName() == database.Name {
			iDatabase = iDatabases[i]
			break
		}
	}

	if iDatabase == nil {
		msg := fmt.Sprintf("failed to found database %s from cloud dbinstance", database.Name)
		db.OpsLog.LogEvent(database, db.ACT_CREATE, msg, self.GetUserCred())
		logclient.AddActionLogWithStartable(self, database, logclient.ACT_CREATE, msg, self.UserCred, false)
		database.SetStatus(self.UserCred, api.DBINSTANCE_UNKNOWN, "")
		self.SetStageComplete(ctx, nil)
		return
	}

	db.SetExternalId(database, self.UserCred, iDatabase.GetGlobalId())

	input := api.DBInstanceDatabaseCreateInput{}
	self.GetParams().Unmarshal(&input)
	if len(input.Accounts) == 0 {
		database.SetStatus(self.UserCred, api.DBINSTANCE_DATABASE_RUNNING, "")
		self.SetStageComplete(ctx, nil)
		return
	}

	_iAccounts, err := iRds.GetIDBInstanceAccounts()
	if err != nil {
		msg := fmt.Sprintf("failed to found accounts from cloud dbinstance error: %v", err)
		db.OpsLog.LogEvent(database, db.ACT_GRANT_PRIVILEGE, msg, self.GetUserCred())
		logclient.AddActionLogWithStartable(self, database, logclient.ACT_GRANT_PRIVILEGE, msg, self.UserCred, false)
		database.SetStatus(self.UserCred, api.DBINSTANCE_DATABASE_RUNNING, "")
		self.SetStageComplete(ctx, nil)
		return
	}

	database.SetStatus(self.UserCred, api.DBINSTANCE_DATABASE_GRANT_PRIVILEGE, "")
	iAccounts := map[string]cloudprovider.ICloudDBInstanceAccount{}
	for _, iAccount := range _iAccounts {
		iAccounts[iAccount.GetName()] = iAccount
	}

	for _, account := range input.Accounts {
		iAccount, ok := iAccounts[account.Account]
		if !ok {
			msg := fmt.Sprintf("failed to found account %s from dbinstance", account.Account)
			db.OpsLog.LogEvent(database, db.ACT_GRANT_PRIVILEGE, msg, self.GetUserCred())
			logclient.AddActionLogWithStartable(self, database, logclient.ACT_GRANT_PRIVILEGE, msg, self.UserCred, false)
			continue
		}
		err = iAccount.GrantPrivilege(database.Name, account.Privilege)
		if err != nil {
			db.OpsLog.LogEvent(database, db.ACT_GRANT_PRIVILEGE, err, self.GetUserCred())
			logclient.AddActionLogWithStartable(self, database, logclient.ACT_GRANT_PRIVILEGE, err, self.UserCred, false)
			continue
		}
		privilege := models.SDBInstancePrivilege{
			Privilege:            account.Privilege,
			DBInstanceaccountId:  account.DBInstanceaccountId,
			DBInstancedatabaseId: database.Id,
		}
		models.DBInstancePrivilegeManager.TableSpec().Insert(ctx, &privilege)
		logclient.AddActionLogWithStartable(self, database, logclient.ACT_GRANT_PRIVILEGE, account, self.UserCred, true)
	}
	database.SetStatus(self.UserCred, api.DBINSTANCE_DATABASE_RUNNING, "")
	self.SetStageComplete(ctx, nil)
}


<|start_filename|>pkg/scheduler/manager/task_queue.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package manager

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	schedapi "yunion.io/x/onecloud/pkg/apis/scheduler"
	computemodels "yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/scheduler/api"
	"yunion.io/x/onecloud/pkg/scheduler/core"
	schedmodels "yunion.io/x/onecloud/pkg/scheduler/models"
)

const (
	TaskExecutorStatusWaiting string = "waiting"
	TaskExecutorStatusRunning string = "running"
	TaskExecutorStatusFailed  string = "failed"
	TaskExecutorStatusKilled  string = "killed"
	TaskExecutorStatusSuccess string = "success"
)

type TaskExecuteCallback func(task *TaskExecutor)

type TaskExecutor struct {
	Tag       string
	Status    string
	Time      time.Time
	Consuming time.Duration
	scheduler Scheduler
	callback  TaskExecuteCallback
	unit      *core.Unit

	resultItems *core.ScheduleResult
	resultError error
	logs        []string
	capacityMap interface{}
	completed   bool
}

func NewTaskExecutor(scheduler Scheduler, taskExecuteCallback TaskExecuteCallback) *TaskExecutor {
	return &TaskExecutor{
		Tag:       scheduler.SchedData().Tag,
		Status:    TaskExecutorStatusWaiting,
		Time:      time.Now(),
		scheduler: scheduler,
		callback:  taskExecuteCallback,
		completed: false,
	}
}

func (te *TaskExecutor) Execute() {
	te.Status = TaskExecutorStatusRunning
	te.resultItems, te.resultError = te.execute()
	te.completed = true

	if te.resultError != nil {
		te.Status = TaskExecutorStatusFailed
	} else {
		te.Status = TaskExecutorStatusSuccess
	}

	if te.callback != nil {
		te.callback(te)
	}
}

func (te *TaskExecutor) execute() (*core.ScheduleResult, error) {
	scheduler := te.scheduler
	genericScheduler, err := core.NewGenericScheduler(scheduler.(core.Scheduler))
	if err != nil {
		return nil, err
	}

	// Get current resources from DB.
	candidates, err := scheduler.Candidates()
	if err != nil {
		return nil, err
	}

	te.unit = scheduler.Unit()
	schedInfo := te.unit.SchedInfo
	helper := GenerateResultHelper(schedInfo)
	result, err := genericScheduler.Schedule(te.unit, candidates, helper)
	if err != nil {
		return nil, errors.Wrap(err, "genericScheduler.Schedule")
	}
	if schedInfo.IsSuggestion {
		return result, nil
	}
	driver := te.unit.GetHypervisorDriver()
	if err := setSchedPendingUsage(driver, schedInfo, result.Result); err != nil {
		return nil, errors.Wrap(err, "setSchedPendingUsage")
	}
	return result, nil
}

func GenerateResultHelper(schedInfo *api.SchedInfo) core.IResultHelper {
	if !schedInfo.IsSuggestion {
		return core.SResultHelperFunc(core.ResultHelp)
	}
	if schedInfo.ShowSuggestionDetails && schedInfo.SuggestionAll {
		return core.SResultHelperFunc(core.ResultHelpForForcast)
	}
	return core.SResultHelperFunc(core.ResultHelpForTest)
}

func setSchedPendingUsage(driver computemodels.IGuestDriver, req *api.SchedInfo, resp *schedapi.ScheduleOutput) error {
	if req.IsSuggestion || IsDriverSkipScheduleDirtyMark(driver) || req.SkipDirtyMarkHost() {
		return nil
	}
	for _, item := range resp.Candidates {
		schedmodels.HostPendingUsageManager.AddPendingUsage(req, item)
	}
	return nil
}

func IsDriverSkipScheduleDirtyMark(driver computemodels.IGuestDriver) bool {
	return !(driver.DoScheduleCPUFilter() && driver.DoScheduleMemoryFilter() && driver.DoScheduleStorageFilter())
}

func (te *TaskExecutor) cleanup() {
	te.unit = nil
	te.scheduler = nil
	te.callback = nil
}

func (te *TaskExecutor) Kill() {
	if te.Status == TaskExecutorStatusWaiting {
		te.Status = TaskExecutorStatusKilled
	}
}

func (te *TaskExecutor) GetResult() (*core.ScheduleResult, error) {
	return te.resultItems, te.resultError
}

func (te *TaskExecutor) GetLogs() []string {
	return te.logs
}

func (te *TaskExecutor) GetCapacityMap() interface{} {
	return te.capacityMap
}

type TaskExecutorQueue struct {
	schedType string
	queue     chan *TaskExecutor
	running   bool
}

func (teq *TaskExecutorQueue) AddTaskExecutor(scheduler Scheduler,
	callback TaskExecuteCallback) *TaskExecutor {
	taskExecutor := NewTaskExecutor(scheduler, callback)
	teq.queue <- taskExecutor
	return taskExecutor
}

func NewTaskExecutorQueue(schedType string, stopCh <-chan struct{}) *TaskExecutorQueue {
	teq := &TaskExecutorQueue{
		schedType: schedType,
		running:   false,
	}
	teq.Start(stopCh)
	return teq
}

func (teq *TaskExecutorQueue) Start(stopCh <-chan struct{}) {
	if teq.running {
		return
	}
	teq.running = true
	teq.queue = make(chan *TaskExecutor, 5000)

	go func() {
		defer close(teq.queue)

		var taskExecutor *TaskExecutor
		for taskExecutor = <-teq.queue; teq.running; taskExecutor = <-teq.queue {
			if taskExecutor.Status == TaskExecutorStatusWaiting {
				taskExecutor.Execute()
			}
		}
	}()

	go func() {
		<-stopCh
		teq.running = false
		teq.queue <- nil
	}()
}

type TaskExecutorQueueManager struct {
	taskExecutorMap map[string]*TaskExecutorQueue
	lock            sync.Mutex
	stopCh          <-chan struct{}
}

func NewTaskExecutorQueueManager(stopCh <-chan struct{}) *TaskExecutorQueueManager {
	return &TaskExecutorQueueManager{
		taskExecutorMap: make(map[string]*TaskExecutorQueue),
		lock:            sync.Mutex{},
		stopCh:          stopCh,
	}
}

func (teqm *TaskExecutorQueueManager) GetQueue(schedType string) *TaskExecutorQueue {
	teqm.lock.Lock()
	defer teqm.lock.Unlock()

	var (
		key               string
		taskExecutorQueue *TaskExecutorQueue
		ok                bool
	)

	if taskExecutorQueue, ok = teqm.taskExecutorMap[key]; !ok {
		taskExecutorQueue = NewTaskExecutorQueue(schedType, teqm.stopCh)
		teqm.taskExecutorMap[key] = taskExecutorQueue
	}

	return taskExecutorQueue
}

func (teqm *TaskExecutorQueueManager) AddTaskExecutor(
	scheduler Scheduler, callback TaskExecuteCallback) *TaskExecutor {
	schedData := scheduler.SchedData()
	taskQueue := teqm.GetQueue(schedData.Hypervisor)
	return taskQueue.AddTaskExecutor(scheduler, callback)
}

type TaskManager struct {
	taskExecutorQueueManager *TaskExecutorQueueManager
	stopCh                   <-chan struct{}
	lock                     sync.Mutex
}

func NewTaskManager(stopCh <-chan struct{}) *TaskManager {
	return &TaskManager{
		taskExecutorQueueManager: NewTaskExecutorQueueManager(stopCh),
		stopCh:                   stopCh,
		lock:                     sync.Mutex{},
	}
}

func (tm *TaskManager) Run() {
	// Do nothing
}

// AddTask provides an interface to increase the scheduling task,
// it will be a scheduling request by the host specification type
// split into multiple scheduling tasks, added to the scheduling
// task manager.
func (tm *TaskManager) AddTask(schedulerManager *SchedulerManager, schedInfo *api.SchedInfo) (*Task, error) {
	var (
		scheduler Scheduler
		err       error
	)

	task := NewTask(schedulerManager, schedInfo)
	// Split into multiple scheduling tasks by host specification type.
	if schedInfo.Hypervisor == api.SchedTypeBaremetal {
		scheduler, err = newBaremetalScheduler(schedulerManager, schedInfo)
	} else {
		scheduler, err = newGuestScheduler(schedulerManager, schedInfo)
	}
	if err != nil {
		return nil, err
	}

	taskExecutorCallback := func(taskExecutor *TaskExecutor) {
		taskExecutor.Consuming = time.Since(taskExecutor.Time)
		task.onTaskCompleted(taskExecutor)
	}

	tm.lock.Lock()
	defer tm.lock.Unlock()

	taskExecutor := tm.taskExecutorQueueManager.AddTaskExecutor(scheduler, taskExecutorCallback)
	task.taskExecutors = append(task.taskExecutors, taskExecutor)

	return task, nil
}

type Task struct {
	Time          time.Time
	SchedInfo     *api.SchedInfo
	Consuming     time.Duration
	taskExecutors []*TaskExecutor
	manager       *SchedulerManager
	lock          sync.Mutex
	waitCh        chan struct{}

	completedCount int
	resultItems    *core.ScheduleResult
	resultError    error
}

func NewTask(manager *SchedulerManager, schedInfo *api.SchedInfo) *Task {
	return &Task{
		Time:          time.Now(),
		SchedInfo:     schedInfo,
		manager:       manager,
		taskExecutors: []*TaskExecutor{},
		lock:          sync.Mutex{},
		waitCh:        make(chan struct{}),
		resultError:   nil,
	}
}

func (t *Task) GetTaskExecutor(tag string) *TaskExecutor {
	for _, executor := range t.taskExecutors {
		if executor.Tag == tag {
			return executor
		}
	}

	return nil
}

func (t *Task) GetSessionID() string {
	return t.SchedInfo.SessionId
}

func (t *Task) GetStatus() string {
	statusMap := make(map[string]int)
	for _, executor := range t.taskExecutors {
		if count, ok := statusMap[executor.Status]; ok {
			statusMap[executor.Status] = count + 1
		} else {
			statusMap[executor.Status] = 1
		}
	}

	ss := []string{}
	for status, count := range statusMap {
		ss = append(ss, fmt.Sprintf("%v %v", count, status))
	}

	return strings.Join(ss, ", ")
}

func (t *Task) onTaskCompleted(taskExecutor *TaskExecutor) {
	t.lock.Lock()
	defer t.lock.Unlock()

	log.V(10).Infof("onTaskCompleted executor: %#v", taskExecutor)
	if taskExecutor.resultError != nil {
		t.resultError = taskExecutor.resultError
		t.onError()
	} else {
		t.resultItems = taskExecutor.resultItems
		t.completedCount += 1
		if t.completedCount >= len(t.taskExecutors) {
			t.onCompleted()
		}
	}

	go func() {
		t.readLog(taskExecutor)
		taskExecutor.cleanup()
	}()
}

func (t *Task) readLog(taskExecutor *TaskExecutor) {
	u := taskExecutor.unit
	if u != nil {
		logs := u.LogManager.Read()
		taskExecutor.logs = logs
		taskExecutor.capacityMap = u.CapacityMap
	}
}

func (t *Task) onError() {
	for _, taskExecutor := range t.taskExecutors {
		taskExecutor.Kill()
	}

	log.Errorf("Remove Session on error: %v", t.SchedInfo.SessionId)

	close(t.waitCh)
}

func (t *Task) onCompleted() {
	t.Consuming = time.Since(t.Time)
	close(t.waitCh)
}

func (t *Task) Wait() (*core.ScheduleResult, error) {
	log.V(10).Infof("Task wait...")
	<-t.waitCh
	return t.GetResult()
}

func (t *Task) GetResult() (*core.ScheduleResult, error) {
	return t.resultItems, t.resultError
}


<|start_filename|>pkg/compute/regiondrivers/esxi.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package regiondrivers

import (
	"context"
	"fmt"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/mcclient"
)

type SEsxiRegionDriver struct {
	SManagedVirtualizationRegionDriver
}

func init() {
	driver := SEsxiRegionDriver{}
	models.RegisterRegionDriver(&driver)
}

func (self *SEsxiRegionDriver) GetProvider() string {
	return api.CLOUD_PROVIDER_VMWARE
}

func (self *SEsxiRegionDriver) ValidateCreateLoadbalancerData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
	return nil, httperrors.NewUnsupportOperationError("%s does not support creating loadbalancer", self.GetProvider())
}

func (self *SEsxiRegionDriver) ValidateCreateLoadbalancerAclData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
	return nil, httperrors.NewNotImplementedError("%s does not support creating loadbalancer acl", self.GetProvider())
}

func (self *SEsxiRegionDriver) ValidateCreateLoadbalancerCertificateData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
	return nil, httperrors.NewNotImplementedError("%s does not support creating loadbalancer certificate", self.GetProvider())
}

func (self *SEsxiRegionDriver) ValidateCreateSnapshotData(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, storage *models.SStorage, input *api.SnapshotCreateInput) error {
	return fmt.Errorf("%s does not support creating snapshot", self.GetProvider())
}

func (self *SEsxiRegionDriver) RequestCreateInstanceSnapshot(ctx context.Context, guest *models.SGuest, isp *models.SInstanceSnapshot, task taskman.ITask, params *jsonutils.JSONDict) error {
	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
		ivm, err := guest.GetIVM()
		if err != nil {
			return nil, errors.Wrap(err, "unable to GetIVM")
		}
		cloudSP, err := ivm.CreateInstanceSnapshot(ctx, isp.GetName(), isp.Description)
		if err != nil {
			return nil, errors.Wrap(err, "unable to CreateInstanceSnapshot")
		}
		_, err = db.Update(isp, func() error {
			isp.SetExternalId(cloudSP.GetGlobalId())
			return nil
		})
		return nil, err
	})
	return nil
}

func (self *SEsxiRegionDriver) RequestDeleteInstanceSnapshot(ctx context.Context, isp *models.SInstanceSnapshot, task taskman.ITask) error {
	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
		guest, err := isp.GetGuest()
		if err != nil {
			return nil, errors.Wrap(err, "GetGuest")
		}
		ivm, err := guest.GetIVM()
		if err != nil {
			return nil, errors.Wrap(err, "unable to GetIVM")
		}
		id := isp.GetExternalId()
		if len(id) == 0 {
			return nil, nil
		}
		cloudSP, err := ivm.GetInstanceSnapshot(id)
		if err != nil {
			return nil, errors.Wrap(err, "unable to GetInstanceSnapshot")
		}
		err = cloudSP.Delete()
		if err != nil {
			return nil, errors.Wrap(err, "unable to delete cloud instance snapshot")
		}
		return nil, nil
	})
	return nil
}

func (self *SEsxiRegionDriver) RequestResetToInstanceSnapshot(ctx context.Context, guest *models.SGuest, isp *models.SInstanceSnapshot, task taskman.ITask, params *jsonutils.JSONDict) error {
	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
		ivm, err := guest.GetIVM()
		if err != nil {
			return nil, errors.Wrap(err, "unable to GetIVM")
		}
		err = ivm.ResetToInstanceSnapshot(ctx, isp.GetExternalId())
		if err != nil {
			return nil, errors.Wrap(err, "unable to ResetToInstanceSnapshot")
		}
		return nil, nil
	})
	return nil
}


<|start_filename|>pkg/compute/models/cloudaccounts_prepare.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package models

import (
	"context"
	"fmt"
	"sort"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/util/netutils"

	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/proxy"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/compute/options"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/multicloud/esxi"
	"yunion.io/x/onecloud/pkg/util/httputils"
	"yunion.io/x/onecloud/pkg/util/rbacutils"
)

func (scm *SCloudaccountManager) AllowPerformPrepareNets(_ context.Context, userCred mcclient.TokenCredential, _ jsonutils.JSONObject) bool {
	return db.IsAdminAllowPerform(userCred, scm, "prepare-nets")
}

type sNetworkInfo struct {
	esxi.SNetworkInfo
	prefix string
}

func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, client *esxi.SESXiClient,
	input api.CloudaccountPerformPrepareNetsInput) ([]sNetworkInfo, error) {
	caName := input.Name
	wireLevel := input.WireLevelForVmware
	ret := make([]sNetworkInfo, 0)
	if len(wireLevel) == 0 {
		wireLevel = api.CLOUD_ACCOUNT_WIRE_LEVEL_VCENTER
	}
	switch wireLevel {
	case api.CLOUD_ACCOUNT_WIRE_LEVEL_VCENTER:
		nInfo, err := client.HostVmIPs(ctx)
		if err != nil {
			return nil, errors.Wrap(err, "unable to fetch ips of hosts and vms")
		}
		ret = append(ret, sNetworkInfo{
			SNetworkInfo: nInfo,
			prefix:       caName,
		})
	case api.CLOUD_ACCOUNT_WIRE_LEVEL_DATACENTER:
		dcs, err := client.GetDatacenters()
		if err != nil {
			return ret, errors.Wrap(err, "GetDatacenters")
		}
		for _, dc := range dcs {
			nInfo, err := client.HostVmIPsInDc(ctx, dc)
			if err != nil {
				return ret, errors.Wrapf(err, "unable to fetch ips of hosts and vms for dc %q", dc.GetName())
			}
			ret = append(ret, sNetworkInfo{
				SNetworkInfo: nInfo,
				prefix:       fmt.Sprintf("%s/%s", caName, dc.GetName()),
			})
		}
	case api.CLOUD_ACCOUNT_WIRE_LEVEL_CLUSTER:
		dcs, err := client.GetDatacenters()
		if err != nil {
			return ret, errors.Wrap(err, "GetDatacenters")
		}
		for _, dc := range dcs {
			clusters, err := dc.ListClusters()
			if err != nil {
				return nil, errors.Wrapf(err, "unable to ListCluster for dc %q", dc.GetName())
			}
			for _, cluster := range clusters {
				nInfo, err := client.HostVmIPsInCluster(ctx, cluster)
				if err != nil {
					return ret, errors.Wrapf(err, "unable to fetch ips of hosts and vms for dc %q cluster %q", dc.GetName(), cluster.GetName())
				}
				ret = append(ret, sNetworkInfo{
					SNetworkInfo: nInfo,
					prefix:       fmt.Sprintf("%s/%s/%s", caName, dc.GetName(), cluster.GetName()),
				})
			}
		}
	default:
		return nil, httperrors.NewInputParameterError("valid wire_level_for_vmware, accept vcenter, datacenter, cluster")
	}
	return ret, nil
}

// Performpreparenets searches for suitable network facilities for physical and virtual machines under the cloud account or provides configuration recommendations for network facilities before importing a cloud account.
func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.CloudaccountPerformPrepareNetsInput) (api.CloudaccountPerformPrepareNetsOutput, error) {
	var (
		err    error
		output api.CloudaccountPerformPrepareNetsOutput
	)
	if input.Provider != api.CLOUD_PROVIDER_VMWARE {
		return output, httperrors.NewNotSupportedError("not support for cloudaccount with provider '%s'", input.Provider)
	}
	// validate first
	ownerId, err := scm.FetchOwnerId(ctx, jsonutils.Marshal(input))
	if err != nil {
		return output, errors.Wrap(err, "FetchOwnerId in PerformPrepareNets")
	}
	if ownerId == nil {
		ownerId = userCred
	}
	// validate domain
	if len(input.ProjectDomainId) > 0 {
		_, input.DomainizedResourceInput, err = db.ValidateDomainizedResourceInput(ctx, input.DomainizedResourceInput)
		if err != nil {
			return output, err
		}
	}

	// Determine the zoneids according to esxiagent. If there is no esxiagent, zone0 is used by default. And the wires are filtered according to the specified domain and zoneids
	// make sure zone
	zoneids, err := scm.FetchEsxiZoneIds()
	if err != nil {
		return output, errors.Wrap(err, "unable to FetchEsxiZoneIds")
	}
	if len(zoneids) == 0 {
		id, err := scm.defaultZoneId(userCred)
		if err != nil {
			return output, errors.Wrap(err, "unable to fetch defaultZoneId")
		}
		zoneids = append(zoneids, id)
	}

	// Find the appropriate wire from above wires according to the Host's IP. The logic for finding is that the networks in the wire can contain the Host as much as possible. If no suitable one is found, a new wire is used.
	// fetch all Host
	factory, err := cloudprovider.GetProviderFactory(input.Provider)
	if err != nil {
		return output, errors.Wrap(err, "cloudprovider.GetProviderFactory")
	}
	input.SCloudaccount, err = factory.ValidateCreateCloudaccountData(ctx, userCred, input.SCloudaccountCredential)
	if err != nil {
		return output, errors.Wrap(err, "providerDriver.ValidateCreateCloudaccountData")
	}
	var proxyFunc httputils.TransportProxyFunc
	{
		if input.ProxySettingId == "" {
			input.ProxySettingId = proxyapi.ProxySettingId_DIRECT
		}
		var proxySetting *proxy.SProxySetting
		proxySetting, input.ProxySettingResourceInput, err = proxy.ValidateProxySettingResourceInput(userCred, input.ProxySettingResourceInput)
		if err != nil {
			return output, errors.Wrap(err, "ValidateProxySettingResourceInput")
		}
		proxyFunc = proxySetting.HttpTransportProxyFunc()
	}
	provider, err := factory.GetProvider(cloudprovider.ProviderConfig{
		Vendor:    input.Provider,
		URL:       input.AccessUrl,
		Account:   input.Account,
		Secret:    input.Secret,
		ProxyFunc: proxyFunc,
		Name:      input.Name,
	})
	if err != nil {
		return output, errors.Wrap(err, "factory.GetProvider")
	}
	iregion, err := provider.GetOnPremiseIRegion()
	if err != nil {
		return output, errors.Wrap(err, "provider.GetOnPremiseIRegion")
	}
	// hack
	client := iregion.(*esxi.SESXiClient)
	return scm.prepareNets(ctx, userCred, client, zoneids, input)
}

func (cam *SCloudaccountManager) prepareNets(ctx context.Context, userCred mcclient.TokenCredential,
	client *esxi.SESXiClient, zoneids []string, input api.CloudaccountPerformPrepareNetsInput) (api.CloudaccountPerformPrepareNetsOutput, error) {
	var output api.CloudaccountPerformPrepareNetsOutput
	obj, err := VpcManager.FetchById(api.DEFAULT_VPC_ID)
	if err != nil {
		return output, errors.Wrapf(err, "can not fetch vpc %q", api.DEFAULT_VPC_ID)
	}
	allNetworks, err := obj.(*SVpc).GetNetworks()
	if err != nil {
		return output, errors.Wrapf(err, "can not get networks of vpc %q", api.DEFAULT_VPC_ID)
	}
	existedIpPool := newIPPool(len(allNetworks))
	for i := range allNetworks {
		startIp, _ := netutils.NewIPV4Addr(allNetworks[i].GuestIpStart)
		endIp, _ := netutils.NewIPV4Addr(allNetworks[i].GuestIpEnd)
		existedIpPool.Insert(startIp, sSimpleNet{
			Diff: endIp - startIp,
			Vlan: int32(allNetworks[i].VlanId),
			Id:   allNetworks[i].Id,
		})
	}

	if !input.Dvs {
		// fetch all wire candidate
		wires, err := cam.fetchWires(userCred, input.ProjectDomainId, zoneids)
		if err != nil {
			return output, errors.Wrap(err, "unable to fetch wires")
		}
		// fetch networks
		networks := make([][]SNetwork, len(wires))
		for i := range networks {
			nets, err := wires[i].getNetworks(userCred, rbacutils.ScopeSystem)
			if err != nil {
				return output, errors.Wrap(err, "wire.getNetwork")
			}
			networks[i] = nets
		}
		nInfos, err := cam.hostVMIPsPrepareNets(ctx, client, input)
		if err != nil {
			return output, err
		}

		return cam.parseAndSuggestSingleWire(sParseAndSuggest{
			NInfos:        nInfos,
			AccountName:   input.Name,
			ZoneIds:       zoneids,
			Wires:         wires,
			Networks:      networks,
			ExistedIpPool: existedIpPool,
		}), nil
	}

	nInfo, err := client.HostVmIPsPro(ctx)
	if err != nil {
		return output, err
	}

	for name, ips := range nInfo.HostIps {
		ipNets := make([]api.CAIPNet, len(ips))
		for i, ip := range ips {
			var suitNetwork string
			if p, ok := existedIpPool.Get(ip); ok {
				suitNetwork = p.Id
			}
			ipNets[i] = api.CAIPNet{
				SuitableNetwork: suitNetwork,
				IP:              ip.String(),
			}
		}
		output.Hosts = append(output.Hosts, api.CAGuestNet{
			Name:   name,
			IPNets: ipNets,
		})
	}
	output.Guests = cam.parseSimpleVms(nInfo.VMs, existedIpPool)

	vsList := nInfo.VsMap.List()
	for i := range vsList {
		vs := vsList[i]
		hosts := make([]api.SimpleHost, len(vs.Hosts))
		for i := range hosts {
			hosts[i] = api.SimpleHost{
				Id:   vs.Hosts[i].Id,
				Name: vs.Hosts[i].Name,
			}
		}
		wire := api.CAPWire{
			Name:        vs.Name,
			Id:          vs.Id,
			Distributed: vs.Distributed,
			Hosts:       hosts,
		}
		// preparenets for host
		for hName, ips := range vs.HostIps {
			if len(ips) == 0 {
				output.Hosts = append(output.Hosts, api.CAGuestNet{
					Name: hName,
				})
			}
			for j := 0; j < len(ips); j++ {
				baseNet := ips[j].NetAddr(24)
				ipWithSameNet := []netutils.IPV4Addr{ips[j]}
				for k := j + 1; k < len(ips); k++ {
					net := ips[k].NetAddr(24)
					if net != baseNet {
						break
					}
					ipWithSameNet = append(ipWithSameNet, ips[k])
				}
				ipLimitLow, ipLimitUp := ips[0], ips[len(ips)-1]
				simNetConfs := cam.expandIPRnage(ips, ipLimitLow, ipLimitUp, func(proc esxi.SIPProc) bool {
					return proc.IsHost
				}, nInfo.IPPool, existedIpPool)
				for k := range simNetConfs {
					wire.HostNetworks = append(wire.HostNetworks, api.CANetConf{
						Name:            fmt.Sprintf("host-network-%d", len(wire.HostNetworks)+1),
						CASimpleNetConf: simNetConfs[k],
					})
				}
			}
		}
		for vlan, ips := range vs.Vlans {
			simNetConfs := cam.expandIPRnage(ips, 0, 0, func(proc esxi.SIPProc) bool {
				return proc.VlanId == vlan
			}, nInfo.IPPool, existedIpPool)
			for j := range simNetConfs {
				simNetConfs[j].VlanID = vlan
				wire.GuestNetworks = append(wire.GuestNetworks, api.CANetConf{
					Name:            fmt.Sprintf("guest-network-%d", len(wire.GuestNetworks)+1),
					CASimpleNetConf: simNetConfs[j],
				})
			}
		}
		output.Wires = append(output.Wires, wire)
	}

	return output, nil
}

func (cam *SCloudaccountManager) parseSimpleVms(vms []esxi.SSimpleVM, existedIpPool *sIPPool) []api.CAGuestNet {
	guests := make([]api.CAGuestNet, len(vms))
	for i := range guests {
		guests[i].Name = vms[i].Name
		for _, ipVlan := range vms[i].IPVlans {
			var suitableNetwork string
			p, ok := existedIpPool.Get(ipVlan.IP)
			if ok {
				suitableNetwork = p.Id
			}
			guests[i].IPNets = append(guests[i].IPNets, api.CAIPNet{
				IP:              ipVlan.IP.String(),
				VlanID:          ipVlan.VlanId,
				SuitableNetwork: suitableNetwork,
			})
		}
	}
	return guests
}

func (cam *SCloudaccountManager) expandIPRnage(ips []netutils.IPV4Addr, limitLow, limitUp netutils.IPV4Addr,
	expand func(esxi.SIPProc) bool, ipPool esxi.SIPPool, existedIpPool *sIPPool) []api.CASimpleNetConf {
	ret := make([]api.CASimpleNetConf, 0)
	for i := 0; i < len(ips); i++ {
		ip := ips[i]
		if _, ok := existedIpPool.Get(ip); ok {
			continue
		}
		net := ip.NetAddr(24)
		netLimitLow := net + 1
		netLimitUp := net + 254
		if limitLow != 0 && limitLow > netLimitLow {
			netLimitLow = limitLow
		}
		if limitUp != 0 && limitUp < netLimitUp {
			netLimitUp = limitUp
		}
		// find startip
		startIp := ip - 1
		for ; startIp >= netLimitLow; startIp-- {
			if _, ok := existedIpPool.Get(startIp); ok {
				break
			}
			if _, ok := ipPool.Get(startIp); ok {
				break
			}
		}
		endIp := ip + 1
		for ; endIp <= netLimitUp; endIp++ {
			if _, ok := existedIpPool.Get(endIp); ok {
				break
			}
			if proc, ok := ipPool.Get(endIp); ok {
				if expand(proc) {
					i++
					continue
				}
				break
			}
		}
		ret = append(ret, api.CASimpleNetConf{
			GuestIpStart: (startIp + 1).String(),
			GuestIpEnd:   (endIp - 1).String(),
			GuestIpMask:  24,
			GuestGateway: (net + netutils.IPV4Addr(options.Options.DefaultNetworkGatewayAddressEsxi)).String(),
		})
		// Avoid assigning already assigned ip subnet
		existedIpPool.Insert(startIp+1, sSimpleNet{
			Diff: endIp - startIp - 2,
		})
	}
	return ret
}

type sParseAndSuggest struct {
	NInfos        []sNetworkInfo
	AccountName   string
	ZoneIds       []string
	Wires         []SWire
	Networks      [][]SNetwork
	ExistedIpPool *sIPPool
}

type sIPPool struct {
	netranges    []netutils.IPV4Addr
	simpleNetMap map[netutils.IPV4Addr]sSimpleNet
}

func newIPPool(length ...int) *sIPPool {
	initLen := 0
	if len(length) > 0 {
		initLen = length[0]
	}
	return &sIPPool{
		netranges:    make([]netutils.IPV4Addr, 0, initLen),
		simpleNetMap: make(map[netutils.IPV4Addr]sSimpleNet, initLen),
	}
}

type sSimpleNet struct {
	Diff netutils.IPV4Addr
	Id   string
	Vlan int32
}

func (pl *sIPPool) Insert(startIp netutils.IPV4Addr, sNet sSimpleNet) {
	// TODO:check
	index := pl.getIndex(startIp)
	pl.netranges = append(pl.netranges, 0)
	pl.netranges = append(pl.netranges[:index+1], pl.netranges[index:len(pl.netranges)-1]...)
	pl.netranges[index] = startIp
	pl.simpleNetMap[startIp] = sNet
}

func (pl *sIPPool) getIndex(ip netutils.IPV4Addr) int {
	index := sort.Search(len(pl.netranges), func(n int) bool {
		return pl.netranges[n] >= ip
	})
	return index
}

func (pl *sIPPool) Get(ip netutils.IPV4Addr) (sSimpleNet, bool) {
	index := pl.getIndex(ip)
	if index >= len(pl.netranges) || index < 0 {
		return sSimpleNet{}, false
	}
	if pl.netranges[index] == ip {
		return pl.simpleNetMap[ip], true
	}
	if index == 0 {
		return sSimpleNet{}, false
	}
	startIp := pl.netranges[index-1]
	simpleNet := pl.simpleNetMap[startIp]
	if ip-startIp <= simpleNet.Diff {
		return simpleNet, true
	}
	return sSimpleNet{}, false
}

func (scm *SCloudaccountManager) parseAndSuggestSingleWire(params sParseAndSuggest) api.CloudaccountPerformPrepareNetsOutput {
	var (
		output   api.CloudaccountPerformPrepareNetsOutput
		nInfos   = params.NInfos
		wires    = params.Wires
		networks = params.Networks
	)
	output.CAWireNets = make([]api.CAWireNet, 0, len(nInfos))
	for _, ni := range nInfos {
		var (
			wireNet api.CAWireNet
			hostIps = ni.HostIps
		)

		// key of ipHosts is host's ip
		ipHosts := make(map[netutils.IPV4Addr]string, len(hostIps))
		for name, ip := range hostIps {
			ipHosts[ip] = name
		}
		// Find suitable wire and the network containing the Host IP in suitable wire.
		var (
			tmpSocre         int
			maxScore         = len(ipHosts)
			suitableWire     *SWire
			suitableNetworks map[netutils.IPV4Addr]*SNetwork
		)
		for i, nets := range networks {
			score := 0
			tmpSNs := make(map[netutils.IPV4Addr]*SNetwork)
			ipRanges := make([]netutils.IPV4AddrRange, len(nets))
			for i2 := range ipRanges {
				ipRanges[i2] = nets[i2].GetIPRange()
			}
			for ip := range ipHosts {
				for i := range ipRanges {
					if !ipRanges[i].Contains(ip) {
						continue
					}
					tmpSNs[ip] = &nets[i]
					score += 1
					break
				}
			}
			if score > tmpSocre {
				tmpSocre = score
				suitableWire = &wires[i]
				suitableNetworks = tmpSNs
			}
			if tmpSocre == maxScore {
				break
			}
		}
		if suitableWire != nil {
			wireNet.SuitableWire = suitableWire.GetId()
		} else {
			wireNet.SuggestedWire = api.CAWireConf{
				ZoneIds:     params.ZoneIds,
				Name:        ni.prefix + "-wire",
				Description: fmt.Sprintf("Auto created Wire for VMware account %q", params.AccountName),
			}
		}

		// Give the suggested network configuration for the Host IP that does not have a corresponding suitable network.
		noNetHostIP := make([]netutils.IPV4Addr, 0, len(ipHosts))
		for ip, name := range ipHosts {
			rnet := api.CAHostNet{
				Name: name,
				IP:   ip.String(),
			}
			if net, ok := suitableNetworks[ip]; ok {
				rnet.SuitableNetwork = net.GetId()
			} else {
				noNetHostIP = append(noNetHostIP, ip)
			}
			wireNet.Hosts = append(wireNet.Hosts, rnet)
		}

		if len(noNetHostIP) > 0 {
			sConfs := scm.suggestHostNetworks(noNetHostIP)
			confs := make([]api.CANetConf, len(sConfs))
			for i := range confs {
				confs[i].CASimpleNetConf = sConfs[i]
				confs[i].Name = fmt.Sprintf("%s-host-network-%d", ni.prefix, i+1)
			}
			wireNet.HostSuggestedNetworks = confs
		}

		for i := range wireNet.HostSuggestedNetworks {
			ipStart, _ := netutils.NewIPV4Addr(wireNet.HostSuggestedNetworks[i].GuestIpStart)
			ipEnd, _ := netutils.NewIPV4Addr(wireNet.HostSuggestedNetworks[i].GuestIpEnd)
			params.ExistedIpPool.Insert(ipStart, sSimpleNet{
				Diff: ipEnd - ipStart,
			})
		}

		wireNet.Guests = scm.parseSimpleVms(ni.VMs, params.ExistedIpPool)

		for vlan, ips := range ni.VlanIps {
			simNetConfs := scm.expandIPRnage(ips, 0, 0, func(proc esxi.SIPProc) bool {
				return proc.VlanId == vlan
			}, ni.IPPool, params.ExistedIpPool)
			for i := range simNetConfs {
				simNetConfs[i].VlanID = vlan
				wireNet.GuestSuggestedNetworks = append(wireNet.GuestSuggestedNetworks, api.CANetConf{
					Name:            fmt.Sprintf("host-network-%d", len(wireNet.GuestSuggestedNetworks)+1),
					CASimpleNetConf: simNetConfs[i],
				})
			}
		}
		output.CAWireNets = append(output.CAWireNets, wireNet)
	}
	return output
}

func (manager *SCloudaccountManager) fetchWires(userCred mcclient.TokenCredential, domainId string, zoneIds []string) ([]SWire, error) {
	q := WireManager.Query().In("zone_id", zoneIds)
	if len(domainId) > 0 {
		ownerId := &db.SOwnerId{}
		ownerId.DomainId = domainId
		q = WireManager.FilterByOwner(q, ownerId, rbacutils.ScopeDomain)
	} else {
		q = WireManager.FilterByOwner(q, userCred, rbacutils.ScopeDomain)
	}
	wires := make([]SWire, 0, 1)
	err := db.FetchModelObjects(WireManager, q, &wires)
	return wires, err
}

func (manager *SCloudaccountManager) defaultZoneId(userCred mcclient.TokenCredential) (string, error) {
	zone, err := ZoneManager.FetchByName(userCred, "zone0")
	if err != nil {
		return "", err
	}
	return zone.GetId(), nil
}

func (manager *SCloudaccountManager) FetchEsxiZoneIds() ([]string, error) {
	q := BaremetalagentManager.Query().Equals("agent_type", "esxiagent").Asc("created_at")
	agents := make([]SBaremetalagent, 0, 1)
	err := db.FetchModelObjects(BaremetalagentManager, q, &agents)
	if err != nil {
		return nil, err
	}
	ids := make([]string, 0, len(agents))
	for i := range agents {
		if agents[i].Status == api.BAREMETAL_AGENT_ENABLED {
			ids = append(ids, agents[i].ZoneId)
		}
	}
	for i := range agents {
		if agents[i].Status != api.BAREMETAL_AGENT_ENABLED {
			ids = append(ids, agents[i].ZoneId)
		}
	}
	return ids, nil
}

// The suggestHostNetworks give the suggest config of network contains the ip in 'ips'.
// The suggested network mask is 24 and the gateway is x.x.x.1.
// The suggests network is the smallest network segment that meets the above conditions.
func (manager *SCloudaccountManager) suggestHostNetworks(ips []netutils.IPV4Addr) []api.CASimpleNetConf {
	if len(ips) == 0 {
		return nil
	}
	sort.Slice(ips, func(i, j int) bool {
		return ips[i] < ips[j]
	})
	var (
		mask        int8 = 24
		lastnetAddr netutils.IPV4Addr
		consequent  []netutils.IPV4Addr
		ret         []api.CASimpleNetConf
	)
	lastnetAddr = ips[0].NetAddr(mask)
	consequent = []netutils.IPV4Addr{ips[0]}
	for i := 1; i < len(ips); i++ {
		ip := ips[i]
		netAddr := ip.NetAddr(mask)
		if netAddr == lastnetAddr && consequent[len(consequent)-1]+1 == ip {
			consequent = append(consequent, ip)
			continue
		}

		if netAddr != lastnetAddr {
			lastnetAddr = netAddr
		}

		gatewayIP := consequent[0].NetAddr(mask) + netutils.IPV4Addr(options.Options.DefaultNetworkGatewayAddressEsxi)
		ret = append(ret, api.CASimpleNetConf{
			GuestIpStart: consequent[0].String(),
			GuestIpEnd:   consequent[len(consequent)-1].String(),
			GuestIpMask:  mask,
			GuestGateway: gatewayIP.String(),
		})
		consequent = []netutils.IPV4Addr{ip}
	}
	gatewayIp := consequent[0].NetAddr(mask) + netutils.IPV4Addr(options.Options.DefaultNetworkGatewayAddressEsxi)
	ret = append(ret, api.CASimpleNetConf{
		GuestIpStart: consequent[0].String(),
		GuestIpEnd:   consequent[len(consequent)-1].String(),
		GuestIpMask:  mask,
		GuestGateway: gatewayIp.String(),
	})
	return ret
}

func (self *SCloudaccount) StartSyncVMwareNetworkTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, networkZone string) error {
	if self.Provider != api.CLOUD_PROVIDER_VMWARE {
		return errors.ErrNotSupported
	}
	params := jsonutils.NewDict()
	params.Set("zone", jsonutils.NewString(networkZone))
	task, err := taskman.TaskManager.NewTask(ctx, "CloudAccountSyncVMwareNetworkTask", self, userCred, params, parentTaskId, "", nil)
	if err != nil {
		return err
	}
	self.SetStatus(userCred, api.CLOUD_PROVIDER_SYNC_NETWORK, "StartSyncVMwareNetworkTask")
	task.ScheduleRun(nil)
	return nil
}


<|start_filename|>pkg/cloudcommon/db/scoperesource.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package db

import (
	"context"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/util/reflectutils"
	"yunion.io/x/pkg/utils"
	"yunion.io/x/sqlchemy"

	"yunion.io/x/onecloud/pkg/apis"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/util/rbacutils"
	"yunion.io/x/onecloud/pkg/util/stringutils2"
)

type SScopedResourceBaseManager struct {
	SProjectizedResourceBaseManager
}

// +onecloud:model-api-gen
type SScopedResourceBase struct {
	SProjectizedResourceBase `"domain_id->default":""`
}

type sUniqValues struct {
	Scope   string
	Project string
	Domain  string
}

func (m *SScopedResourceBaseManager) FetchUniqValues(ctx context.Context, data jsonutils.JSONObject) jsonutils.JSONObject {
	parentScope := rbacutils.ScopeSystem
	scope, _ := data.GetString("scope")
	if scope != "" {
		parentScope = rbacutils.TRbacScope(scope)
	}
	uniqValues := sUniqValues{}
	switch parentScope {
	case rbacutils.ScopeSystem:
	case rbacutils.ScopeDomain:
		domain, _ := data.GetString("project_domain")
		uniqValues.Domain = domain
	case rbacutils.ScopeProject:
		project, _ := data.GetString("project")
		uniqValues.Project = project
	}
	uniqValues.Scope = scope
	return jsonutils.Marshal(uniqValues)
}

func (m *SScopedResourceBaseManager) FilterByScope(q *sqlchemy.SQuery, scope rbacutils.TRbacScope, scopeResId string) *sqlchemy.SQuery {
	isNotNullOrEmpty := func(field string) sqlchemy.ICondition {
		return sqlchemy.AND(sqlchemy.IsNotNull(q.Field(field)), sqlchemy.IsNotEmpty(q.Field(field)))
	}
	switch scope {
	case rbacutils.ScopeSystem:
		q = q.IsNullOrEmpty("domain_id").IsNullOrEmpty("tenant_id")
	case rbacutils.ScopeDomain:
		q = q.IsNullOrEmpty("tenant_id").Filter(isNotNullOrEmpty("domain_id"))
		if scopeResId != "" {
			q = q.Equals("domain_id", scopeResId)
		}
	case rbacutils.ScopeProject:
		q = q.Filter(isNotNullOrEmpty("domain_id")).Filter(isNotNullOrEmpty("tenant_id"))
		if scopeResId != "" {
			q = q.Equals("tenant_id", scopeResId)
		}
	}
	return q
}

func (m *SScopedResourceBaseManager) FilterByUniqValues(q *sqlchemy.SQuery, values jsonutils.JSONObject) *sqlchemy.SQuery {
	uniqValues := &sUniqValues{}
	values.Unmarshal(uniqValues)
	if len(uniqValues.Domain) > 0 {
		return m.FilterByScope(q, rbacutils.TRbacScope(uniqValues.Scope), uniqValues.Domain)
	} else if len(uniqValues.Project) > 0 {
		return m.FilterByScope(q, rbacutils.TRbacScope(uniqValues.Scope), uniqValues.Project)
	} else {
		return m.FilterByScope(q, rbacutils.TRbacScope(uniqValues.Scope), "")
	}
}

func (m *SScopedResourceBase) IsOwner(userCred mcclient.TokenCredential) bool {
	scope := m.GetResourceScope()
	switch scope {
	case rbacutils.ScopeDomain:
		return userCred.GetProjectDomainId() == m.GetDomainId()
	case rbacutils.ScopeProject:
		return userCred.GetProjectId() == m.GetProjectId()
	}
	// system scope
	return userCred.HasSystemAdminPrivilege()
}

func (m *SScopedResourceBaseManager) FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
	if userCred == nil {
		return q
	}
	switch scope {
	case rbacutils.ScopeDomain:
		q = q.Equals("domain_id", userCred.GetProjectDomainId())
	case rbacutils.ScopeProject:
		q = q.Equals("tenant_id", userCred.GetProjectId())
	}
	return q
}

func (m *SScopedResourceBaseManager) ValidateCreateData(man IScopedResourceManager, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input apis.ScopedResourceCreateInput) (apis.ScopedResourceCreateInput, error) {
	if input.Scope == "" {
		input.Scope = string(rbacutils.ScopeSystem)
	}
	if !utils.IsInStringArray(input.Scope, []string{
		string(rbacutils.ScopeSystem),
		string(rbacutils.ScopeDomain),
		string(rbacutils.ScopeProject)}) {
		return input, httperrors.NewInputParameterError("invalid scope %s", input.Scope)
	}
	var allowCreate bool
	switch rbacutils.TRbacScope(input.Scope) {
	case rbacutils.ScopeSystem:
		allowCreate = IsAdminAllowCreate(userCred, man)
	case rbacutils.ScopeDomain:
		allowCreate = IsDomainAllowCreate(userCred, man)
		input.ProjectDomainId = ownerId.GetDomainId()
	case rbacutils.ScopeProject:
		allowCreate = IsProjectAllowCreate(userCred, man)
		input.ProjectDomainId = ownerId.GetDomainId()
		input.ProjectId = ownerId.GetProjectId()
	}
	if !allowCreate {
		return input, httperrors.NewForbiddenError("not allow create %s in scope %s", man.ResourceScope(), input.Scope)
	}
	return input, nil
}

func getScopedResourceScope(domainId, projectId string) rbacutils.TRbacScope {
	if domainId == "" && projectId == "" {
		return rbacutils.ScopeSystem
	}
	if domainId != "" && projectId == "" {
		return rbacutils.ScopeDomain
	}
	if domainId != "" && projectId != "" {
		return rbacutils.ScopeProject
	}
	return rbacutils.ScopeNone
}

func (s *SScopedResourceBase) GetResourceScope() rbacutils.TRbacScope {
	return getScopedResourceScope(s.DomainId, s.ProjectId)
}

func (s *SScopedResourceBase) GetDomainId() string {
	return s.DomainId
}

func (s *SScopedResourceBase) GetProjectId() string {
	return s.ProjectId
}

func (s *SScopedResourceBase) SetResourceScope(domainId, projectId string) error {
	s.DomainId = domainId
	s.ProjectId = projectId
	return nil
}

func (s *SScopedResourceBase) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
	scope, _ := data.GetString("scope")
	switch rbacutils.TRbacScope(scope) {
	case rbacutils.ScopeSystem:
		s.DomainId = ""
		s.ProjectId = ""
	case rbacutils.ScopeDomain:
		s.DomainId = ownerId.GetDomainId()
		s.ProjectId = ""
	case rbacutils.ScopeProject:
		s.DomainId = ownerId.GetDomainId()
		s.ProjectId = ownerId.GetProjectId()
	}
	return nil
}

func (s *SScopedResourceBase) GetMoreColumns(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
	if s.ProjectId != "" {
		extra.Add(jsonutils.NewString(s.ProjectId), "project_id")
	}
	return extra
}

type IScopedResourceModel interface {
	IModel

	GetDomainId() string
	GetProjectId() string
	SetResourceScope(domainId, projectId string) error
}

func PerformSetScope(
	ctx context.Context,
	obj IScopedResourceModel,
	userCred mcclient.TokenCredential,
	data jsonutils.JSONObject,
) (jsonutils.JSONObject, error) {
	domainId := jsonutils.GetAnyString(data, []string{"domain_id", "domain", "project_domain_id", "project_domain"})
	projectId := jsonutils.GetAnyString(data, []string{"project_id", "project"})
	if projectId != "" {
		project, err := DefaultProjectFetcher(ctx, projectId)
		if err != nil {
			return nil, err
		}
		projectId = project.GetProjectId()
		domainId = project.GetProjectDomainId()
	}
	if domainId != "" {
		domain, err := DefaultDomainFetcher(ctx, domainId)
		if err != nil {
			return nil, err
		}
		domainId = domain.GetProjectDomainId()
	}
	scopeToSet := getScopedResourceScope(domainId, projectId)
	var err error
	switch scopeToSet {
	case rbacutils.ScopeSystem:
		err = setScopedResourceToSystem(obj, userCred)
	case rbacutils.ScopeDomain:
		err = setScopedResourceToDomain(obj, userCred, domainId)
	case rbacutils.ScopeProject:
		err = setScopedResourceToProject(obj, userCred, projectId)
	}
	return nil, err
}

func setScopedResourceIds(model IScopedResourceModel, userCred mcclient.TokenCredential, domainId, projectId string) error {
	diff, err := Update(model, func() error {
		model.SetResourceScope(domainId, projectId)
		return nil
	})
	if err == nil {
		OpsLog.LogEvent(model, ACT_UPDATE, diff, userCred)
	}
	return err
}

func setScopedResourceToSystem(model IScopedResourceModel, userCred mcclient.TokenCredential) error {
	if !IsAdminAllowPerform(userCred, model, "set-scope") {
		return httperrors.NewForbiddenError("Not allow set scope to system")
	}
	if model.GetProjectId() == "" && model.GetDomainId() == "" {
		return nil
	}
	return setScopedResourceIds(model, userCred, "", "")
}

func setScopedResourceToDomain(model IScopedResourceModel, userCred mcclient.TokenCredential, domainId string) error {
	if !IsDomainAllowPerform(userCred, model, "set-scope") {
		return httperrors.NewForbiddenError("Not allow set scope to domain %s", domainId)
	}
	if model.GetDomainId() == domainId && model.GetProjectId() == "" {
		return nil
	}
	domain, err := TenantCacheManager.FetchDomainById(context.TODO(), domainId)
	if err != nil {
		return err
	}
	return setScopedResourceIds(model, userCred, domain.GetId(), "")
}

func setScopedResourceToProject(model IScopedResourceModel, userCred mcclient.TokenCredential, projectId string) error {
	if !IsProjectAllowPerform(userCred, model, "set-scope") {
		return httperrors.NewForbiddenError("Not allow set scope to project %s", projectId)
	}
	if model.GetProjectId() == projectId {
		return nil
	}
	project, err := TenantCacheManager.FetchTenantById(context.TODO(), projectId)
	if err != nil {
		return err
	}
	return setScopedResourceIds(model, userCred, project.GetProjectDomainId(), projectId)
}

func (m *SScopedResourceBaseManager) ListItemFilter(
	ctx context.Context,
	q *sqlchemy.SQuery,
	userCred mcclient.TokenCredential,
	query apis.ScopedResourceBaseListInput,
) (*sqlchemy.SQuery, error) {
	q, err := m.SProjectizedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ProjectizedResourceListInput)
	if err != nil {
		return nil, errors.Wrap(err, "SProjectizedResourceBaseManager.ListItemFilter")
	}
	if query.BelongScope != "" {
		q = m.FilterByScope(q, rbacutils.TRbacScope(query.BelongScope), "")
	}
	return q, nil
}

func (m *SScopedResourceBaseManager) OrderByExtraFields(
	ctx context.Context,
	q *sqlchemy.SQuery,
	userCred mcclient.TokenCredential,
	query apis.ScopedResourceBaseListInput,
) (*sqlchemy.SQuery, error) {
	q, err := m.SProjectizedResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.ProjectizedResourceListInput)
	if err != nil {
		return nil, errors.Wrap(err, "SProjectizedResourceBaseManager.OrderByExtraFields")
	}
	return q, nil
}

func (manager *SScopedResourceBaseManager) FetchCustomizeColumns(
	ctx context.Context,
	userCred mcclient.TokenCredential,
	query jsonutils.JSONObject,
	objs []interface{},
	fields stringutils2.SSortedStrings,
	isList bool,
) []apis.ScopedResourceBaseInfo {
	rows := make([]apis.ScopedResourceBaseInfo, len(objs))

	projRows := manager.SProjectizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)

	for i := range rows {
		rows[i] = apis.ScopedResourceBaseInfo{
			ProjectizedResourceInfo: projRows[i],
		}
		var base *SScopedResourceBase
		reflectutils.FindAnonymouStructPointer(objs[i], &base)
		if base != nil {
			if base.ProjectId != "" {
				project, _ := DefaultProjectFetcher(ctx, base.ProjectId)
				if project != nil {
					rows[i].Project = project.Name
					rows[i].ProjectDomain = project.Domain
				}
			} else if base.DomainId != "" {
				domain, _ := DefaultDomainFetcher(ctx, base.DomainId)
				if domain != nil {
					rows[i].ProjectDomain = domain.Name
				}
			}
			rows[i].Scope = string(base.GetResourceScope())
		}
	}

	return rows
}


<|start_filename|>pkg/keystone/cronjobs/project_resources.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cronjobs

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/utils"

	"yunion.io/x/onecloud/pkg/apis"
	api "yunion.io/x/onecloud/pkg/apis/identity"
	"yunion.io/x/onecloud/pkg/appsrv"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/keystone/models"
	"yunion.io/x/onecloud/pkg/keystone/tokens"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/mcclient/auth"
	"yunion.io/x/onecloud/pkg/util/httputils"
)

var (
	serviceBlackList map[string]time.Time
)

func init() {
	serviceBlackList = make(map[string]time.Time)
}

type sServiceEndpoints struct {
	regionId  string
	serviceId string
	internal  string
	external  string
}

func FetchScopeResourceCount(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
	err := refreshScopeResourceCount(ctx)
	if err != nil {
		log.Errorf("refreshScopeResourceCount error: %v", err)
	}
}
func refreshScopeResourceCount(ctx context.Context) error {
	log.Debugf("FetchScopeResourceCount")
	eps, err := models.EndpointManager.FetchAll()
	if err != nil {
		return errors.Wrapf(err, "EndpointManager.FetchAll")
	}
	serviceTbl := make(map[string]*sServiceEndpoints)
	for _, ep := range eps {
		if utils.IsInStringArray(ep.ServiceType, apis.NO_RESOURCE_SERVICES) {
			// skip self and offline cloudmeta
			continue
		}
		key := fmt.Sprintf("%s-%s", ep.RegionId, ep.ServiceId)
		if _, ok := serviceTbl[key]; !ok {
			serviceTbl[key] = &sServiceEndpoints{
				regionId:  ep.RegionId,
				serviceId: ep.ServiceId,
			}
		}
		switch ep.Interface {
		case api.EndpointInterfacePublic:
			serviceTbl[key].external = ep.Url
		case api.EndpointInterfaceInternal:
			serviceTbl[key].internal = ep.Url
		}
	}
	for srvId, ep := range serviceTbl {
		if to, ok := serviceBlackList[srvId]; ok {
			if to.IsZero() || to.After(time.Now()) {
				continue
			}
		}
		url := ep.internal
		if url == "" {
			url = ep.external
		}
		url = httputils.JoinPath(url, "scope-resources")
		tk, _ := tokens.GetDefaultToken()
		hdr := http.Header{}
		hdr.Add("X-Auth-Token", tk)
		_, ret, err := httputils.JSONRequest(
			httputils.GetDefaultClient(),
			ctx, "GET",
			url,
			hdr,
			nil, false)
		if err != nil {
			// ignore errors
			// log.Errorf("fetch from %s fail: %s", url, err)
			errCode := httputils.ErrorCode(err)
			if errCode == 404 {
				serviceBlackList[srvId] = time.Time{}
			} else {
				serviceBlackList[srvId] = time.Now().Add(time.Hour)
			}
			continue
		}
		if _, ok := serviceBlackList[srvId]; ok {
			delete(serviceBlackList, srvId)
		}
		projectResCounts := make(map[string][]db.SScopeResourceCount)
		err = ret.Unmarshal(&projectResCounts)
		if err != nil {
			continue
		}
		syncScopeResourceCount(ctx, ep.regionId, ep.serviceId, projectResCounts)
	}
	return nil
}

func syncScopeResourceCount(ctx context.Context, regionId string, serviceId string, projResCnt map[string][]db.SScopeResourceCount) {
	projList := make([]string, 0)
	domainList := []string{}
	ownerList := []string{}
	for res, resCnts := range projResCnt {
		for i := range resCnts {
			if len(resCnts[i].TenantId) == 0 && len(resCnts[i].DomainId) == 0 && len(resCnts[i].OwnerId) == 0 {
				continue
			}

			scopeRes := models.SScopeResource{
				DomainId:  resCnts[i].DomainId,
				ProjectId: resCnts[i].TenantId,
				OwnerId:   resCnts[i].OwnerId,
			}
			scopeRes.RegionId = regionId
			scopeRes.ServiceId = serviceId
			scopeRes.Resource = res
			scopeRes.Count = resCnts[i].ResCount

			if len(scopeRes.ProjectId) > 0 {
				projList = append(projList, scopeRes.ProjectId)
			}
			if len(scopeRes.DomainId) > 0 {
				domainList = append(domainList, scopeRes.DomainId)
			}
			if len(scopeRes.OwnerId) > 0 {
				ownerList = append(ownerList, scopeRes.OwnerId)
			}

			scopeRes.SetModelManager(models.ScopeResourceManager, &scopeRes)

			err := models.ScopeResourceManager.TableSpec().InsertOrUpdate(ctx, &scopeRes)
			if err != nil {
				log.Errorf("table insert error %s", err)
			}
		}

		q := models.ScopeResourceManager.Query()
		if len(projList) > 0 {
			q = q.NotIn("project_id", projList)
		}
		if len(domainList) > 0 {
			q = q.NotIn("domain_id", domainList)
		}
		if len(ownerList) > 0 {
			q = q.NotIn("owner_id", ownerList)
		}
		q = q.Equals("region_id", regionId)
		q = q.Equals("service_id", serviceId)
		q = q.Equals("resource", res)
		q = q.NotEquals("count", 0)

		emptySets := make([]models.SScopeResource, 0)
		err := db.FetchModelObjects(models.ScopeResourceManager, q, &emptySets)
		if err != nil {
			log.Errorf("db.FetchModelObjects %s", err)
		}

		for i := range emptySets {
			_, err := db.Update(&emptySets[i], func() error {
				emptySets[i].Count = 0
				return nil
			})
			if err != nil {
				log.Errorf("db.Update %s", err)
			}
		}
	}
}

func AddRefreshHandler(prefix string, app *appsrv.Application) {
	app.AddHandler2("POST", fmt.Sprintf("%s/scope-resource/refresh", prefix), auth.Authenticate(refreshHandler), nil, "scope_resource_refresh", nil)
}

func refreshHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	userCred := auth.FetchUserCredential(ctx, nil)
	if userCred == nil || !db.IsDomainAllowList(userCred, models.DomainManager) {
		httperrors.ForbiddenError(ctx, w, "not enough privilege")
		return
	}

	err := refreshScopeResourceCount(ctx)
	if err != nil {
		httperrors.GeneralServerError(ctx, w, err)
		return
	}
	ret := map[string]interface{}{
		"scope-resource": map[string]string{
			"status": "ok",
		},
	}
	fmt.Fprintf(w, jsonutils.Marshal(ret).String())
}


<|start_filename|>cmd/redfishcli/main.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"net/url"
	"os"
	"strings"

	"yunion.io/x/structarg"

	_ "yunion.io/x/onecloud/cmd/redfishcli/shell"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/redfish"
	"yunion.io/x/onecloud/pkg/util/redfish/bmconsole"
	_ "yunion.io/x/onecloud/pkg/util/redfish/loader"
	"yunion.io/x/onecloud/pkg/util/shellutils"
)

type BaseOptions struct {
	Debug      bool   `help:"debug mode"`
	Help       bool   `help:"Show help"`
	Endpoint   string `help:"Endpoint, usually https://<host_ipmi_ip>" default:"$REDFISH_ENDPOINT" metavar:"REDFISH_ENDPOINT"`
	Username   string `help:"Username, usually root" default:"$REDFISH_USERNAME" metavar:"REDFISH_USERNAME"`
	Password   string `help:"Password" default:"<PASSWORD>" metavar:"REDFISH_PASSWORD"`
	SUBCOMMAND string `help:"s3cli subcommand" subcommand:"true"`
}

var (
	options = &BaseOptions{}
)

func getSubcommandParser() (*structarg.ArgumentParser, error) {
	parse, e := structarg.NewArgumentParser(options,
		"redfishcli",
		"Command-line interface to redfish API.",
		`See "redfishcli help COMMAND" for help on a specific command.`)

	if e != nil {
		return nil, e
	}

	subcmd := parse.GetSubcommand()
	if subcmd == nil {
		return nil, fmt.Errorf("No subcommand argument.")
	}
	type HelpOptions struct {
		SUBCOMMAND string `help:"sub-command name"`
	}
	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
		if e != nil {
			return e
		} else {
			fmt.Print(helpstr)
			return nil
		}
	})
	bmcJnlp()
	for _, v := range shellutils.CommandTable {
		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
		if e != nil {
			return nil, e
		}
	}
	return parse, nil
}

func bmcJnlp() {
	type BmcGetOptions struct {
		BRAND string `help:"brand of baremetal" choices:"Lenovo|Huawei|HPE|Dell|Supermicro|Dell6|Dell7|Dell9"`
		Save  string `help:"save to file"`
		Debug bool   `help:"turn on debug mode"`
		Sku   string `help:"sku"`
		Model string `help:"model"`
	}
	shellutils.R(&BmcGetOptions{}, "bmc-jnlp", "Get Java Console JNLP file", func(args *BmcGetOptions) error {
		ctx := context.Background()
		parts, err := url.Parse(options.Endpoint)
		if err != nil {
			return err
		}
		bmc := bmconsole.NewBMCConsole(parts.Hostname(), options.Username, options.Password, args.Debug)
		var jnlp string
		switch strings.ToLower(args.BRAND) {
		case "hp", "hpe":
			jnlp, err = bmc.GetIloConsoleJNLP(ctx)
		case "dell", "dell inc.":
			jnlp, err = bmc.GetIdracConsoleJNLP(ctx, args.Sku, args.Model)
		case "dell6":
			jnlp, err = bmc.GetIdrac6ConsoleJNLP(ctx, args.Sku, args.Model)
		case "dell7":
			jnlp, err = bmc.GetIdrac7ConsoleJNLP(ctx, args.Sku, args.Model)
		case "dell9":
			jnlp, err = bmc.GetIdrac9ConsoleJNLP(ctx)
		case "supermicro":
			jnlp, err = bmc.GetSupermicroConsoleJNLP(ctx)
		case "lenovo":
			jnlp, err = bmc.GetLenovoConsoleJNLP(ctx)
		}
		if err != nil {
			return err
		}
		if len(args.Save) > 0 {
			return fileutils2.FilePutContents(args.Save, jnlp, false)
		} else {
			fmt.Println(jnlp)
			return nil
		}
	})
}

func showErrorAndExit(e error) {
	fmt.Fprintf(os.Stderr, "%s", e)
	fmt.Fprintln(os.Stderr)
	os.Exit(1)
}

func newClient() (redfish.IRedfishDriver, error) {
	if len(options.Endpoint) == 0 {
		return nil, fmt.Errorf("Missing endpoint")
	}

	if len(options.Username) == 0 {
		return nil, fmt.Errorf("Missing username")
	}

	if len(options.Password) == 0 {
		return nil, fmt.Errorf("Missing password")
	}

	cli := redfish.NewRedfishDriver(context.Background(), options.Endpoint, options.Username, options.Password, options.Debug)
	if cli == nil {
		return nil, fmt.Errorf("no approriate driver")
	}

	return cli, nil
}

func main() {
	parser, e := getSubcommandParser()
	if e != nil {
		showErrorAndExit(e)
	}
	e = parser.ParseArgs(os.Args[1:], false)
	// options := parser.Options().(*BaseOptions)

	if options.Help {
		fmt.Print(parser.HelpString())
	} else {
		subcmd := parser.GetSubcommand()
		subparser := subcmd.GetSubParser()
		if e != nil {
			if subparser != nil {
				fmt.Print(subparser.Usage())
			} else {
				fmt.Print(parser.Usage())
			}
			showErrorAndExit(e)
		} else {
			suboptions := subparser.Options()
			if options.SUBCOMMAND == "help" {
				e = subcmd.Invoke(suboptions)
			} else if options.SUBCOMMAND == "bmc-jnlp" {
				e = subcmd.Invoke(suboptions)
			} else {
				var client redfish.IRedfishDriver
				client, e = newClient()
				if e != nil {
					showErrorAndExit(e)
				}
				e = subcmd.Invoke(client, suboptions)
			}
			if e != nil {
				showErrorAndExit(e)
			}
		}
	}
}


<|start_filename|>pkg/mcclient/options/monitor/suggestsysrule.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package monitor

import (
	"strings"

	"yunion.io/x/jsonutils"

	"yunion.io/x/onecloud/pkg/apis/monitor"
	"yunion.io/x/onecloud/pkg/mcclient/options"
)

type SuggestRuleListOptions struct {
	options.BaseListOptions
}

func (o *SuggestRuleListOptions) Params() (jsonutils.JSONObject, error) {
	return options.ListStructToParams(o)
}

type SuggestRuleShowOptions struct {
	ID string `help:"ID or name of the alert" json:"-"`
}

func (o *SuggestRuleShowOptions) Params() (jsonutils.JSONObject, error) {
	return options.StructToParams(o)
}

func (o *SuggestRuleShowOptions) GetId() string {
	return o.ID
}

type SuggestRuleConfigOptions struct {
	ID       string `help:"ID or name of the alert" json:"-"`
	Period   string `help:"Period of suggest rule e.g. '5s', '1m'"`
	TimeFrom string `help:"TimeFrom of suggest rule e.g. '24h'"`
}

func (o *SuggestRuleConfigOptions) Params() (jsonutils.JSONObject, error) {
	return options.StructToParams(o)
}

func (o *SuggestRuleConfigOptions) GetId() string {
	return o.ID
}

type SuggestSysRuleAlertSettingOptions struct {
	Status string `help:"Status of eip_unused rule"`
}

type SuggestRuleCreateOptions struct {
	SuggestSysRuleAlertSettingOptions
	Name    string `help:"Name of the alert"`
	Type    string `help:"Type of suggest rule" choices:"EIP_UNUSED|DISK_UNUSED|LB_UNUSED|SCALE_DOWN"`
	Enabled bool   `help:"Enable rule"`
	Period  string `help:"Period of suggest rule e.g. '5s', '1m'" default:"30s"`
}

func (opt *SuggestRuleCreateOptions) Params() (jsonutils.JSONObject, error) {
	input := new(monitor.SuggestSysRuleCreateInput)
	input.Name = opt.Name
	input.Period = opt.Period
	input.Type = strings.ToUpper(opt.Type)
	input.Enabled = &opt.Enabled
	input.Setting = newSuggestSysAlertSetting(input.Type)
	return input.JSON(input), nil
}

type SuggestRuleUpdateOptions struct {
	SuggestSysRuleAlertSettingOptions
	ID      string `help:"ID or name of the alert" json:"-"`
	Name    string `help:"Name of the alert"`
	Period  string `help:"Period of suggest rule e.g. '5s', '1m'" default:"30s"`
	Enabled bool   `help:"Enable rule"`
	Type    string `help:"Type of suggest rule" choices:"EIP_UNUSED|"`
}

func (opt SuggestRuleUpdateOptions) Params() (jsonutils.JSONObject, error) {
	input := new(monitor.SuggestSysRuleUpdateInput)
	input.Period = opt.Period
	input.Enabled = &opt.Enabled
	input.Name = opt.Name
	input.Setting = newSuggestSysAlertSetting(input.Type)
	return input.JSON(input), nil
}

func newSuggestSysAlertSetting(tp string) *monitor.SSuggestSysAlertSetting {
	setting := new(monitor.SSuggestSysAlertSetting)
	switch monitor.SuggestDriverType(tp) {
	case monitor.EIP_UNUSED:
		setting = &monitor.SSuggestSysAlertSetting{
			EIPUnused: &monitor.EIPUnused{},
		}
	case monitor.DISK_UNUSED:
		setting = &monitor.SSuggestSysAlertSetting{
			DiskUnused: &monitor.DiskUnused{},
		}
	case monitor.LB_UNUSED:
		setting = &monitor.SSuggestSysAlertSetting{
			LBUnused: &monitor.LBUnused{},
		}
	case monitor.SCALE_DOWN:
		scaleRuel := make(monitor.ScaleRule, 0)
		scale := monitor.Scale{
			Database:    "telegraf",
			Measurement: "vm_cpu",
			Operator:    "and",
			Field:       "usage_active",
			EvalType:    ">=",
			Threshold:   50,
			Tag:         "",
			TagVal:      "",
		}
		scaleRuel = append(scaleRuel, scale)
		scale = monitor.Scale{
			Database:    "telegraf",
			Measurement: "vm_diskio",
			Operator:    "or",
			Field:       "read_bps",
			EvalType:    ">=",
			Threshold:   500,
			Tag:         "",
			TagVal:      "",
		}
		scaleRuel = append(scaleRuel, scale)
		setting = &monitor.SSuggestSysAlertSetting{
			ScaleRule: &scaleRuel,
		}

	}
	return setting
}

type SuggestRuleDeleteOptions struct {
	ID string `help:"ID of alert to delete"`
}

func (o *SuggestRuleDeleteOptions) GetId() string {
	return o.ID
}

func (o *SuggestRuleDeleteOptions) Params() (jsonutils.JSONObject, error) {
	return options.StructToParams(o)
}


<|start_filename|>pkg/util/fuseutils/fuseutils.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuseutils

import (
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/procutils"
)

const DEFAULT_BLOCKSIZE = 8

func MountFusefs(fetcherfsPath, url, tmpdir, token, mntpath string, blocksize int) error {
	var metaPath = path.Join(mntpath, "meta")
	if f, err := os.OpenFile(metaPath, os.O_RDONLY, 0644); err == nil {
		f.Close()
		log.Infof("Already mounted: %s, skip...", mntpath)
		return nil
	}

	// is mounted
	if err := procutils.NewCommand("mountpoint", mntpath).Run(); err == nil {
		out, err := procutils.NewCommand("umount", mntpath).Output()
		if err != nil {
			return errors.Wrapf(err, "umount %s failed: %s", mntpath, out)
		}
	}

	if !fileutils2.Exists(tmpdir) {
		if out, err := procutils.NewCommand("mkdir", "-p", tmpdir).Output(); err != nil {
			return errors.Wrapf(err, "mkdir %s failed: %s", tmpdir, out)
		}
	}

	if !fileutils2.Exists(mntpath) {
		if out, err := procutils.NewCommand("mkdir", "-p", mntpath).Output(); err != nil {
			return errors.Wrapf(err, "mkdir %s failed: %s", mntpath, out)
		}
	}

	var cmd = []string{
		fetcherfsPath,
		"--url", url,
		"--token", token,
		"--tmpdir", tmpdir,
		"--blocksize", strconv.Itoa(blocksize),
		"--mount-point", mntpath,
	}
	log.Infof("%s", strings.Join(cmd, " "))
	out, err := procutils.NewRemoteCommandAsFarAsPossible(cmd[0], cmd[1:]...).Output()
	if err != nil {
		log.Errorf("%v Mount fetcherfs filed: %s %s", cmd, err, out)
		out2, err2 := procutils.NewCommand("umount", mntpath).Output()
		if err2 != nil {
			log.Errorf("umount fetcherfs failed %s %s", err2, out2)
		}
		return errors.Wrapf(err, "mount fetcherfs failed: %s", out)
	}

	var mounted = false
	for i := 0; i < 3; i++ {
		time.Sleep(1 * time.Second)
		if f, err := os.OpenFile(metaPath, os.O_RDONLY, 0644); err == nil {
			f.Close()
			mounted = true
			break
		} else {
			log.Warningf("failed open metaPath %s: %s", metaPath, err)
		}
	}
	if !mounted {
		out2, err2 := procutils.NewCommand("umount", mntpath).Output()
		if err2 != nil {
			log.Errorf("umount fetcherfs failed %s %s", err2, out2)
		}
		return errors.Error("failed open metaPath")
	} else {
		return nil
	}
}


<|start_filename|>pkg/multicloud/azure/classic_instancenic.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package azure

import (
	"yunion.io/x/log"
	"yunion.io/x/pkg/util/netutils"

	"yunion.io/x/onecloud/pkg/cloudprovider"
)

type SClassicInstanceNic struct {
	instance *SClassicInstance

	ID       string
	IP       string
	Name     string
	Type     string
	Location string

	cloudprovider.DummyICloudNic
}

func (self *SClassicInstanceNic) GetId() string {
	return ""
}

func (self *SClassicInstanceNic) GetIP() string {
	return self.IP
}

func (self *SClassicInstanceNic) Delete() error {
	return cloudprovider.ErrNotImplemented
}

func (self *SClassicInstanceNic) GetMAC() string {
	ip, _ := netutils.NewIPV4Addr(self.GetIP())
	return ip.ToMac("00:16:")
}

func (self *SClassicInstanceNic) GetDriver() string {
	return "virtio"
}

func (self *SClassicInstanceNic) InClassicNetwork() bool {
	return true
}

func (self *SClassicInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
	wires, err := self.instance.host.GetIWires()
	if err != nil {
		log.Errorf("GetINetwork error: %v", err)
		return nil
	}
	for i := 0; i < len(wires); i++ {
		wire := wires[i].(*SClassicWire)
		if network := wire.getNetworkById(self.ID); network != nil {
			return network
		}
	}
	return nil
}


<|start_filename|>pkg/keystone/driver/saml/saml.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package saml

import (
	"context"
	"encoding/base64"
	"fmt"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/identity"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/keystone/driver"
	"yunion.io/x/onecloud/pkg/keystone/models"
	"yunion.io/x/onecloud/pkg/keystone/saml"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/util/samlutils"
	"yunion.io/x/onecloud/pkg/util/samlutils/sp"
)

// SAML 2.0 Service Provider Driver
type SSAMLDriver struct {
	driver.SBaseIdentityDriver

	samlConfig *api.SSAMLIdpConfigOptions

	isDebug bool
}

func NewSAMLDriver(idpId, idpName, template, targetDomainId string, conf api.TConfigs) (driver.IIdentityBackend, error) {
	base, err := driver.NewBaseIdentityDriver(idpId, idpName, template, targetDomainId, conf)
	if err != nil {
		return nil, errors.Wrap(err, "NewBaseIdentityDriver")
	}
	drv := SSAMLDriver{SBaseIdentityDriver: base}
	drv.SetVirtualObject(&drv)
	err = drv.prepareConfig()
	if err != nil {
		return nil, errors.Wrap(err, "prepareConfig")
	}
	return &drv, nil
}

func (self *SSAMLDriver) prepareConfig() error {
	if self.samlConfig == nil {
		confJson := jsonutils.Marshal(self.Config["saml"])
		conf := api.SSAMLIdpConfigOptions{}
		switch self.Template {
		case api.IdpTemplateSAMLTest:
			conf = SAMLTestTemplate
		case api.IdpTemplateAzureADSAML:
			conf = AzureADTemplate
			tenantId, _ := confJson.GetString("tenant_id")
			conf.EntityId = fmt.Sprintf("https://sts.windows.net/%s/", tenantId)
			conf.RedirectSSOUrl = fmt.Sprintf("https://login.microsoftonline.com/%s/saml2", tenantId)
		}
		err := confJson.Unmarshal(&conf)
		if err != nil {
			return errors.Wrap(err, "json.Unmarshal")
		}
		log.Debugf("%s %s %#v", self.Config, confJson, self.samlConfig)
		self.samlConfig = &conf
	}
	return nil
}

func (self *SSAMLDriver) GetSsoRedirectUri(ctx context.Context, callbackUrl, state string) (string, error) {
	spLoginFunc := func(ctx context.Context, idp *sp.SSAMLIdentityProvider) (sp.SSAMLSpInitiatedLoginRequest, error) {
		result := sp.SSAMLSpInitiatedLoginRequest{}
		result.RequestID = samlutils.GenerateSAMLId()
		result.RelayState = state
		return result, nil
	}
	spInst := sp.NewSpInstance(saml.SAMLInstance(), self.IdpName, nil, spLoginFunc)
	spInst.SetAssertionConsumerUri(callbackUrl)
	err := spInst.AddIdp(self.samlConfig.EntityId, self.samlConfig.RedirectSSOUrl)
	if err != nil {
		return "", errors.Wrap(err, "Invalid SAMLIdentityProvider")
	}
	input := samlutils.SSpInitiatedLoginInput{
		EntityID: self.samlConfig.EntityId,
	}
	redir, err := spInst.ProcessSpInitiatedLogin(ctx, input)
	if err != nil {
		return "", errors.Wrap(err, "ProcessSpInitiatedLogin")
	}
	return redir, nil
}

func (self *SSAMLDriver) Authenticate(ctx context.Context, ident mcclient.SAuthenticationIdentity) (*api.SUserExtended, error) {
	samlRespBytes, err := base64.StdEncoding.DecodeString(ident.SAMLAuth.Response)
	if err != nil {
		return nil, errors.Wrap(err, "base64.StdEncoding.DecodeString")
	}

	resp, err := saml.SAMLInstance().UnmarshalResponse(samlRespBytes)
	if err != nil {
		return nil, errors.Wrap(err, "decode SAMLResponse error")
	}

	if !resp.IsSuccess() {
		return nil, errors.Wrap(httperrors.ErrInvalidCredential, "SAML auth unsuccess")
	}

	attrs := resp.FetchAttribtues()

	var domainId, domainName, usrId, usrName string
	if v, ok := attrs[self.samlConfig.DomainIdAttribute]; ok && len(v) > 0 {
		domainId = v[0]
	}
	if v, ok := attrs[self.samlConfig.DomainNameAttribute]; ok && len(v) > 0 {
		domainName = v[0]
	}
	if v, ok := attrs[self.samlConfig.UserIdAttribute]; ok && len(v) > 0 {
		usrId = v[0]
	}
	if v, ok := attrs[self.samlConfig.UserNameAttribute]; ok && len(v) > 0 {
		usrName = v[0]
	}

	idp, err := models.IdentityProviderManager.FetchIdentityProviderById(self.IdpId)
	if err != nil {
		return nil, errors.Wrap(err, "self.GetIdentityProvider")
	}

	domain, usr, err := idp.SyncOrCreateDomainAndUser(ctx, domainId, domainName, usrId, usrName)
	if err != nil {
		return nil, errors.Wrap(err, "idp.SyncOrCreateDomainAndUser")
	}
	extUser, err := models.UserManager.FetchUserExtended(usr.Id, "", "", "")
	if err != nil {
		return nil, errors.Wrap(err, "models.UserManager.FetchUserExtended")
	}

	idp.TryUserJoinProject(self.samlConfig.SIdpAttributeOptions, ctx, usr, domain.Id, attrs)

	return extUser, nil
}

func (self *SSAMLDriver) Sync(ctx context.Context) error {
	return nil
}

func (self *SSAMLDriver) Probe(ctx context.Context) error {
	return nil
}


<|start_filename|>pkg/compute/tasks/inter_vpc_network_create_task.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
	"context"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type InterVpcNetworkCreateTask struct {
	taskman.STask
}

func init() {
	taskman.RegisterTask(InterVpcNetworkCreateTask{})
}

func (self *InterVpcNetworkCreateTask) taskFailed(ctx context.Context, network *models.SInterVpcNetwork, err error) {
	network.SetStatus(self.UserCred, api.INTER_VPC_NETWORK_STATUS_CREATE_FAILED, err.Error())
	db.OpsLog.LogEvent(network, db.ACT_CREATE, err, self.UserCred)
	logclient.AddActionLogWithStartable(self, network, logclient.ACT_CREATE, err, self.UserCred, false)
	self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}

func (self *InterVpcNetworkCreateTask) taskComplete(ctx context.Context, network *models.SInterVpcNetwork) {
	network.SetStatus(self.GetUserCred(), api.INTER_VPC_NETWORK_STATUS_AVAILABLE, "")
	logclient.AddActionLogWithStartable(self, network, logclient.ACT_CREATE, nil, self.UserCred, true)
	self.SetStageComplete(ctx, nil)
}

func (self *InterVpcNetworkCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
	interVpcNetwork := obj.(*models.SInterVpcNetwork)
	provider, err := interVpcNetwork.GetProvider()
	if err != nil {
		self.taskFailed(ctx, interVpcNetwork, errors.Wrapf(err, "GetProvider"))
		return
	}
	opts := &cloudprovider.SInterVpcNetworkCreateOptions{
		Name: interVpcNetwork.Name,
		Desc: interVpcNetwork.Description,
	}

	inetwork, err := provider.CreateICloudInterVpcNetwork(opts)
	if err != nil {
		self.taskFailed(ctx, interVpcNetwork, errors.Wrap(err, "provider.CreateICloudInterVpcNetwork()"))
		return
	}
	err = interVpcNetwork.SyncWithCloudInterVpcNetwork(ctx, self.UserCred, inetwork)
	if err != nil {
		self.taskFailed(ctx, interVpcNetwork, errors.Wrap(err, "snetwork.SyncWithCloudInterVpcNetwork()"))
		return
	}
	self.taskComplete(ctx, interVpcNetwork)
}


<|start_filename|>pkg/multicloud/qcloud/shell/dnspod.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shell

import (
	"strconv"

	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/multicloud/qcloud"
	"yunion.io/x/onecloud/pkg/util/shellutils"
)

func init() {
	type DomianListOptions struct {
		Offset    int
		Limit     int
		ProjectId string
	}
	shellutils.R(&DomianListOptions{}, "domain-list", "List domains", func(cli *qcloud.SRegion, args *DomianListOptions) error {
		domains, total, e := cli.GetClient().GetDomains(args.ProjectId, args.Offset, args.Limit)
		if e != nil {
			return e
		}
		printList(domains, total, args.Offset, args.Limit, []string{})
		// cli.GetClient().GetAllDomains()
		return nil
	})

	type DomianCreateOptions struct {
		DOMAIN string
	}
	shellutils.R(&DomianCreateOptions{}, "domain-create", "create domain", func(cli *qcloud.SRegion, args *DomianCreateOptions) error {
		domain, e := cli.GetClient().CreateDomian(args.DOMAIN)
		if e != nil {
			return e
		}
		printObject(domain)
		return nil
	})

	type DomianDeleteOptions struct {
		DOMAIN string
	}
	shellutils.R(&DomianDeleteOptions{}, "domain-delete", "delete domains", func(cli *qcloud.SRegion, args *DomianDeleteOptions) error {
		e := cli.GetClient().DeleteDomian(args.DOMAIN)
		if e != nil {
			return e
		}
		return nil
	})

	type DnsRecordListOptions struct {
		DOMAIN    string
		ProjectId string
		Offset    int
		Limit     int
	}
	shellutils.R(&DnsRecordListOptions{}, "dnsrecord-list", "List dndrecord", func(cli *qcloud.SRegion, args *DnsRecordListOptions) error {
		records, total, e := cli.GetClient().GetDnsRecords(args.ProjectId, args.DOMAIN, args.Offset, args.Limit)
		if e != nil {
			return e
		}
		printList(records, total, args.Offset, args.Limit, []string{})
		// cli.GetClient().GetAllDnsRecords(args.Domain)
		return nil
	})

	type DnsRecordCreateOptions struct {
		DOMAIN string
		NAME   string
		VALUE  string //joined by '*'
		TTL    int64
		TYPE   string
	}
	shellutils.R(&DnsRecordCreateOptions{}, "dnsrecord-create", "create dndrecord", func(cli *qcloud.SRegion, args *DnsRecordCreateOptions) error {
		change := cloudprovider.DnsRecordSet{}
		change.DnsName = args.NAME
		change.DnsValue = args.VALUE
		change.Ttl = args.TTL
		change.DnsType = cloudprovider.TDnsType(args.TYPE)
		_, e := cli.GetClient().CreateDnsRecord(&change, args.DOMAIN)
		if e != nil {
			return e
		}
		return nil
	})

	type DnsRecordUpdateOptions struct {
		DOMAIN   string
		RECORDID int
		NAME     string
		VALUE    string //joined by '*'
		TTL      int64
		TYPE     string
	}
	shellutils.R(&DnsRecordUpdateOptions{}, "dnsrecord-update", "update dndrecord", func(cli *qcloud.SRegion, args *DnsRecordUpdateOptions) error {
		change := cloudprovider.DnsRecordSet{}
		change.DnsName = args.NAME
		change.ExternalId = strconv.Itoa(args.RECORDID)
		change.DnsValue = args.VALUE
		change.Ttl = args.TTL
		change.DnsType = cloudprovider.TDnsType(args.TYPE)
		e := cli.GetClient().ModifyDnsRecord(&change, args.DOMAIN)
		if e != nil {
			return e
		}
		return nil
	})

	type DnsRecordUpdateStatusOptions struct {
		DOMAIN   string
		RECORDID int
		STATUS   string `choices:"disable|enable"`
	}
	shellutils.R(&DnsRecordUpdateStatusOptions{}, "dnsrecord-updatestatus", "update dndrecord", func(cli *qcloud.SRegion, args *DnsRecordUpdateStatusOptions) error {
		e := cli.GetClient().ModifyRecordStatus(args.STATUS, strconv.Itoa(args.RECORDID), args.DOMAIN)
		if e != nil {
			return e
		}
		return nil
	})

	type DnsRecordRemoveOptions struct {
		DOMAIN   string
		RECORDID int
	}
	shellutils.R(&DnsRecordRemoveOptions{}, "dnsrecord-delete", "delete dndrecord", func(cli *qcloud.SRegion, args *DnsRecordRemoveOptions) error {
		e := cli.GetClient().DeleteDnsRecord(args.RECORDID, args.DOMAIN)
		if e != nil {
			return e
		}
		return nil
	})
}


<|start_filename|>pkg/compute/tasks/server_sku_delete_task.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
	"context"

	"github.com/pkg/errors"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/utils"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type ServerSkuDeleteTask struct {
	taskman.STask
}

func init() {
	taskman.RegisterTask(ServerSkuDeleteTask{})
}

func (self *ServerSkuDeleteTask) taskFail(ctx context.Context, sku *models.SServerSku, err error) {
	sku.SetStatus(self.UserCred, api.SkuStatusDeleteFailed, err.Error())
	db.OpsLog.LogEvent(sku, db.ACT_DELOCATE, err, self.GetUserCred())
	logclient.AddActionLogWithStartable(self, sku, logclient.ACT_DELETE, err, self.UserCred, false)
	self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}

func (self *ServerSkuDeleteTask) taskComplete(ctx context.Context, sku *models.SServerSku) {
	err := sku.RealDelete(ctx, self.UserCred)
	if err != nil {
		self.taskFail(ctx, sku, errors.Wrapf(err, "RealDelete"))
		return
	}
	logclient.AddActionLogWithStartable(self, sku, logclient.ACT_DELETE, nil, self.UserCred, true)
	self.SetStageComplete(ctx, nil)
}

func (self *ServerSkuDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	sku := obj.(*models.SServerSku)
	purge := jsonutils.QueryBoolean(self.GetParams(), "purge", false)
	if !purge && utils.IsInStringArray(sku.Provider, api.PRIVATE_CLOUD_PROVIDERS) {
		iSku, err := sku.GetICloudSku()
		if err != nil {
			if errors.Cause(err) == cloudprovider.ErrNotFound {
				self.taskComplete(ctx, sku)
				return
			}
			self.taskFail(ctx, sku, errors.Wrapf(err, "GetICloudSku"))
			return
		}
		err = iSku.Delete()
		if err != nil {
			self.taskFail(ctx, sku, errors.Wrapf(err, "iSku.Delete"))
			return
		}
	}
	self.taskComplete(ctx, sku)
}


<|start_filename|>pkg/hostman/storageman/storage_nas.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storageman

import (
	"context"
	"fmt"
	"path"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"

	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/hostman/hostutils"
	"yunion.io/x/onecloud/pkg/mcclient/modules"
)

type INasStorage interface {
	newDisk(diskId string) IDisk
	StorageType() string
}

type SNasStorage struct {
	SLocalStorage

	ins INasStorage
}

func NewNasStorage(manager *SStorageManager, path string, ins INasStorage) *SNasStorage {
	ret := &SNasStorage{*NewLocalStorage(manager, path, 0), ins}
	return ret
}

func (s *SNasStorage) GetComposedName() string {
	return fmt.Sprintf("host_%s_%s_storage_%d", s.Manager.host.GetMasterIp(), s.ins.StorageType(), s.Index)
}

func (s *SNasStorage) GetSnapshotDir() string {
	return path.Join(s.Path, _SNAPSHOT_PATH_)
}

func (s *SNasStorage) CreateDisk(diskId string) IDisk {
	s.DiskLock.Lock()
	defer s.DiskLock.Unlock()
	disk := s.ins.newDisk(diskId)
	s.Disks = append(s.Disks, disk)
	return disk
}

func (s *SNasStorage) GetDiskById(diskId string) (IDisk, error) {
	s.DiskLock.Lock()
	defer s.DiskLock.Unlock()
	for i := 0; i < len(s.Disks); i++ {
		if s.Disks[i].GetId() == diskId {
			return s.Disks[i], s.Disks[i].Probe()
		}
	}
	var disk = s.ins.newDisk(diskId)
	if disk.Probe() == nil {
		s.Disks = append(s.Disks, disk)
		return disk, nil
	}
	return nil, cloudprovider.ErrNotFound
}

func (s *SNasStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {
	if len(s.StorageId) == 0 {
		return nil, fmt.Errorf("Sync nfs storage without storage id")
	}
	content := jsonutils.NewDict()
	content.Set("capacity", jsonutils.NewInt(int64(s.GetAvailSizeMb())))
	content.Set("storage_type", jsonutils.NewString(s.ins.StorageType()))
	content.Set("zone", jsonutils.NewString(s.GetZoneName()))
	log.Infof("Sync storage info %s", s.StorageId)
	res, err := modules.Storages.Put(
		hostutils.GetComputeSession(context.Background()),
		s.StorageId, content)
	if err != nil {
		log.Errorf("SyncStorageInfo Failed: %s: %s", content, err)
	}
	return res, err
}

func (s *SNasStorage) CreateDiskFromSnapshot(
	ctx context.Context, disk IDisk, createParams *SDiskCreateByDiskinfo,
) error {
	var (
		snapshotUrl, _ = createParams.DiskInfo.GetString("snapshot_url")
		diskSize, _    = createParams.DiskInfo.Int("size")
	)
	return disk.CreateFromSnapshotLocation(ctx, snapshotUrl, diskSize)
}


<|start_filename|>pkg/httperrors/errors.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package httperrors

import (
	"yunion.io/x/pkg/errors"

	"yunion.io/x/onecloud/pkg/util/httputils"
)

func NewBadGatewayError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrBadGateway], string(ErrBadGateway), msg, params...)
}

func NewNotImplementedError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrNotImplemented], string(ErrNotImplemented), msg, params...)
}

func NewInternalServerError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrInternalError], string(ErrInternalError), msg, params...)
}

func NewResourceNotReadyError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrResourceNotReady], string(ErrResourceNotReady), msg, params...)
}

func NewOutOfResourceError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrOutOfResource], string(ErrOutOfResource), msg, params...)
}

func NewServerStatusError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrServerStatus], string(ErrServerStatus), msg, params...)
}

func NewPaymentError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrPayment], string(ErrPayment), msg, params...)
}

func NewImageNotFoundError(imageId string) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrImageNotFound], string(ErrImageNotFound), "Image %s not found", imageId)
}

func NewResourceNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrResourceNotFound], string(ErrResourceNotFound), msg, params...)
}

func NewResourceNotFoundError2(keyword, id string) *httputils.JSONClientError {
	return NewResourceNotFoundError("%s %s not found", keyword, id)
}

func NewSpecNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrSpecNotFound], string(ErrSpecNotFound), msg, params...)
}

func NewActionNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrActionNotFound], string(ErrActionNotFound), msg, params...)
}

func NewTenantNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrTenantNotFound], string(ErrTenantNotFound), msg, params...)
}

func NewUserNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrUserNotFound], string(ErrUserNotFound), msg, params...)
}

func NewInvalidStatusError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrInvalidStatus], string(ErrInvalidStatus), msg, params...)
}

func NewInputParameterError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrInputParameter], string(ErrInputParameter), msg, params...)
}

func NewWeakPasswordError() *httputils.JSONClientError {
	msg := ("password must be 12 chars of at least one digit, letter, uppercase letter and punctuate")
	return httputils.NewJsonClientError(httpErrorCode[ErrWeakPassword], string(ErrWeakPassword), msg)
}

func NewMissingParameterError(paramName string) *httputils.JSONClientError {
	msg := "Missing parameter %s"
	return httputils.NewJsonClientError(httpErrorCode[ErrMissingParameter], string(ErrMissingParameter), msg, paramName)
}

func NewPolicyDefinitionError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrPolicyDefinition], string(ErrPolicyDefinition), msg, params...)
}

func NewInsufficientResourceError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrInsufficientResource], string(ErrInsufficientResource), msg, params...)
}

func NewOutOfQuotaError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrOutOfQuota], string(ErrOutOfQuota), msg, params...)
}

func NewOutOfRangeError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrOutOfRange], string(ErrOutOfRange), msg, params...)
}

func NewOutOfLimitError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrOutOfLimit], string(ErrOutOfLimit), msg, params...)
}

func NewNotSufficientPrivilegeError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrNotSufficientPrivilege], string(ErrNotSufficientPrivilege), msg, params...)
}

func NewUnsupportOperationError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrUnsupportedOperation], string(ErrUnsupportedOperation), msg, params...)
}

func NewNotSupportedError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrNotSupported], string(ErrNotSupported), msg, params...)
}

func NewNotEmptyError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrNotEmpty], string(ErrNotEmpty), msg, params...)
}

func NewBadRequestError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrBadRequest], string(ErrBadRequest), msg, params...)
}

func NewUnauthorizedError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrUnauthorized], string(ErrUnauthorized), msg, params...)
}

func NewInvalidCredentialError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrInvalidCredential], string(ErrInvalidCredential), msg, params...)
}

func NewForbiddenError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrForbidden], string(ErrForbidden), msg, params...)
}

func NewNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrNotFound], string(ErrNotFound), msg, params...)
}

func NewNotAcceptableError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrNotAcceptable], string(ErrNotAcceptable), msg, params...)
}

func NewDuplicateNameError(resName string, resId string) *httputils.JSONClientError {
	msg := "Duplicate name %s %s"
	return httputils.NewJsonClientError(httpErrorCode[ErrDuplicateName], string(ErrDuplicateName), msg, resName, resId)
}

func NewDuplicateIdError(resName string, resId string) *httputils.JSONClientError {
	msg := "Duplicate ID %s %s"
	return httputils.NewJsonClientError(httpErrorCode[ErrDuplicateId], string(ErrDuplicateId), msg, resName, resId)
}

func NewDuplicateResourceError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrDuplicateResource], string(ErrDuplicateResource), msg, params...)
}

func NewConflictError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrConflict], string(ErrConflict), msg, params...)
}

func NewResourceBusyError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrResourceBusy], string(ErrResourceBusy), msg, params...)
}

func NewRequireLicenseError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrRequireLicense], string(ErrRequireLicense), msg, params...)
}

func NewTimeoutError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrTimeout], string(ErrTimeout), msg, params...)
}

func NewProtectedResourceError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrProtectedResource], string(ErrProtectedResource), msg, params...)
}

func NewNoProjectError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrNoProject], string(ErrNoProject), msg, params...)
}

func NewServerError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[errors.ErrServer], string(errors.ErrServer), msg, params...)
}

func NewClientError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[errors.ErrClient], string(errors.ErrClient), msg, params...)
}

func NewUnclassifiedError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[errors.ErrUnclassified], string(errors.ErrUnclassified), msg, params...)
}

func NewTooLargeEntityError(msg string, params ...interface{}) *httputils.JSONClientError {
	return httputils.NewJsonClientError(httpErrorCode[ErrTooLarge], string(ErrTooLarge), msg, params...)
}

func NewJsonClientError(err errors.Error, msg string, params ...interface{}) *httputils.JSONClientError {
	code, ok := httpErrorCode[err]
	if !ok {
		code = 500
	}
	return httputils.NewJsonClientError(code, string(err), msg, params...)
}


<|start_filename|>pkg/multicloud/apsara/host.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apsara

import (
	"fmt"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/multicloud"
	"yunion.io/x/onecloud/pkg/util/billing"
)

type SHost struct {
	multicloud.SHostBase
	zone *SZone
}

func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) {
	return self.zone.GetIWires()
}

func (self *SHost) GetIStorages() ([]cloudprovider.ICloudStorage, error) {
	return self.zone.GetIStorages()
}

func (self *SHost) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) {
	return self.zone.GetIStorageById(id)
}

func (self *SHost) GetIVMs() ([]cloudprovider.ICloudVM, error) {
	vms := make([]SInstance, 0)
	for {
		parts, total, err := self.zone.region.GetInstances(self.zone.ZoneId, nil, len(vms), 50)
		if err != nil {
			return nil, err
		}
		vms = append(vms, parts...)
		if len(vms) >= total {
			break
		}
	}
	ivms := make([]cloudprovider.ICloudVM, len(vms))
	for i := 0; i < len(vms); i += 1 {
		vms[i].host = self
		ivms[i] = &vms[i]
	}
	return ivms, nil
}

func (self *SHost) VMGlobalId2Id(gid string) string {
	return gid
}

func (self *SHost) GetIVMById(gid string) (cloudprovider.ICloudVM, error) {
	id := self.VMGlobalId2Id(gid)
	parts, _, err := self.zone.region.GetInstances(self.zone.ZoneId, []string{id}, 0, 1)
	if err != nil {
		return nil, err
	}
	if len(parts) == 0 {
		return nil, cloudprovider.ErrNotFound
	}
	if len(parts) > 1 {
		return nil, cloudprovider.ErrDuplicateId
	}
	parts[0].host = self
	return &parts[0], nil
}

func (self *SHost) GetId() string {
	return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId())
}

func (self *SHost) GetName() string {
	return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId())
}

func (self *SHost) GetGlobalId() string {
	return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId())
}

func (self *SHost) IsEmulated() bool {
	return true
}

func (self *SHost) GetStatus() string {
	return api.HOST_STATUS_RUNNING
}

func (self *SHost) Refresh() error {
	return nil
}

func (self *SHost) GetHostStatus() string {
	return api.HOST_ONLINE
}

func (self *SHost) GetEnabled() bool {
	return true
}

func (self *SHost) GetAccessIp() string {
	return ""
}

func (self *SHost) GetAccessMac() string {
	return ""
}

func (self *SHost) GetSysInfo() jsonutils.JSONObject {
	info := jsonutils.NewDict()
	info.Add(jsonutils.NewString(CLOUD_PROVIDER_APSARA), "manufacture")
	return info
}

func (self *SHost) GetSN() string {
	return ""
}

func (self *SHost) GetCpuCount() int {
	return 0
}

func (self *SHost) GetNodeCount() int8 {
	return 0
}

func (self *SHost) GetCpuDesc() string {
	return ""
}

func (self *SHost) GetCpuMhz() int {
	return 0
}

func (self *SHost) GetMemSizeMB() int {
	return 0
}

func (self *SHost) GetStorageSizeMB() int {
	return 0
}

func (self *SHost) GetStorageType() string {
	return api.DISK_TYPE_HYBRID
}

func (self *SHost) GetHostType() string {
	return api.HOST_TYPE_APSARA
}

func (self *SHost) GetInstanceById(instanceId string) (*SInstance, error) {
	inst, err := self.zone.region.GetInstance(instanceId)
	if err != nil {
		return nil, err
	}
	inst.host = self
	return inst, nil
}

func (self *SHost) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (cloudprovider.ICloudVM, error) {
	vmId, err := self._createVM(desc.Name, desc.ExternalImageId, desc.SysDisk, desc.Cpu, desc.MemoryMB,
		desc.InstanceType, desc.ExternalNetworkId, desc.IpAddr, desc.Description, desc.Password,
		desc.DataDisks, desc.PublicKey, desc.ExternalSecgroupId, desc.UserData, desc.BillingCycle,
		desc.ProjectId, desc.OsType, desc.Tags)
	if err != nil {
		return nil, err
	}
	vm, err := self.GetInstanceById(vmId)
	if err != nil {
		return nil, err
	}
	// err = vm.waitStatus(InstanceStatusStopped, time.Second*10, time.Second*1800)
	return vm, err
}

func (self *SHost) _createVM(name string, imgId string,
	sysDisk cloudprovider.SDiskInfo, cpu int, memMB int, instanceType string,
	vswitchId string, ipAddr string, desc string, passwd string,
	dataDisks []cloudprovider.SDiskInfo, publicKey string, secgroupId string,
	userData string, bc *billing.SBillingCycle, projectId, osType string,
	tags map[string]string,
) (string, error) {
	net := self.zone.getNetworkById(vswitchId)
	if net == nil {
		return "", fmt.Errorf("invalid switch ID %s", vswitchId)
	}
	if net.wire == nil {
		log.Errorf("vsiwtch's wire is empty")
		return "", fmt.Errorf("vsiwtch's wire is empty")
	}
	if net.wire.vpc == nil {
		log.Errorf("vsiwtch's wire' vpc is empty")
		return "", fmt.Errorf("vsiwtch's wire's vpc is empty")
	}

	var err error
	keypair := ""
	if len(publicKey) > 0 {
		keypair, err = self.zone.region.syncKeypair(publicKey)
		if err != nil {
			return "", err
		}
	}

	img, err := self.zone.region.GetImage(imgId)
	if err != nil {
		log.Errorf("GetImage fail %s", err)
		return "", err
	}
	if img.Status != ImageStatusAvailable {
		log.Errorf("image %s status %s", imgId, img.Status)
		return "", fmt.Errorf("image not ready")
	}

	disks := make([]SDisk, len(dataDisks)+1)
	disks[0].Size = img.Size
	if sysDisk.SizeGB > 0 && sysDisk.SizeGB > img.Size {
		disks[0].Size = sysDisk.SizeGB
	}
	storage, err := self.zone.getStorageByCategory(sysDisk.StorageType)
	if err != nil {
		return "", fmt.Errorf("Storage %s not avaiable: %s", sysDisk.StorageType, err)
	}
	disks[0].Category = storage.storageType

	for i, dataDisk := range dataDisks {
		disks[i+1].Size = dataDisk.SizeGB
		storage, err := self.zone.getStorageByCategory(dataDisk.StorageType)
		if err != nil {
			return "", fmt.Errorf("Storage %s not avaiable: %s", dataDisk.StorageType, err)
		}
		disks[i+1].Category = storage.storageType
	}

	if len(instanceType) > 0 {
		log.Debugf("Try instancetype : %s", instanceType)
		vmId, err := self.zone.region.CreateInstance(name, imgId, instanceType, secgroupId, self.zone.ZoneId, desc, passwd, disks, vswitchId, ipAddr, keypair, userData, bc, projectId, osType, tags)
		if err != nil {
			log.Errorf("Failed for %s: %s", instanceType, err)
			return "", fmt.Errorf("Failed to create specification %s.%s", instanceType, err.Error())
		}
		return vmId, nil
	}

	instanceTypes, err := self.zone.region.GetMatchInstanceTypes(cpu, memMB, 0, self.zone.ZoneId)
	if err != nil {
		return "", err
	}
	if len(instanceTypes) == 0 {
		return "", fmt.Errorf("instance type %dC%dMB not avaiable", cpu, memMB)
	}

	var vmId string
	for _, instType := range instanceTypes {
		instanceTypeId := instType.InstanceTypeId
		log.Debugf("Try instancetype : %s", instanceTypeId)
		vmId, err = self.zone.region.CreateInstance(name, imgId, instanceTypeId, secgroupId, self.zone.ZoneId, desc, passwd, disks, vswitchId, ipAddr, keypair, userData, bc, projectId, osType, tags)
		if err != nil {
			log.Errorf("Failed for %s: %s", instanceTypeId, err)
		} else {
			return vmId, nil
		}
	}

	return "", fmt.Errorf("Failed to create, %s", err.Error())
}

func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) {
	return nil, cloudprovider.ErrNotSupported
}

func (host *SHost) GetIsMaintenance() bool {
	return false
}

func (host *SHost) GetVersion() string {
	return APSARA_API_VERSION
}


<|start_filename|>cmd/raidcli/main.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"

	"yunion.io/x/structarg"

	_ "yunion.io/x/onecloud/cmd/raidcli/shell"
	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
	"yunion.io/x/onecloud/pkg/util/shellutils"
	"yunion.io/x/onecloud/pkg/util/ssh"
)

type BaseOptions struct {
	Debug      bool   `help:"debug mode"`
	Help       bool   `help:"Show help"`
	Host       string `help:"SSH Host IP" default:"$RAID_HOST" metavar:"RAID_HOST"`
	Username   string `help:"Username, usually root" default:"$RAID_USERNAME" metavar:"RAID_USERNAME"`
	Password   string `help:"Password" default:"<PASSWORD>" metavar:"RAID_PASSWORD"`
	Driver     string `help:"Raid dirver" default:"$RAID_DRIVER" metavar:"RAID_DRIVER" choices:"MegaRaid|HPSARaid|Mpt2SAS|MarvelRaid"`
	LocalHost  bool   `help:"Run raidcli in localhost"`
	SUBCOMMAND string `help:"s3cli subcommand" subcommand:"true"`
}

var (
	options = &BaseOptions{}
)

func getSubcommandParser() (*structarg.ArgumentParser, error) {
	parse, e := structarg.NewArgumentParser(options,
		"raidcli",
		"Command-line interface to test RAID drivers.",
		`See "raidcli help COMMAND" for help on a specific command.`)

	if e != nil {
		return nil, e
	}

	subcmd := parse.GetSubcommand()
	if subcmd == nil {
		return nil, fmt.Errorf("No subcommand argument.")
	}
	type HelpOptions struct {
		SUBCOMMAND string `help:"sub-command name"`
	}
	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
		if e != nil {
			return e
		} else {
			fmt.Print(helpstr)
			return nil
		}
	})
	for _, v := range shellutils.CommandTable {
		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
		if e != nil {
			return nil, e
		}
	}
	return parse, nil
}

func showErrorAndExit(e error) {
	fmt.Fprintf(os.Stderr, "%s", e)
	fmt.Fprintln(os.Stderr)
	os.Exit(1)
}

func newClient() (raid.IRaidDriver, error) {
	if options.Debug {
		raid.Debug = true
	}

	if len(options.Driver) == 0 {
		return nil, fmt.Errorf("Missing driver")
	}

	var drv raid.IRaidDriver
	if !options.LocalHost {
		if len(options.Host) == 0 {
			return nil, fmt.Errorf("Missing host")
		}

		if len(options.Username) == 0 {
			return nil, fmt.Errorf("Missing username")
		}

		if len(options.Password) == 0 {
			return nil, fmt.Errorf("Missing password")
		}

		sshClient, err := ssh.NewClient(
			options.Host,
			22,
			options.Username,
			options.Password,
			"",
		)
		if err != nil {
			return nil, fmt.Errorf("ssh client init fail: %s", err)
		}

		drv = drivers.GetDriver(options.Driver, sshClient)
		if drv == nil {
			return nil, fmt.Errorf("not supported driver %s", options.Driver)
		}
	} else {
		drv = drivers.GetLocalDriver(options.Driver)
	}

	err := drv.ParsePhyDevs()
	if err != nil {
		return nil, fmt.Errorf("parse phyical devices error %s", err)
	}

	return drv, nil
}

func main() {
	parser, e := getSubcommandParser()
	if e != nil {
		showErrorAndExit(e)
	}
	e = parser.ParseArgs(os.Args[1:], false)
	// options := parser.Options().(*BaseOptions)

	if options.Help {
		fmt.Print(parser.HelpString())
	} else {
		subcmd := parser.GetSubcommand()
		subparser := subcmd.GetSubParser()
		if e != nil {
			if subparser != nil {
				fmt.Print(subparser.Usage())
			} else {
				fmt.Print(parser.Usage())
			}
			showErrorAndExit(e)
		} else {
			suboptions := subparser.Options()
			if options.SUBCOMMAND == "help" {
				e = subcmd.Invoke(suboptions)
			} else {
				var client raid.IRaidDriver
				client, e = newClient()
				if e != nil {
					showErrorAndExit(e)
				}
				e = subcmd.Invoke(client, suboptions)
			}
			if e != nil {
				showErrorAndExit(e)
			}
		}
	}
}


<|start_filename|>pkg/compute/tasks/dnszone_sync_vpcs_task.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tasks

import (
	"context"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/utils"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/cloudprovider"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type DnsZoneSyncVpcsTask struct {
	taskman.STask
}

func init() {
	taskman.RegisterTask(DnsZoneSyncVpcsTask{})
}

func (self *DnsZoneSyncVpcsTask) taskFailed(ctx context.Context, dnsZone *models.SDnsZone, err error) {
	dnsZone.SetStatus(self.GetUserCred(), api.DNS_ZONE_STATUS_SYNC_VPCS_FAILED, err.Error())
	db.OpsLog.LogEvent(dnsZone, db.ACT_SYNC_VPCS, dnsZone.GetShortDesc(ctx), self.GetUserCred())
	logclient.AddActionLogWithContext(ctx, dnsZone, logclient.ACT_SYNC_VPCS, err, self.UserCred, false)
	self.SetStageFailed(ctx, jsonutils.NewString(err.Error()))
}

func (self *DnsZoneSyncVpcsTask) taskComplete(ctx context.Context, dnsZone *models.SDnsZone) {
	db.OpsLog.LogEvent(dnsZone, db.ACT_SYNC_VPCS, dnsZone.GetShortDesc(ctx), self.GetUserCred())
	logclient.AddActionLogWithContext(ctx, dnsZone, logclient.ACT_SYNC_VPCS, "", self.UserCred, true)
	dnsZone.SetStatus(self.GetUserCred(), api.DNS_ZONE_STATUS_AVAILABLE, "")
	self.SetStageComplete(ctx, nil)
}

func (self *DnsZoneSyncVpcsTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	dnsZone := obj.(*models.SDnsZone)

	accounts, err := dnsZone.GetCloudaccounts()
	if err != nil {
		self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "GetCloudaccounts"))
		return
	}
	accountIds := []string{}
	for i := range accounts {
		cache, err := dnsZone.RegisterCache(ctx, self.GetUserCred(), accounts[i].Id)
		if err != nil {
			self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "dnsZone.RegisterCache"))
			return
		}
		provider, err := cache.GetProvider()
		if err != nil {
			self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "GetProvider"))
			return
		}

		if len(cache.ExternalId) == 0 {
			opts := cloudprovider.SDnsZoneCreateOptions{
				Name:     dnsZone.Name,
				Desc:     dnsZone.Description,
				ZoneType: cloudprovider.TDnsZoneType(dnsZone.ZoneType),
				Options:  dnsZone.Options,
			}
			vpcs, err := cache.GetVpcs()
			if err != nil {
				self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "GetVpcs"))
				return
			}
			for _, vpc := range vpcs {
				iVpc, err := vpc.GetIVpc()
				if err != nil {
					self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "GetIVpc for vpc %s", vpc.Name))
					return
				}
				opts.Vpcs = append(opts.Vpcs, cloudprovider.SPrivateZoneVpc{
					Id:       iVpc.GetGlobalId(),
					RegionId: iVpc.GetRegion().GetId(),
				})
			}
			iDnsZone, err := provider.CreateICloudDnsZone(&opts)
			if err != nil {
				self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "CreateICloudDnsZone"))
				return
			}
			err = cache.SyncWithCloudDnsZone(ctx, self.GetUserCred(), iDnsZone)
			if err != nil {
				self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "SyncWithCloudDnsZone"))
				return
			}
		} else {
			err := cache.SyncVpcForCloud(ctx, self.GetUserCred())
			if err != nil {
				self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "SyncVpcForCloud"))
				return
			}
		}
		accountIds = append(accountIds, accounts[i].Id)
	}

	caches, err := dnsZone.GetDnsZoneCaches()
	if err != nil {
		self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "GetDnsZoneCaches"))
		return
	}

	for i := range caches {
		lockman.LockObject(ctx, &caches[i])
		defer lockman.ReleaseObject(ctx, &caches[i])

		if !utils.IsInStringArray(caches[i].CloudaccountId, accountIds) {
			if len(caches[i].ExternalId) > 0 {
				iDnsZone, err := caches[i].GetICloudDnsZone()
				if err != nil {
					if errors.Cause(err) != cloudprovider.ErrNotFound {
						self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "GetICloudDnsZone"))
						return
					}
				} else {
					err = iDnsZone.Delete()
					if err != nil {
						self.taskFailed(ctx, dnsZone, errors.Wrapf(err, "iDnsZone.Delete"))
						return
					}
				}
				caches[i].RealDelete(ctx, self.GetUserCred())
			}
		}
	}

	self.SetStage("OnSyncRecordSetComplete", nil)
	dnsZone.StartDnsZoneSyncRecordSetsTask(ctx, self.GetUserCred(), self.GetTaskId())
}

func (self *DnsZoneSyncVpcsTask) OnSyncRecordSetComplete(ctx context.Context, dnsZone *models.SDnsZone, data jsonutils.JSONObject) {
	self.taskComplete(ctx, dnsZone)
}

func (self *DnsZoneSyncVpcsTask) OnSyncRecordSetCompleteFailed(ctx context.Context, cache *models.SDnsZoneCache, data jsonutils.JSONObject) {
	self.SetStageFailed(ctx, data)
}


<|start_filename|>cmd/climc/shell/compute/schedtagresources.go<|end_filename|>
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compute

import (
	"fmt"

	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
	"yunion.io/x/onecloud/pkg/mcclient/modules"
	"yunion.io/x/onecloud/pkg/mcclient/options"
)

type schedtagModelHelper struct {
	managers []modulebase.JointResourceManager
}

func newSchedtagModelHelper(mans ...modulebase.JointResourceManager) *schedtagModelHelper {
	return &schedtagModelHelper{managers: mans}
}

func (h *schedtagModelHelper) register() {
	for _, man := range h.managers {
		h.list(man.Slave, man.Slave.GetKeyword())
		h.add(man, man.Slave.GetKeyword())
		h.remove(man, man.Slave.GetKeyword())
		h.setTags(man, man.Slave.GetKeyword())
	}
}

func (h *schedtagModelHelper) list(slave modulebase.Manager, kw string) {
	R(
		&options.SchedtagModelListOptions{},
		fmt.Sprintf("schedtag-%s-list", kw),
		fmt.Sprintf("List all scheduler tag and %s pairs", kw),
		func(s *mcclient.ClientSession, args *options.SchedtagModelListOptions) error {
			mod, err := modulebase.GetJointModule2(s, &modules.Schedtags, slave)
			if err != nil {
				return err
			}
			params, err := args.Params()
			if err != nil {
				return err
			}
			var result *modulebase.ListResult
			if len(args.Schedtag) > 0 {
				result, err = mod.ListDescendent(s, args.Schedtag, params)
			} else {
				result, err = mod.List(s, params)
			}
			if err != nil {
				return err
			}
			printList(result, mod.GetColumns(s))
			return nil
		},
	)
}

func (h *schedtagModelHelper) add(man modulebase.JointResourceManager, kw string) {
	R(
		&options.SchedtagModelPairOptions{},
		fmt.Sprintf("schedtag-%s-add", kw),
		fmt.Sprintf("Add a schedtag to a %s", kw),
		func(s *mcclient.ClientSession, args *options.SchedtagModelPairOptions) error {
			schedtag, err := man.Attach(s, args.SCHEDTAG, args.OBJECT, nil)
			if err != nil {
				return err
			}
			printObject(schedtag)
			return nil
		})
}

func (h *schedtagModelHelper) remove(man modulebase.JointResourceManager, kw string) {
	R(
		&options.SchedtagModelPairOptions{},
		fmt.Sprintf("schedtag-%s-remove", kw),
		fmt.Sprintf("Remove a schedtag to a %s", kw),
		func(s *mcclient.ClientSession, args *options.SchedtagModelPairOptions) error {
			schedtag, err := man.Detach(s, args.SCHEDTAG, args.OBJECT, nil)
			if err != nil {
				return err
			}
			printObject(schedtag)
			return nil
		})
}

func (h *schedtagModelHelper) setTags(man modulebase.JointResourceManager, kw string) {
	R(
		&options.SchedtagSetOptions{},
		fmt.Sprintf("%s-set-schedtag", kw),
		fmt.Sprintf("Set schedtags to %v", kw),
		func(s *mcclient.ClientSession, args *options.SchedtagSetOptions) error {
			params, err := args.Params()
			if err != nil {
				return err
			}
			ret, err := man.Slave.PerformAction(s, args.ID, "set-schedtag", params)
			if err != nil {
				return err
			}
			printObject(ret)
			return nil
		})
}

func init() {
	newSchedtagModelHelper(
		modules.Schedtaghosts,
		modules.Schedtagstorages,
		modules.Schedtagnetworks,
		modules.Schedtagcloudproviders,
		modules.Schedtagcloudregions,
		modules.Schedtagzones,
	).register()
}
