text:
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include <fstream>
#include <map>
#include <utility>

#include "core/common/common.h"
#include "core/framework/execution_provider.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/kernel_registry.h"
#include "core/framework/compute_capability.h"
#include "core/graph/graph_proto_serializer.h"
#include "core/platform/env.h"
#include "core/graph/model.h"

#include "tvm_execution_provider.h"
#include "xpu_data_transfer.h"
#include "tvm_allocator.h"
#include "tvm_utils.h"
#include "tvm_api.h"

using namespace ONNX_NAMESPACE;

namespace onnxruntime {
namespace tvm {

// Information to construct kernel function state.
struct TVMFuncState {
  AllocateFunc allocate_func = nullptr;
  DestroyFunc release_func = nullptr;
  AllocatorHandle allocator = nullptr;
  std::shared_ptr<TVMCompilerBase> compiler = nullptr;
};

TvmExecutionProvider::TvmExecutionProvider(const TvmEPOptions& options)
    : IExecutionProvider{kTvmExecutionProvider},
      options_{options} {
  AllocatorCreationInfo default_memory_info = {[](int) {
                                                 return std::make_unique<TVMAllocator>();
                                               },
                                               0, false};
  allocator_ = CreateAllocator(default_memory_info);
  InsertAllocator(allocator_);

  // Get environment variables
  const Env& env_instance = Env::Default();

  const std::string dump_subgraphs_env = env_instance.GetEnvironmentVar(env_vars::kDumpSubgraphs);
  if (!dump_subgraphs_env.empty()) {
    dump_subgraphs_ = std::stoi(dump_subgraphs_env) != 0;
  }
}

TvmExecutionProvider::~TvmExecutionProvider() {}

std::vector<std::unique_ptr<ComputeCapability>>
TvmExecutionProvider::GetCapability(const GraphViewer& graph_viewer,
                                    const IKernelLookup& /*kernel_lookup*/) const {
  std::vector<std::unique_ptr<ComputeCapability>> result;
  if (graph_viewer.IsSubgraph()) {
    return result;
  }

  const auto& init_tensors = graph_viewer.GetAllInitializedTensors();

  std::unordered_set<std::string> required_initializers;
  const std::vector<NodeIndex>& sorted_nodes = graph_viewer.GetNodesInTopologicalOrder();
  std::unique_ptr<IndexedSubGraph> sub_graph = std::make_unique<IndexedSubGraph>();
  for (auto& node_idx : sorted_nodes) {
    graph_viewer.GetNode(node_idx)->ForEachDef([&required_initializers, &init_tensors]
                                               (const NodeArg& node_arg, bool is_input) {
              if(is_input && init_tensors.count(node_arg.Name())) {
                  required_initializers.insert(node_arg.Name());
              } }, true);
  }

  auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>();
  meta_def->name = "TVMStandalone";
  meta_def->domain = "StandaloneTest";
  std::vector<std::string> inputs;
  std::vector<std::string> outputs;

  for (auto& nodeArgPtr : graph_viewer.GetInputs()) {
    inputs.push_back(nodeArgPtr->Name());
  }

  for (auto& name : required_initializers) {
    inputs.push_back(name);
  }

  for (auto& nodeArgPtr : graph_viewer.GetOutputs()) {
    outputs.push_back(nodeArgPtr->Name());
  }
  meta_def->inputs = inputs;
  meta_def->outputs = outputs;
  meta_def->since_version = 1;
  meta_def->status = ONNX_NAMESPACE::EXPERIMENTAL;
  sub_graph->SetMetaDef(std::move(meta_def));
  sub_graph->nodes = sorted_nodes;
  result.push_back(
      std::make_unique<ComputeCapability>(std::move(sub_graph)));
  return result;
}

common::Status TvmExecutionProvider::Compile(const std::vector<FusedNodeAndGraph>& fused_nodes_and_graphs,
                                             std::vector<NodeComputeInfo>& node_compute_funcs) {
  printOptions();
  for (auto& fused_node_graph : fused_nodes_and_graphs) {
    const GraphViewer& graph_body_viewer = fused_node_graph.filtered_graph;
    const Node& fused_node = fused_node_graph.fused_node;
    const std::string func_name = fused_node.Name();
    Model model(graph_body_viewer.Name(), true, ModelMetaData(), PathString(),
                IOnnxRuntimeOpSchemaRegistryList(), graph_body_viewer.DomainToVersionMap(),
                             std::vector<ONNX_NAMESPACE::FunctionProto>(), *GetLogger());
    ONNX_NAMESPACE::ModelProto model_proto = model.ToProto();
    // TVM EP is using static lib approach, so invoke serializer directly.
    GraphViewerToProto(graph_body_viewer, *model_proto.mutable_graph(), true, true);
    auto opset = model_proto.add_opset_import();
    opset->set_domain(kOnnxDomain);
    opset->set_version(graph_body_viewer.DomainToVersionMap().at(kOnnxDomain));

    std::string onnx_model_str;
    model_proto.SerializeToString(&onnx_model_str);
    compilers_[func_name] = std::make_shared<TVMCompiler>(std::move(onnx_model_str),
                              ToUTF8String(fused_node.ModelPath().ToPathString()),
                              int(opset->version()));
    InputsInfoMap all_input_shapes;
    auto mod = compileModel(func_name, graph_body_viewer, all_input_shapes);

    std::vector<DLTensor> output_tensors;
    prepareOutputTensors(mod, output_tensors, graph_body_viewer.GetOutputs().size());

    runners_[func_name] = std::make_shared<Runner>(options_, mod, all_input_shapes, output_tensors);

    if (dump_subgraphs_) {
        std::fstream dump("/tmp/" + func_name + ".onnx",
                          std::ios::out | std::ios::trunc | std::ios::binary);
        model_proto.SerializeToOstream(&dump);
    }

    // TODO(vvchernov): implement ops checking and mechanism of gracefully passing the responsibility to other EPs
    // if the checking fails due to unsupported op(s)
    NodeComputeInfo compute_info = prepareComputeInfo(func_name);

    node_compute_funcs.push_back(compute_info);
  }
  return Status::OK();
}

std::unique_ptr<IDataTransfer> TvmExecutionProvider::GetDataTransfer() const {
  //TODO(vvchernov): target or target host?
  if (TvmEPOptionsHelper::checkGPUTarget(options_.target)) {
    return std::make_unique<XPUDataTransfer>();
  } else if (TvmEPOptionsHelper::checkCPUTarget(options_.target)) {
    return std::make_unique<TvmCPUDataTransfer>();
  } else {
    ORT_NOT_IMPLEMENTED("TVM GetDataTransfer is not implemented for target ", options_.target);
  }
}

AllocatorPtr TvmExecutionProvider::GetAllocator(int id, OrtMemType mem_type) const {
  return allocator_;
}

void TvmExecutionProvider::printOptions() {
  LOGS(*GetLogger(), INFO) << options_;
}

std::shared_ptr<TvmModule> TvmExecutionProvider::compileModel(const std::string& func_name,
                                                              const GraphViewer& graph_viewer,
                                                              InputsInfoMap& all_input_shapes) {
  all_input_shapes.clear();

  TVMTensorShapes input_shapes;
  if (options_.freeze_weights) {
    setInputShapesForFreezedNN(graph_viewer, input_shapes, all_input_shapes);
  } else {
    setInputShapesForUnfreezedNN(graph_viewer, input_shapes, all_input_shapes);
  }

  std::shared_ptr<TvmModule> mod = compilers_[func_name]->operator()(options_, input_shapes);

  return mod;
}

void TvmExecutionProvider::setInputShapesForFreezedNN(const GraphViewer& graph_viewer,
                                                      TVMTensorShapes& input_shapes,
                                                      InputsInfoMap& all_input_shapes) {
  const std::vector<const NodeArg*>& all_nodes = graph_viewer.GetInputsIncludingInitializers();

  size_t indx = 0;
  for (const auto* node : all_nodes) {
    if (!graph_viewer.IsInitializedTensor(node->Name())) {
      TensorShapeVector shape = getInputShape(node);
      all_input_shapes[indx++] = shape;
      input_shapes.emplace_back(shape);
    }
  }
}

void TvmExecutionProvider::setInputShapesForUnfreezedNN(const GraphViewer& graph_viewer,
                                                        TVMTensorShapes& input_shapes,
                                                        InputsInfoMap& all_input_shapes) {
  const std::vector<const NodeArg*>& all_nodes = graph_viewer.GetInputsIncludingInitializers();

  size_t indx = 0;
  for (const auto* node : all_nodes) {
    TensorShapeVector shape = getInputShape(node);
    all_input_shapes[indx++] = shape;
    if (!graph_viewer.IsInitializedTensor(node->Name())) {
      input_shapes.emplace_back(shape);
    }
  }
}

TensorShapeVector TvmExecutionProvider::getInputShape(const NodeArg* node) {
    TensorShapeVector shape;
    const auto& node_name = node->Name();
    if(!options_.input_shapes.empty() &&
        options_.input_shapes.count(node_name)) {
      shape = options_.input_shapes[node_name];
    } else {
      shape = convertTensorShape(*node->Shape());
    }

    return shape;
}

TensorShapeVector TvmExecutionProvider::convertTensorShape(const TensorShapeProto& shape_proto) {
  TensorShape ort_shape = utils::GetTensorShapeFromTensorShapeProto(shape_proto);
  size_t dims = ort_shape.NumDimensions();

  TensorShapeVector shape(dims);
  for (size_t j = 0; j < dims; ++j) {
    int64_t dim = int64_t(ort_shape[j]);
    ORT_ENFORCE(dim > 0, "Input dimension is not positive value (dim = " + std::to_string(dim) + "). " +
      "Please use provider options to setup input_names and input_shapes");
    shape[j] = dim;
  }

  return shape;
}

void TvmExecutionProvider::prepareOutputTensors(const std::shared_ptr<TvmModule>& mod,
                                                std::vector<DLTensor>& output_tensors,
                                                size_t num) {
  ORT_ENFORCE(mod != nullptr, "TVM module is not compiled");
  output_tensors.clear();
  options_.output_shapes.clear();
  options_.output_shapes.resize(num);

  if (options_.executor != "vm") {
    TVMGetOutputShapes(*mod, options_.output_shapes);
  }

  for (auto& output_shape : options_.output_shapes) {
    DLTensor t;
    // Draft for tensor, correct data is defined during inference
    t.strides = nullptr;
    t.byte_offset = 0;
    t.data = nullptr;
    if (options_.executor == "vm") {
      t.ndim = 0;
      t.shape = nullptr;
    } else {
      t.ndim = output_shape.size();
      t.shape = output_shape.data();
    }

    output_tensors.push_back(t);
  }
}

NodeComputeInfo TvmExecutionProvider::prepareComputeInfo(const std::string& func_name) {
  NodeComputeInfo compute_info;
  compute_info.create_state_func = std::bind(&TvmExecutionProvider::createStateFunc,
                                              this,
                                              std::placeholders::_1,
                                              std::placeholders::_2);

  compute_info.release_state_func = [](FunctionState state) {
    if (state)
      delete static_cast<TVMFuncState*>(state);
  };

  compute_info.compute_func = *runners_[func_name].get();

  return compute_info;
}

int TvmExecutionProvider::createStateFunc(ComputeContext* context, FunctionState* state) {
  auto* state_ptr = new TVMFuncState();
  *state_ptr = {context->allocate_func,
                context->release_func,
                context->allocator_handle,
                compilers_[context->node_name]};
  // TODO(vvchernov): Who and when release state?
  *state = state_ptr;
  return 0;
}

}  // namespace tvm
}  // namespace onnxruntime


text:
from gevent import monkey; monkey.patch_all()
from ws4py.websocket import EchoWebSocket
from ws4py.server.geventserver import WSGIServer
from ws4py.server.wsgiutils import WebSocketWSGIApplication

server = WSGIServer(('localhost', 9000), WebSocketWSGIApplication(handler_cls=EchoWebSocket))
server.serve_forever()

redpajama_set_name: RedPajamaGithub


text:
import * as mn from 'marionette';

mn.Application.prototype._initChannel = function() {};


text:
<!DOCTYPE frameset SYSTEM "frameset.dtd">
<frameset>
    <predicate lemma="attribute">
        <note>
            Frames file for 'attribute' based on survey of initial sentences of
            big corpus and comparison with 'give' (really!)
        </note>

        <roleset id="attribute.01" name="assign attribute to">
            <roles>
                <role descr="attributor" n="0"/>
                <role descr="thing attributed, attribute" n="1"/>
                <role descr="attributed to (benefactive?)" n="2"/>
            </roles>

            <example name="typical usage">
                <text>
                    Mr. Hahn attributes the gains to the philosophy of concentrating on
                    what a company knows best.
                </text>
                <arg n="0">Mr. Hahn</arg>
                <rel>attributes</rel>
                <arg n="1">the gains</arg>
                <arg n="2">to the philosophy of concentrating on
                    what a company knows best.
                </arg>
            </example>

            <note>
                This might be the only verb (so far) where the attribute comes before
                the entity with the attribute.  Curious.
            </note>

        </roleset>
    </predicate>
</frameset>


text:
package iso20022

// Provides the expected collateral type and direction for the variation margin and the segregated independent amount, or the segregated independent amount only.
type ExpectedCollateral2Choice struct {

	// Provides the expected collateral type and direction for the variation margin and optionaly the segregated independent amount.
	ExpectedCollateralDetails *ExpectedCollateral2 `xml:"XpctdCollDtls"`

	// Provides the expected collateral type and direction for the segregated independent amount.
	SegregatedIndependentAmount *ExpectedCollateralMovement2 `xml:"SgrtdIndpdntAmt"`
}

func (e *ExpectedCollateral2Choice) AddExpectedCollateralDetails() *ExpectedCollateral2 {
	e.ExpectedCollateralDetails = new(ExpectedCollateral2)
	return e.ExpectedCollateralDetails
}

func (e *ExpectedCollateral2Choice) AddSegregatedIndependentAmount() *ExpectedCollateralMovement2 {
	e.SegregatedIndependentAmount = new(ExpectedCollateralMovement2)
	return e.SegregatedIndependentAmount
}


text:
"""
conference.py -- Udacity conference server-side Python App Engine API;
    uses Google Cloud Endpoints

$Id: conference.py,v 1.25 2014/05/24 23:42:19 wesc Exp wesc $

created by wesc on 2014 apr 21

"""

__author__ = 'wesc+api@google.com (Wesley Chun)'

from datetime import datetime, timedelta, time as timed
import logging

import endpoints
from protorpc import messages
from protorpc import message_types
from protorpc import remote

from google.appengine.api import memcache
from google.appengine.api import taskqueue
from google.appengine.ext import ndb

from models import ConflictException
from models import Profile
from models import ProfileMiniForm
from models import ProfileForm
from models import StringMessage
from models import BooleanMessage
from models import Conference
from models import ConferenceForm
from models import ConferenceForms
from models import Session
from models import SessionForm
from models import SessionForms
from models import ConferenceQueryForm
from models import ConferenceQueryForms
from models import TeeShirtSize
from models import AddSessionToWishListForm

from utils import getUserId
from settings import WEB_CLIENT_ID

EMAIL_SCOPE = endpoints.EMAIL_SCOPE
API_EXPLORER_CLIENT_ID = endpoints.API_EXPLORER_CLIENT_ID
MEMCACHE_ANNOUNCEMENTS_KEY = "RECENT_ANNOUNCEMENTS"
MEMCACHE_FEATURED_SPEAKER_KEY = "FEATURED_SPEAKER"

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

DEFAULTS = {
    "city": "Default City",
    "maxAttendees": 0,
    "seatsAvailable": 0,
    "topics": [ "Default", "Topic" ],
}

OPERATORS = {
            'EQ':   '=',
            'GT':   '>',
            'GTEQ': '>=',
            'LT':   '<',
            'LTEQ': '<=',
            'NE':   '!='
            }

FIELDS =    {
            'CITY': 'city',
            'TOPIC': 'topics',
            'MONTH': 'month',
            'MAX_ATTENDEES': 'maxAttendees',
            }

CONF_GET_REQUEST = endpoints.ResourceContainer(
    message_types.VoidMessage,
    websafeConferenceKey=messages.StringField(1),
)

CONF_POST_REQUEST = endpoints.ResourceContainer(
    ConferenceForm,
    websafeConferenceKey=messages.StringField(1),
)

# Resource containers used in GET and Post Requests
SESSION_GET_REQUEST = endpoints.ResourceContainer(
    message_types.VoidMessage,
    websafeConferenceKey=messages.StringField(1, required=True),
    typeOfSession=messages.StringField(2)
)

SESSION_POST_REQUEST = endpoints.ResourceContainer(
    SessionForm,
    websafeConferenceKey=messages.StringField(1, required=True),
)

SPEAKER_GET_REQUEST = endpoints.ResourceContainer(
    message_types.VoidMessage,
    speaker=messages.StringField(1, required=True),
)

ADD_SESSION_TO_WISHLIST_REQUEST=endpoints.ResourceContainer(
    message_types.VoidMessage,
    sessionKey=messages.StringField(1,required=True)
)

REMOVE_SESSION_TO_WISHLIST_REQUEST=endpoints.ResourceContainer(
    message_types.VoidMessage,
    sessionKey=messages.StringField(1,required=True)
)

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

@endpoints.api(name='conference', version='v1',
    allowed_client_ids=[WEB_CLIENT_ID, API_EXPLORER_CLIENT_ID],
    scopes=[EMAIL_SCOPE])
class ConferenceApi(remote.Service):
    """Conference API v0.1"""

# - - - Conference objects - - - - - - - - - - - - - - - - -

    def _copyConferenceToForm(self, conf, displayName):
        """Copy relevant fields from Conference to ConferenceForm."""
        cf = ConferenceForm()
        for field in cf.all_fields():
            if hasattr(conf, field.name):
                # convert Date to date string; just copy others
                if field.name.endswith('Date'):
                    setattr(cf, field.name, str(getattr(conf, field.name)))
                else:
                    setattr(cf, field.name, getattr(conf, field.name))
            elif field.name == "websafeKey":
                setattr(cf, field.name, conf.key.urlsafe())
        if displayName:
            setattr(cf, 'organizerDisplayName', displayName)
        cf.check_initialized()
        return cf

    def _createConferenceObject(self, request):
        """Create or update Conference object, returning ConferenceForm/request."""
        # preload necessary data items
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')
        user_id = getUserId(user)

        if not request.name:
            raise endpoints.BadRequestException("Conference 'name' field required")

        # copy ConferenceForm/ProtoRPC Message into dict
        data = {field.name: getattr(request, field.name) for field in request.all_fields()}
        del data['websafeKey']
        del data['organizerDisplayName']

        # add default values for those missing (both data model & outbound Message)
        for df in DEFAULTS:
            if data[df] in (None, []):
                data[df] = DEFAULTS[df]
                setattr(request, df, DEFAULTS[df])

        # convert dates from strings to Date objects; set month based on start_date
        if data['startDate']:
            data['startDate'] = datetime.strptime(data['startDate'][:10], "%Y-%m-%d").date()
            data['month'] = data['startDate'].month
        else:
            data['month'] = 0
        if data['endDate']:
            data['endDate'] = datetime.strptime(data['endDate'][:10], "%Y-%m-%d").date()

        # set seatsAvailable to be same as maxAttendees on creation
        if data["maxAttendees"] > 0:
            data["seatsAvailable"] = data["maxAttendees"]
        # generate Profile Key based on user ID and Conference
        # ID based on Profile key get Conference key from ID
        p_key = ndb.Key(Profile, user_id)
        c_id = Conference.allocate_ids(size=1, parent=p_key)[0]
        c_key = ndb.Key(Conference, c_id, parent=p_key)
        data['key'] = c_key
        data['organizerUserId'] = request.organizerUserId = user_id

        # create Conference, send email to organizer confirming
        # creation of Conference & return (modified) ConferenceForm
        Conference(**data).put()
        taskqueue.add(params={'email': user.email(),
            'conferenceInfo': repr(request)},
            url='/tasks/send_confirmation_email'
        )

        return request

    @ndb.transactional()
    def _updateConferenceObject(self, request):
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')
        user_id = getUserId(user)

        # copy ConferenceForm/ProtoRPC Message into dict
        data = {field.name: getattr(request, field.name) for field in request.all_fields()}

        # update existing conference
        conf = ndb.Key(urlsafe=request.websafeConferenceKey).get()
        # check that conference exists
        if not conf:
            raise endpoints.NotFoundException(
                'No conference found with key: %s' % request.websafeConferenceKey)

        # check that user is owner
        if user_id != conf.organizerUserId:
            raise endpoints.ForbiddenException(
                'Only the owner can update the conference.')

        # Not getting all the fields, so don't create a new object; just
        # copy relevant fields from ConferenceForm to Conference object
        for field in request.all_fields():
            data = getattr(request, field.name)
            # only copy fields where we get data
            if data not in (None, []):
                # special handling for dates (convert string to Date)
                if field.name in ('startDate', 'endDate'):
                    data = datetime.strptime(data, "%Y-%m-%d").date()
                    if field.name == 'startDate':
                        conf.month = data.month
                # write to Conference object
                setattr(conf, field.name, data)
        conf.put()
        prof = ndb.Key(Profile, user_id).get()
        return self._copyConferenceToForm(conf, getattr(prof, 'displayName'))

    @endpoints.method(ConferenceForm, ConferenceForm, path='conference',
            http_method='POST', name='createConference')
    def createConference(self, request):
        """Create new conference."""
        return self._createConferenceObject(request)

    @endpoints.method(CONF_POST_REQUEST, ConferenceForm,
            path='conference/{websafeConferenceKey}',
            http_method='PUT', name='updateConference')
    def updateConference(self, request):
        """Update conference w/provided fields & return w/updated info."""
        return self._updateConferenceObject(request)

    @endpoints.method(CONF_GET_REQUEST, ConferenceForm,
            path='conference/{websafeConferenceKey}',
            http_method='GET', name='getConference')
    def getConference(self, request):
        """Return requested conference (by websafeConferenceKey)."""
        # get Conference object from request; bail if not found
        conf = ndb.Key(urlsafe=request.websafeConferenceKey).get()
        if not conf:
            raise endpoints.NotFoundException(
                'No conference found with key: %s' % request.websafeConferenceKey)
        prof = conf.key.parent().get()
        # return ConferenceForm
        return self._copyConferenceToForm(conf, getattr(prof, 'displayName'))

    @endpoints.method(message_types.VoidMessage, ConferenceForms,
            path='getConferencesCreated',
            http_method='POST', name='getConferencesCreated')
    def getConferencesCreated(self, request):
        """Return conferences created by user."""
        # make sure user is authed
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')
        user_id =  getUserId(user)
        # create ancestor query for all key matches for this user
        confs = Conference.query(ancestor=ndb.Key(Profile, user_id))
        prof = ndb.Key(Profile, user_id).get()
        # return set of ConferenceForm objects per Conference
        return ConferenceForms(
            items=[self._copyConferenceToForm(conf, getattr(prof, 'displayName')) for conf in confs]
        )

    def _getQuery(self, request):
        """Return formatted query from the submitted filters."""
        q = Conference.query()
        inequality_filter, filters = self._formatFilters(request.filters)

        # If exists, sort on inequality filter first
        if not inequality_filter:
            q = q.order(Conference.name)
        else:
            q = q.order(ndb.GenericProperty(inequality_filter))
            q = q.order(Conference.name)

        for filtr in filters:
            if filtr["field"] in ["month", "maxAttendees"]:
                filtr["value"] = int(filtr["value"])
            formatted_query = ndb.query.FilterNode(filtr["field"], filtr["operator"], filtr["value"])
            q = q.filter(formatted_query)
        return q

    def _formatFilters(self, filters):
        """Parse, check validity and format user supplied filters."""
        formatted_filters = []
        inequality_field = None

        for f in filters:
            filtr = {field.name: getattr(f, field.name) for field in f.all_fields()}

            try:
                filtr["field"] = FIELDS[filtr["field"]]
                filtr["operator"] = OPERATORS[filtr["operator"]]
            except KeyError:
                raise endpoints.BadRequestException("Filter contains invalid field or operator.")

            # Every operation except "=" is an inequality
            if filtr["operator"] != "=":
                # check if inequality operation has been used in previous filters
                # disallow the filter if inequality was performed on a different field before
                # track the field on which the inequality operation is performed
                if inequality_field and inequality_field != filtr["field"]:
                    raise endpoints.BadRequestException("Inequality filter is allowed on only one field.")
                else:
                    inequality_field = filtr["field"]

            formatted_filters.append(filtr)
        return (inequality_field, formatted_filters)

    @endpoints.method(ConferenceQueryForms, ConferenceForms,
            path='queryConferences',
            http_method='POST',
            name='queryConferences')
    def queryConferences(self, request):
        """Query for conferences."""
        conferences = self._getQuery(request)

        # Grabs display name for organizerUserId
        organisers = [(ndb.Key(Profile, conf.organizerUserId)) for conf in conferences]
        profiles = ndb.get_multi(organisers)

        # Puts display name for fetching simplification
        names = {}
        for profile in profiles:
            names[profile.key.id()] = profile.displayName

        # Returns a form for each conference
        return ConferenceForms(
                items=[self._copyConferenceToForm(conf, names[conf.organizerUserId]) for conf in \
                conferences]
        )
    @endpoints.method(CONF_GET_REQUEST, SessionForms,
            path='conference/{websafeConferenceKey}/sessions',
            http_method='GET', name='getConferenceSessions')
    def getConferenceSessions(self, request):
        """Given a conference, return all sessions."""
        # Copy ConferenceForm/ProtoRPC Message into dict
        data = {field.name: getattr(request, field.name) for field in request.all_fields()}
        # Get conference and check to see if it exists
        conf = ndb.Key(urlsafe=request.websafeConferenceKey).get()
        if not conf:
            raise endpoints.NotFoundException(
                'No conference found with key: %s' % request.websafeConferenceKey)

        # Create ancestor query for all key matches for this conference
        sessions = Session.query(ancestor=conf.key)

        # return set of SessionForm objects per Session
        return SessionForms(
            items=[self._copySessionToForm(session) for session in sessions]
        )

    @endpoints.method(SESSION_GET_REQUEST, SessionForms,
            path='conference/{websafeConferenceKey}/sessions/by_type/{typeOfSession}',
            http_method='GET', name='getConferenceSessionsByType')
    def getConferenceSessionsByType(self, request):
        """Given a conference, return all sessions of a specified type."""

        # Captures conference form
        data = {field.name: getattr(request, field.name) for field in request.all_fields()}
        typeOfSession = data['typeOfSession']
        # Get conference and check to see if it exists
        conf = ndb.Key(urlsafe=request.websafeConferenceKey).get()
        if not conf:
            raise endpoints.NotFoundException(
                'No conference found with key: %s' % request.websafeConferenceKey)

        # create ancestor query for all key matches for this conference
        sessions = Session.query(Session.typeOfSession == typeOfSession, ancestor=conf.key)

        # return set of SessionForm objects per Session
        return SessionForms(
            items=[self._copySessionToForm(session) for session in sessions]
        )

# - - - Session objects - - - - - - - - - - - - - - - - - - -

    def _createSessionObject(self, request):
        """Create or update Session object, returning SessionForm/request."""
        # preload necessary data items
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')
        user_id = getUserId(user)

        if not request.name:
            raise endpoints.BadRequestException("Session 'name' field needed")

        # Get conference and check to see if it exists
        conference = ndb.Key(urlsafe=request.websafeConferenceKey).get()

        if not conference:
            raise endpoints.NotFoundException(
                'No conferences found with this key: %s' % request.websafeConferenceKey)

        # Check that the user is the owner of the conference
        if user_id != conference.organizerUserId:
            raise endpoints.ForbiddenException(
                'Only owners can add sessions to their own conference.')

        data = {field.name: getattr(request, field.name) for field in request.all_fields()}
        wsck = data['websafeConferenceKey']
        p_key =  ndb.Key(urlsafe=request.websafeConferenceKey)
        c_id = Session.allocate_ids(size=1, parent=p_key)[0]
        c_key = ndb.Key(Session, c_id, parent=p_key)

        if data['startTime']:
            data['startTime']= datetime.strptime(data['startTime'], '%H:%M:%S').time()

        if data['date']:
            data['date']= datetime.strptime(data['date'], '%Y-%m-%d').date()

        data['key'] = c_key
        data['organizerUserId'] = user_id

        # add the new session data to datastore
        del data['websafeKey']
        del data['websafeConferenceKey']
        Session(**data).put()
        taskqueue.add(params={
                'speaker': request.speaker,
                'conferenceKey': request.websafeConferenceKey
                      },
                url='/tasks/set_featured_speaker'
            )
        return request

    def _copySessionToForm(self, session):
        """Copy fields into SessionForm."""
        session_form = SessionForm()
        for field in session_form.all_fields():
            if hasattr(session, field.name):
                # Date and time should be date string, others are just copied
                if field.name in ['startTime', 'date']:
                    setattr(session_form, field.name, str(getattr(session, field.name)))
                else:
                    setattr(session_form, field.name, getattr(session, field.name))
            elif field.name == "websafeKey":
                setattr(session_form, field.name, session.key.urlsafe())
        session_form.check_initialized()
        return session_form

    @endpoints.method(SessionForm, SessionForm,
            path='sessions',
            http_method='POST', name='createSession')
    def createSession(self, request):
        """Open only to the organizer of the conference."""
        return self._createSessionObject(request)

    @endpoints.method(SPEAKER_GET_REQUEST, SessionForms,
            path='sessions/{speaker}',
            http_method='GET', name='getSessionsBySpeaker')
    def getSessionsBySpeaker(self, request):
        """Given a speaker, return all sessions by this particular speaker, across all conferences."""
        data = {field.name: getattr(request, field.name) for field in request.all_fields()}
        speaker = data['speaker']
        sessions = Session.query(Session.speaker == speaker)
        return SessionForms(
            items=[self._copySessionToForm(session) for session in sessions]
        )

    # Filters for sesions that are less than 60 minutes in length for those
    # who are impatient and can't stand the idea of sitting still too long
    @endpoints.method(message_types.VoidMessage, SessionForms,
            http_method='GET', name='getShortSessions')
    def getShortSessions(self, request):
        """Returns sessions that are 1 hour or less"""

        sessions = Session.query(ndb.OR(
            Session.duration < 60
            ))
        return SessionForms(
            items=[self._copySessionToForm(session) for session in sessions]
        )

    # Filters for sessions that have a cloud highlight for those interested in
    # cloud topics
    @endpoints.method(message_types.VoidMessage, SessionForms,
            http_method='GET', name='getCloudSessions')
    def getCloudSessions(self, request):
        """Returns sessions featuring a cloud highlight"""

        sessions = Session.query(ndb.OR(
            Session.highlights == 'cloud'
            ))
        return SessionForms(
            items=[self._copySessionToForm(session) for session in sessions]
        )

    # Filters for sessions that occur before 7PM and are not a workshop type
    @endpoints.method(message_types.VoidMessage, SessionForms,
            http_method='GET', name='getBefore7PMWorkshopSessions')
    def getBefore7PMWorkshopSessions(self, request):
        """Returns sessions that occur before 7PM and are not a workshop"""

        # First filter sessions based on time using NBD
        sessions = Session.query(ndb.AND(
                Session.startTime != None,
                Session.startTime <= timed(hour=19)
                ))
        # Second, fileter session type using native Python since NBD cannot
        # handle two inequality filters in one query
        time_filter_sessions = []
        for session in sessions:
            if 'workshop' in session.typeOfSession:
                continue
            else:
                time_filter_sessions.append(session)

        return SessionForms(
            items=[self._copySessionToForm(session) for session in time_filter_sessions]
        )

# - - - Featured Speaker - - - - - - - - - - - - - - - - - - -

    # Endpoints menthod to create API to get featured speaker.
    @endpoints.method(message_types.VoidMessage, StringMessage,
            path='getFeaturedSpeaker',
            http_method='GET', name='getFeaturedSpeaker')
    def getFeaturedSpeaker(self, request):
        """Return featured speaker"""
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')

        return StringMessage(data=memcache.get("featuredSpeaker") or "")

# - - - Profile objects - - - - - - - - - - - - - - - - - - -

    def _copyProfileToForm(self, prof):
        """Copy relevant fields from Profile to ProfileForm."""
        # copy relevant fields from Profile to ProfileForm
        pf = ProfileForm()
        for field in pf.all_fields():
            if hasattr(prof, field.name):
                # convert t-shirt string to Enum; just copy others
                if field.name == 'teeShirtSize':
                    setattr(pf, field.name, getattr(TeeShirtSize, getattr(prof, field.name)))
                else:
                    setattr(pf, field.name, getattr(prof, field.name))
        pf.check_initialized()
        return pf

    def _getProfileFromUser(self):
        """Return user Profile from datastore, creating new one if non-existent."""
        # make sure user is authed
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')

        # get Profile from datastore
        user_id = getUserId(user)
        p_key = ndb.Key(Profile, user_id)
        profile = p_key.get()
        # create new Profile if not there
        if not profile:
            profile = Profile(
                key = p_key,
                displayName = user.nickname(),
                mainEmail= user.email(),
                teeShirtSize = str(TeeShirtSize.NOT_SPECIFIED),
            )
            profile.put()

        return profile      # return Profile

    def _doProfile(self, save_request=None):
        """Get user Profile and return to user, possibly updating it first."""
        # get user Profile
        prof = self._getProfileFromUser()

        # if saveProfile(), process user-modifyable fields
        if save_request:
            for field in ('displayName', 'teeShirtSize'):
                if hasattr(save_request, field):
                    val = getattr(save_request, field)
                    if val:
                        setattr(prof, field, str(val))
                        #if field == 'teeShirtSize':
                        #    setattr(prof, field, str(val).upper())
                        #else:
                        #    setattr(prof, field, val)
            prof.put()

        # return ProfileForm
        return self._copyProfileToForm(prof)

    @endpoints.method(message_types.VoidMessage, ProfileForm,
            path='profile', http_method='GET', name='getProfile')
    def getProfile(self, request):
        """Return user profile."""
        return self._doProfile()

    @endpoints.method(ProfileMiniForm, ProfileForm,
            path='profile', http_method='POST', name='saveProfile')
    def saveProfile(self, request):
        """Update & return user profile."""
        return self._doProfile(request)

# - - - Wishlist - - - - - - - - - - - - - - - - - - - -
    # Add a session to a user's wishlist
    @endpoints.method(ADD_SESSION_TO_WISHLIST_REQUEST, StringMessage,
            path='addSessionToWishList', http_method='POST',
            name='addSessionToWishlist')
    def addSessionToWishlist(self, request):
        """Adds the session to the user's list of sessions they are interested
        in attending"""
        # make sure user is authed
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')

        # Check to make sure session exists
        session_key = ndb.Key(urlsafe=request.sessionKey)
        if not session_key.get():
            raise endpoints.BadRequestException("Session could not be found")

        user = self._getProfileFromUser()
        # Add session to user's wishlist unless it is already there
        if not user.sessionWishList:
            user.sessionWishList = [request.sessionKey]
            msg = StringMessage(data="Session added to wishlist.")
        elif request.sessionKey in user.sessionWishList:
            msg = StringMessage(data="Duplicate session found in wishlist.")
        else:
            user.sessionWishList.append(request.sessionKey)
            msg = StringMessage(data="Session added to wishlist.")
        user.put()
        return msg

    # Get a user's wishlist
    @endpoints.method(message_types.VoidMessage, SessionForms,
            path='getSessionsInWishlist', http_method='GET',
            name='getSessionsInWishlist')
    def getSessionsInWishlist(self, request):
        """Query for all the sessions that the user is interested in"""
        # make sure user is authed
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')

        user = self._getProfileFromUser()

        return SessionForms(items=[
            self._copySessionToForm(ndb.Key(urlsafe=sessionKey).get())
            for sessionKey in user.sessionWishList])

    # Remove a session from a user's wishlist
    @endpoints.method(REMOVE_SESSION_TO_WISHLIST_REQUEST, StringMessage,
            path='removeSessionsFromWishlist', http_method='GET',
            name='removeSessionsFromWishlist')
    def removeSessionsFromWishlist(self, request):
        """Query for all the sessions that the user is interested in"""
        # make sure user is authed
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')

        # Check to make sure session exists
        session_key = ndb.Key(urlsafe=request.sessionKey)
        if not session_key.get():
            raise endpoints.BadRequestException("Session could not be found")
        user = self._getProfileFromUser()
        if not session_key.get():
            raise endpoints.BadRequestException("Session could not be found")
        user.sessionWishList.remove(request.sessionKey)
        msg = StringMessage(data="Session deleted from wishlist.")
        user.put()
        return msg

# - - - Announcements - - - - - - - - - - - - - - - - - - - -

    @staticmethod
    def _cacheAnnouncement():
        """Create Announcement & assign to memcache; used by
        memcache cron job & putAnnouncement().
        """
        confs = Conference.query(ndb.AND(
            Conference.seatsAvailable <= 5,
            Conference.seatsAvailable > 0)
        ).fetch(projection=[Conference.name])

        if confs:
            # If there are almost sold out conferences,
            # format announcement and set it in memcache
            announcement = '%s %s' % (
                'Last chance to attend! The following conferences '
                'are nearly sold out:',
                ', '.join(conf.name for conf in confs))
            memcache.set(MEMCACHE_ANNOUNCEMENTS_KEY, announcement)
        else:
            # If there are no sold out conferences,
            # delete the memcache announcements entry
            announcement = ""
            memcache.delete(MEMCACHE_ANNOUNCEMENTS_KEY)

        return announcement

    @staticmethod
    def _cacheFeaturedSpeaker(speakerName, conferenceKey):
        """
        Set the featured speaker if they have more than one session in a conference.
        """
        conf = ndb.Key(urlsafe=conferenceKey)
        sessions = Session.query(ancestor=conf).filter(Session.speaker == speakerName)
        if sessions.count() > 1:
            print("Inserting on cache.")
            displayedMessage = "Featured speaker: {0} " \
                               "Sessions: " \
                               " {1}".format(speakerName,
                                ",".join([str(session.name) for session in sessions]))
            memcache.set(MEMCACHE_FEATURED_SPEAKER_KEY, displayedMessage)

        return displayedMessage

    @endpoints.method(message_types.VoidMessage, StringMessage,
            path='conference/announcement/get',
            http_method='GET', name='getAnnouncement')
    def getAnnouncement(self, request):
        """Return Announcement from memcache."""
        announcement = memcache.get(MEMCACHE_ANNOUNCEMENTS_KEY)
        if not announcement:
            announcement = ""
        return StringMessage(data=announcement)

    @endpoints.method(message_types.VoidMessage, StringMessage,
            path='conference/announcement/put',
            http_method='GET', name='putAnnouncement')
    def putAnnouncement(self, request):
        """Put Announcement into memcache"""
        return StringMessage(data=self._cacheAnnouncement())

# - - - Registration - - - - - - - - - - - - - - - - - - - -

    @ndb.transactional(xg=True)
    def _conferenceRegistration(self, request, reg=True):
        """Register or unregister user for selected conference."""
        retval = None
        prof = self._getProfileFromUser() # get user Profile

        # check if conf exists given websafeConfKey
        # get conference; check that it exists
        wsck = request.websafeConferenceKey
        conf = ndb.Key(urlsafe=wsck).get()
        if not conf:
            raise endpoints.NotFoundException(
                'No conference found with key: %s' % wsck)

        # register
        if reg:
            # check if user already registered otherwise add
            if wsck in prof.conferenceKeysToAttend:
                raise ConflictException(
                    "You have already registered for this conference")

            # check if seats avail
            if conf.seatsAvailable <= 0:
                raise ConflictException(
                    "There are no seats available.")

            # register user, take away one seat
            prof.conferenceKeysToAttend.append(wsck)
            conf.seatsAvailable -= 1
            retval = True

        # unregister
        else:
            # check if user already registered
            if wsck in prof.conferenceKeysToAttend:

                # unregister user, add back one seat
                prof.conferenceKeysToAttend.remove(wsck)
                conf.seatsAvailable += 1
                retval = True
            else:
                retval = False

        # write things back to the datastore & return
        prof.put()
        conf.put()
        return BooleanMessage(data=retval)

    @endpoints.method(message_types.VoidMessage, ConferenceForms,
            path='conferences/attending',
            http_method='GET', name='getConferencesToAttend')
    def getConferencesToAttend(self, request):
        """Get list of conferences that user has registered for."""
        prof = self._getProfileFromUser() # get user Profile
        conf_keys = [ndb.Key(urlsafe=wsck) for wsck in prof.conferenceKeysToAttend]
        conferences = ndb.get_multi(conf_keys)

        # get organizers
        organisers = [ndb.Key(Profile, conf.organizerUserId) for conf in conferences]
        profiles = ndb.get_multi(organisers)

        # put display names in a dict for easier fetching
        names = {}
        for profile in profiles:
            names[profile.key.id()] = profile.displayName

        # return set of ConferenceForm objects per Conference
        return ConferenceForms(items=[self._copyConferenceToForm(conf, names[conf.organizerUserId])\
         for conf in conferences]
        )

    @endpoints.method(CONF_GET_REQUEST, BooleanMessage,
            path='conference/{websafeConferenceKey}',
            http_method='POST', name='registerForConference')
    def registerForConference(self, request):
        """Register user for selected conference."""
        return self._conferenceRegistration(request)

    @endpoints.method(CONF_GET_REQUEST, BooleanMessage,
            path='conference/{websafeConferenceKey}',
            http_method='DELETE', name='unregisterFromConference')
    def unregisterFromConference(self, request):
        """Unregister user for selected conference."""
        return self._conferenceRegistration(request, reg=False)

# - - - Register API - - - - - - - - - - - - - - - - - - - -

api = endpoints.api_server([ConferenceApi]) # register API


text:
package eu.bcvsolutions.idm.core.model.event.processor;

import java.util.Collection;
import java.util.List;

import org.junit.Assert;
import org.junit.Test;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.transaction.annotation.Transactional;

import eu.bcvsolutions.idm.core.api.dto.IdmIdentityContractDto;
import eu.bcvsolutions.idm.core.api.dto.IdmIdentityDto;
import eu.bcvsolutions.idm.core.api.dto.IdmIdentityRoleDto;
import eu.bcvsolutions.idm.core.api.dto.IdmRoleDto;
import eu.bcvsolutions.idm.core.api.dto.IdmTokenDto;
import eu.bcvsolutions.idm.core.model.domain.CoreGroupPermission;
import eu.bcvsolutions.idm.core.security.api.domain.DefaultGrantedAuthority;
import eu.bcvsolutions.idm.core.security.api.domain.IdmBasePermission;
import eu.bcvsolutions.idm.core.security.api.domain.IdmGroupPermission;

/**
 * Tests authorities change for role removal.
 * - tokens related to identity with role removal should be disabled.
 *
 * @author Jan Helbich
 * @author Radek Tomiška
 *
 */
@Transactional
public class IdentityRoleDeleteAuthoritiesProcessorTest extends AbstractIdentityAuthoritiesProcessorTest {

	/**
	 * Removing a role which grants authorities must raise
	 * the authorities modification event on identity.
	 */
	@Test
	public void testRoleRemovedAuthorityRemoved() {
		IdmRoleDto role = getTestRole();
		IdmIdentityDto i = getHelper().createIdentity();
		IdmIdentityContractDto c = getTestContract(i);
		IdmIdentityRoleDto ir = getTestIdentityRole(role, c);

		List<IdmTokenDto> tokens = tokenManager.getTokens(i);
		//
		Assert.assertTrue(tokens.isEmpty());
		Assert.assertEquals(1, identityRoleService.findAllByIdentity(i.getId()).size());
		checkAssignedAuthorities(i);
		//
		// login - one token
		getHelper().login(i.getUsername(), i.getPassword());
		try {
			tokens = tokenManager.getTokens(i);
			Assert.assertEquals(1, tokens.size());
			Assert.assertFalse(tokens.get(0).isDisabled());
			//
			// remove role - token should be disabled
			identityRoleService.delete(ir);
			//
			tokens = tokenManager.getTokens(i);
			Assert.assertEquals(1, tokens.size());
			Assert.assertTrue(tokens.get(0).isDisabled());
			Assert.assertEquals(0, identityRoleService.findAllByIdentity(i.getId()).size());
			Assert.assertEquals(0, authoritiesFactory.getGrantedAuthoritiesForIdentity(i.getId()).size());
		} finally {
			getHelper().logout();
		}
	}

	/**
	 * User has to roles with same authorities - removing just one role
	 * shall not change the authorities modification flag.
	 */
	@Test
	public void testRoleRemovedAuthorityStays() {
		// two roles with same authorities
		IdmRoleDto role = getTestRole();
		IdmRoleDto role2 = getTestRole();
		IdmIdentityDto i = getHelper().createIdentity();
		IdmIdentityContractDto c = getTestContract(i);
		IdmIdentityRoleDto ir = getTestIdentityRole(role, c);
		IdmIdentityRoleDto ir2 = getTestIdentityRole(role2, c);
		//
		List<IdmTokenDto> tokens = tokenManager.getTokens(i);
		//
		Assert.assertTrue(tokens.isEmpty());
		Assert.assertEquals(2, identityRoleService.findAllByIdentity(i.getId()).size());
		checkAssignedAuthorities(i);
		//
		// login - one token
		getHelper().login(i.getUsername(), i.getPassword());
		try {
			tokens = tokenManager.getTokens(i);
			Assert.assertEquals(1, tokens.size());
			Assert.assertFalse(tokens.get(0).isDisabled());

			identityRoleService.delete(ir2);

			tokens = tokenManager.getTokens(i);
			Assert.assertEquals(1, tokens.size());
			Assert.assertFalse(tokens.get(0).isDisabled());
			Assert.assertEquals(1, identityRoleService.findAllByIdentity(i.getId()).size());
			Assert.assertEquals(ir.getId(), identityRoleService.findAllByIdentity(i.getId()).get(0).getId());
			Assert.assertEquals(1, authoritiesFactory.getGrantedAuthoritiesForIdentity(i.getId()).size());
		} finally {
			getHelper().logout();
		}
	}

	/**
	 * User has to roles with same authorities - removing just one role
	 * shall not change the authorities modification flag.
	 */
	@Test
	public void testRoleRemovedSuperAuthorityStays() {
		// role with APP_ADMIN authority
		IdmRoleDto r = getHelper().createRole();
		createTestPolicy(r, IdmBasePermission.ADMIN, IdmGroupPermission.APP);
		//
		IdmRoleDto role2 = getTestRole();
		IdmIdentityDto i = getHelper().createIdentity();
		IdmIdentityContractDto c = getTestContract(i);
		IdmIdentityRoleDto ir = getTestIdentityRole(r, c);
		IdmIdentityRoleDto ir2 = getTestIdentityRole(role2, c);
		//
		List<IdmTokenDto> tokens = tokenManager.getTokens(i);
		//
		Assert.assertTrue(tokens.isEmpty());
		Assert.assertEquals(2, identityRoleService.findAllByIdentity(i.getId()).size());
		//
		// login - one token
		getHelper().login(i.getUsername(), i.getPassword());
		try {
			tokens = tokenManager.getTokens(i);
			Assert.assertEquals(1, tokens.size());
			Assert.assertFalse(tokens.get(0).isDisabled());

			identityRoleService.delete(ir2);

			tokens = tokenManager.getTokens(i);
			Assert.assertEquals(1, tokens.size());
			Assert.assertFalse(tokens.get(0).isDisabled());
			Assert.assertEquals(1, identityRoleService.findAllByIdentity(i.getId()).size());
			Assert.assertEquals(ir.getId(), identityRoleService.findAllByIdentity(i.getId()).get(0).getId());
			Assert.assertEquals(1, authoritiesFactory.getGrantedAuthoritiesForIdentity(i.getId()).size());
		} finally {
			getHelper().logout();
		}
	}

	private void checkAssignedAuthorities(IdmIdentityDto i) {
		GrantedAuthority g = new DefaultGrantedAuthority(CoreGroupPermission.IDENTITY, IdmBasePermission.DELETE);
		Collection<GrantedAuthority> authorities = authoritiesFactory.getGrantedAuthoritiesForIdentity(i.getId());
		Assert.assertEquals(1, authorities.size());
		authorities.stream().forEach(a -> Assert.assertEquals(g, a));
	}

}


text:
// generated by ginger from go generate -- DO NOT EDIT

package main

import (
	"io/ioutil"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"gopkg.in/yaml.v2"
)

func getConfig(c *cli.Context) (Config, error) {
	yamlPath := c.GlobalString("config")
	config := Config{}

	if _, err := os.Stat(yamlPath); err != nil {
		config = Config{
			":8080",
			"root",
			"",
			"localhost",
			"/data/Onion.db",
			"",
			"",
		}
		return config, nil
	}

	ymlData, err := ioutil.ReadFile(yamlPath)
	if err != nil {
		return config, err
	}

	err = yaml.Unmarshal([]byte(ymlData), &config)
	return config, err
}

func init() {
	// Log as JSON instead of the default ASCII formatter.
	log.SetFormatter(&log.TextFormatter{})

	// Output to stderr instead of stdout, could also be a file.
	log.SetOutput(os.Stderr)

	// Only log the warning severity or above.
	log.SetLevel(log.WarnLevel)
}

func main() {
	app := cli.NewApp()
	app.Name = "goslack"
	app.Usage = "micro service main"
	app.Version = "0.0.3"

	app.Flags = []cli.Flag{
		//	cli.StringFlag{"host", "http://localhost:8080", "use sevice host", "APP_HOST"},
		cli.StringFlag{"config, c", "config.yaml", "config file to use", "APP_CONFIG"},
	}

	app.Commands = []cli.Command{
		{
			Name:  "serve",
			Usage: "start micro service",
			Action: func(c *cli.Context) {
				//host := c.GlobalString("host")
				cfg, err := getConfig(c)
				if err != nil {
					log.WithFields(log.Fields{
						"file:": "main.go",
						"func:": "main",
						"line:": 74,
					}).Fatal("getConfig return error")
					return
				}
				svc := TwitterService{}

				if err = svc.Run(cfg); err != nil {
					log.Fatal(err)
				}

			},
		},
		{
			Name:  "migratedb",
			Usage: "Perform database migrations",
			Action: func(c *cli.Context) {
				cfg, err := getConfig(c)
				if err != nil {
					log.Fatal(err)
					return
				}

				svc := TwitterService{}

				if err = svc.Migrate(cfg); err != nil {
					log.Fatal(err)
				}
			},
		},
	}

	app.Run(os.Args)
}


text:
package net.ceedubs.ficus
package readers

import com.typesafe.config.ConfigFactory

class OptionReadersSpec extends Spec with OptionReader with AnyValReaders { def is = s2"""
  An option value reader should
    wrap an existing value in a Some $optionSome
    return a None for a non-existing value $optionNone
  """

  def optionSome = prop { i: Int =>
    val cfg = ConfigFactory.parseString(s"myValue = $i")
    optionValueReader[Int].read(cfg, "myValue") must beSome(i)
  }

  def optionNone = {
    val cfg = ConfigFactory.parseString("")
    optionValueReader[Boolean].read(cfg, "myValue") must beNone
  }
}


text:
namespace Domain.Categories.ValueObject
{
    public class Subcategory
    {
        public string Id { get; set; }
        public string Code { get; set; }
        public string Name { get; set; }
        public Audit Audit { get; set; }
    }
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android"
    android:shape="rectangle">
    <!--<solid android:color="#FFFFFF"/>-->
    <stroke
        android:width="1dp"
        android:color="@color/medical"/>
    <!--<padding
        android:bottom="5dp"
        android:left="5dp"
        android:right="5dp"
        android:top="5dp"/>-->
    <!--<corners
        android:bottomLeftRadius="7dp"
        android:bottomRightRadius="7dp"
        android:topLeftRadius="7dp"
        android:topRightRadius="7dp"/>-->
    <corners
        android:radius="2dp" />
</shape>


text:
extern crate futures;
extern crate indy_sys;
extern crate libc;

use indy::{IndyError, ErrorCode};
use indy::payments;
use self::futures::Future;
use self::indy_sys::{payments as payments_sys};

use std::collections::VecDeque;
use std::ffi::CString;
use self::libc::c_char;
use std::sync::{Once, ONCE_INIT, Mutex};

use utils::callback;

#[macro_export]
macro_rules! mocked_handler {
    ($first_param_name: ident: $first_param_type: ty $(, $param_name: ident: $param_type: ty)*) => (
        use super::*;

        lazy_static! {
          static ref INJECTIONS: Mutex<VecDeque<(i32, CString)>> = Default::default();
        }

        pub extern fn handle(cmd_handle: i32,
                                    $first_param_name: $first_param_type,
                                    $($param_name: $param_type,)*
                                    cb: Option<IndyPaymentCallback>) -> i32 {

            let cb = cb.unwrap_or_else(|| {
                panic!("Null passed as callback!")
            });

            if let Ok(mut injections) = INJECTIONS.lock() {
                if let Some((err, res)) = injections.pop_front() {
                    return (cb)(cmd_handle, err, res.as_ptr());
                }
            } else {
                panic!("Can't lock injections mutex");
            }

            panic!("No injections left!");
        }

        pub fn inject_mock(err: ErrorCode, res: &str) {
            if let Ok(mut injections) = INJECTIONS.lock() {
                let res = CString::new(res).unwrap();
                injections.push_back((err as i32, res))
            } else {
                panic!("Can't lock injections mutex");
            }
        }

        pub fn clear_mocks() {
            if let Ok(mut injections) = INJECTIONS.lock() {
                injections.clear();
            } else {
                panic!("Can't lock injections mutex");
            }
        }
    )
}

type IndyPaymentCallback = extern fn(command_handle_: i32,
                                     err: i32,
                                     payment_address: *const c_char) -> i32;

lazy_static! {
        static ref CREATE_PAYMENT_METHOD_INIT: Once = ONCE_INIT;
}

pub mod mock_method {
    use super::*;

    pub fn init() {
        CREATE_PAYMENT_METHOD_INIT.call_once(|| {
            let (receiver, cmd_handle, cb) = callback::_closure_to_cb_ec();
            let payment_method_name = CString::new("null").unwrap();
            unsafe {
                payments_sys::indy_register_payment_method(cmd_handle,
                                                       payment_method_name.as_ptr(),
                                                       Some(create_payment_address::handle),
                                                       Some(add_request_fees::handle),
                                                       Some(parse_response_with_fees::handle),
                                                       Some(build_get_payment_sources_request::handle),
                                                       Some(parse_get_payment_sources_response::handle),
                                                       Some(build_payment_req::handle),
                                                       Some(parse_payment_response::handle),
                                                       Some(build_mint_req::handle),
                                                       Some(build_set_txn_fees_req::handle),
                                                       Some(build_get_txn_fees_req::handle),
                                                       Some(parse_get_txn_fees_response::handle),
                                                       Some(build_verify_payment_req::handle),
                                                       Some(parse_verify_payment_response::handle),
                                                       cb,
                );
            }

            receiver.recv().unwrap();
        });
    }

    pub mod create_payment_address {
        mocked_handler!(_wallet_handle: i32, _config: *const c_char);
    }

    pub mod add_request_fees {
        mocked_handler!(_wallet_handle: i32, _submitter_did: *const c_char, _req_json: *const c_char, _inputs_json: *const c_char, _outputs_json: *const c_char, _extra: *const c_char);
    }

    pub mod parse_response_with_fees {
        mocked_handler!(_resp_json: *const c_char);
    }

    pub mod build_get_payment_sources_request {
        mocked_handler!(_wallet_handle: i32, _submitter_did: *const c_char, _payment_address: *const c_char);
    }

    pub mod parse_get_payment_sources_response {
        mocked_handler!(_resp_json: *const c_char);
    }

    pub mod build_payment_req {
        mocked_handler!(_wallet_handle: i32, _submitter_did: *const c_char, _inputs_json: *const c_char, _outputs_json: *const c_char, _extra: *const c_char);
    }

    pub mod parse_payment_response {
        mocked_handler!(_resp_json: *const c_char);
    }

    pub mod build_mint_req {
        mocked_handler!(_wallet_handle: i32, _submitter_did: *const c_char, _outputs_json: *const c_char, _extra: *const c_char);
    }

    pub mod build_set_txn_fees_req {
        mocked_handler!(_wallet_handle: i32, _submitter_did: *const c_char, _fees_json: *const c_char);
    }

    pub mod build_get_txn_fees_req {
        mocked_handler!(_wallet_handle: i32, _submitter_did: *const c_char);
    }

    pub mod parse_get_txn_fees_response {
        mocked_handler!(_resp_json: *const c_char);
    }

    pub mod build_verify_payment_req {
        mocked_handler!(_wallet_handle: i32, _submitter_did: *const c_char, _receipt: *const c_char);
    }

    pub mod parse_verify_payment_response {
        mocked_handler!(_resp_json: *const c_char);
    }
}

pub fn register_payment_method(payment_method_name: &str,
                               create_payment_address: Option<payments_sys::CreatePaymentAddressCB>,
                               add_request_fees: Option<payments_sys::AddRequestFeesCB>,
                               parse_response_with_fees: Option<payments_sys::ParseResponseWithFeesCB>,
                               build_get_payment_sources_request: Option<payments_sys::BuildGetPaymentSourcesRequestCB>,
                               parse_get_payment_sources_response: Option<payments_sys::ParseGetPaymentSourcesResponseCB>,
                               build_payment_req: Option<payments_sys::BuildPaymentReqCB>,
                               parse_payment_response: Option<payments_sys::ParsePaymentResponseCB>,
                               build_mint_req: Option<payments_sys::BuildMintReqCB>,
                               build_set_txn_fees_req: Option<payments_sys::BuildSetTxnFeesReqCB>,
                               build_get_txn_fees_req: Option<payments_sys::BuildGetTxnFeesReqCB>,
                               parse_get_txn_fees_response: Option<payments_sys::ParseGetTxnFeesResponseCB>,
                               build_verify_payment_req: Option<payments_sys::BuildVerifyPaymentReqCB>,
                               parse_verify_payment_response: Option<payments_sys::ParseVerifyPaymentResponseCB>,
) -> Result<(), ErrorCode> {
    let (receiver, cmd_handle, cb) = callback::_closure_to_cb_ec();

    let payment_method_name = CString::new(payment_method_name).unwrap();

    let err = unsafe {
        payments_sys::indy_register_payment_method(cmd_handle,
                                               payment_method_name.as_ptr(),
                                               create_payment_address,
                                               add_request_fees,
                                               parse_response_with_fees,
                                               build_get_payment_sources_request,
                                               parse_get_payment_sources_response,
                                               build_payment_req,
                                               parse_payment_response,
                                               build_mint_req,
                                               build_set_txn_fees_req,
                                               build_get_txn_fees_req,
                                               parse_get_txn_fees_response,
                                               build_verify_payment_req,
                                               parse_verify_payment_response,
                                               cb,
        )
    };

    super::results::result_to_empty(err, receiver)
}

pub fn create_payment_address(wallet_handle: i32, config: &str, payment_method: &str) -> Result<String, IndyError> {
    payments::create_payment_address(wallet_handle, payment_method, config).wait()
}

pub fn list_payment_addresses(wallet_handle: i32) -> Result<String, IndyError> {
    payments::list_payment_addresses(wallet_handle).wait()
}

pub fn add_request_fees(wallet_handle: i32, submitter_did: Option<&str>, req_json: &str, inputs_json: &str, outputs_json: &str, extra: Option<&str>) -> Result<(String, String), IndyError> {
    payments::add_request_fees(wallet_handle, submitter_did, req_json, inputs_json, outputs_json, extra).wait()
}

pub fn build_get_payment_sources_request(wallet_handle: i32, submitter_did: Option<&str>, payment_address: &str) -> Result<(String, String), IndyError> {
    payments::build_get_payment_sources_request(wallet_handle, submitter_did, payment_address).wait()
}

pub fn build_payment_req(wallet_handle: i32, submitter_did: Option<&str>, inputs_json: &str, outputs_json: &str, extra: Option<&str>) -> Result<(String, String), IndyError> {
    payments::build_payment_req(wallet_handle, submitter_did, inputs_json, outputs_json, extra).wait()
}

pub fn parse_response_with_fees(payment_method: &str, resp_json: &str) -> Result<String, IndyError> {
    payments::parse_response_with_fees(payment_method, resp_json).wait()
}

pub fn parse_get_payment_sources_response(payment_method: &str, resp_json: &str) -> Result<String, IndyError> {
    payments::parse_get_payment_sources_response(payment_method, resp_json).wait()
}

pub fn parse_payment_response(payment_method: &str, resp_json: &str) -> Result<String, IndyError> {
    payments::parse_payment_response(payment_method, resp_json).wait()
}

pub fn build_mint_req(wallet_handle: i32, submitter_did: Option<&str>, outputs_json: &str, extra: Option<&str>) -> Result<(String, String), IndyError> {
    payments::build_mint_req(wallet_handle, submitter_did, outputs_json, extra).wait()
}

pub fn build_set_txn_fees_req(wallet_handle: i32, submitter_did: Option<&str>, payment_method: &str, fees_json: &str) -> Result<String, IndyError> {
    payments::build_set_txn_fees_req(wallet_handle, submitter_did, payment_method, fees_json).wait()
}

pub fn build_get_txn_fees_req(wallet_handle: i32, submitter_did: Option<&str>, payment_method: &str) -> Result<String, IndyError> {
    payments::build_get_txn_fees_req(wallet_handle, submitter_did, payment_method).wait()
}

pub fn parse_get_txn_fees_response(payment_method: &str, resp_json: &str) -> Result<String, IndyError> {
    payments::parse_get_txn_fees_response(payment_method, resp_json).wait()
}

pub fn build_verify_payment_req(wallet_handle: i32, submitter_did: Option<&str>, receipt: &str) -> Result<(String, String), IndyError> {
    payments::build_verify_payment_req(wallet_handle, submitter_did, receipt).wait()
}

pub fn parse_verify_payment_response(payment_method: &str, resp_json: &str) -> Result<String, IndyError> {
    payments::parse_verify_payment_response(payment_method, resp_json).wait()
}


text:
if [ $# -lt 3 ]; then
    echo "Usage: msf_simple_map_creator.sh [bags folder] [extrinsic_file] [zone_id]"
    exit 1;
fi

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "${DIR}/.."

source "${DIR}/apollo_base.sh"

GNSS_LOC_TOPIC="/apollo/localization/msf_gnss"
LIDAR_LOC_TOPIC="/apollo/localization/msf_lidar"
FUSION_LOC_TOPIC="/apollo/localization/pose"
ODOMETRY_LOC_TOPIC="/apollo/sensor/gnss/odometry"
CLOUD_TOPIC="/apollo/sensor/velodyne64/compensator/PointCloud2"

GNSS_LOC_FILE="gnss_loc.txt"
LIDAR_LOC_FILE="lidar_loc.txt"
FUSION_LOC_FILE="fusion_loc.txt"
ODOMETRY_LOC_FILE="odometry_loc.txt"

IN_FOLDER=$1
EXTRINSIC_FILE=$2
ZONE_ID=$3

function data_exporter() {
  local BAG_FILE=$1
  local OUT_FOLDER=$2
  $APOLLO_BIN_PREFIX/modules/localization/msf/local_tool/data_extraction/monitor_data_exporter \
    --bag_file $BAG_FILE \
    --out_folder $OUT_FOLDER \
    --cloud_topic $CLOUD_TOPIC \
    --gnss_loc_topic $GNSS_LOC_TOPIC \
    --lidar_loc_topic $LIDAR_LOC_TOPIC \
    --fusion_loc_topic $FUSION_LOC_TOPIC \
    --odometry_loc_topic $ODOMETRY_LOC_TOPIC
}

function poses_interpolation() {
  local INPUT_POSES_PATH=$1
  local REF_TIMESTAMPS_PATH=$2
  local EXTRINSIC_PATH=$3
  local OUTPUT_POSES_PATH=$4
  $APOLLO_BIN_PREFIX/modules/localization/msf/local_tool/map_creation/poses_interpolator \
   --input_poses_path $INPUT_POSES_PATH \
   --ref_timestamps_path $REF_TIMESTAMPS_PATH \
   --extrinsic_path $EXTRINSIC_PATH \
   --output_poses_path $OUTPUT_POSES_PATH
}

function create_lossless_map() {
  $APOLLO_BIN_PREFIX/modules/localization/msf/local_tool/map_creation/lossless_map_creator \
      --use_plane_inliers_only true \
      --pcd_folders $1 \
      --pose_files $2 \
      --map_folder $IN_FOLDER \
      --zone_id $ZONE_ID \
      --coordinate_type UTM \
      --map_resolution_type single
}

function create_lossy_map() {
  $APOLLO_BIN_PREFIX/modules/localization/msf/local_tool/map_creation/lossless_map_to_lossy_map \
    --srcdir $IN_FOLDER/lossless_map \
    --dstdir $IN_FOLDER \

  rm -fr $IN_FOLDER/lossless_map
}

cd $IN_FOLDER
for item in $(ls -l *.bag | awk '{print $9}')
do
  DIR_NAME=$(echo $item | cut -d . -f 1)
  mkdir $DIR_NAME
  data_exporter "${item}" "${DIR_NAME}"
  poses_interpolation "${DIR_NAME}/pcd/${ODOMETRY_LOC_FILE}" "${DIR_NAME}/pcd/pcd_timestamp.txt" "${EXTRINSIC_FILE}" "${DIR_NAME}/pcd/corrected_poses.txt"
  create_lossless_map "${DIR_NAME}/pcd" "${DIR_NAME}/pcd/corrected_poses.txt"
done

create_lossy_map

echo "Done."


text:
/**
 * "First, solve the problem. Then, write the code. -John Johnson"
 * "Or use Vangav M"
 * www.vangav.com
 * */

/**
 * MIT License
 *
 * Copyright (c) 2016 Vangav
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 * */

/**
 * Community
 * Facebook Group: Vangav Open Source - Backend
 *   fb.com/groups/575834775932682/
 * Facebook Page: Vangav
 *   fb.com/vangav.f
 *
 * Third party communities for Vangav Backend
 *   - play framework
 *   - cassandra
 *   - datastax
 *
 * Tag your question online (e.g.: stack overflow, etc ...) with
 *   #vangav_backend
 *   to easier find questions/answers online
 * */

package com.vangav.backend.properties;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;

import com.vangav.backend.exceptions.CodeException;
import com.vangav.backend.exceptions.VangavException.ExceptionClass;
import com.vangav.backend.files.FileLoaderInl;

/**
 * @author mustapha
 * fb.com/mustapha.abdallah
 */
/**
 * PropertiesLoader Loads all properties files and stores
 *   key:value pairs in memory
 * It's also reload/get safe so a real-time-reload is safe with other threads
 *   getting properties' values
 * */
public class PropertiesLoader {

  private static PropertiesLoader instance = null;

  /**
   * propertiesFiles holds all properties files (with .prop extension)
   *   under ./conf/prop/
   */
  private Map<String, Properties> propertiesFiles;
  /**
   * loadLatch is used activated on loading/reloading properties to make new
   *   get requests wait till loading/reloading is finished
   */
  private CountDownLatch loadLatch;
  /**
   * getLatch is used to make loading/reloading requests wait till current
   *   get requests are finished
   */
  private AtomicInteger getLatch;

  /**
   * Constructor - PropertiesLoader
   * @return new PropertiesLoader Object
   * @throws Exception
   */
  protected PropertiesLoader () throws Exception {

    this.propertiesFiles = new HashMap<String, Properties>();
    this.loadLatch = new CountDownLatch(0);
    this.getLatch = new AtomicInteger(0);

    this.loadProperties();
  }

  /**
   * load
   * OPTIONAL method to load properties files ahead of using them to save
   *   time on the first time use
   * @throws Exception
   */
  public static void load () throws Exception {

    if (instance == null) {

      instance = new PropertiesLoader();
    }
  }

  /**
   * i
   * singleton method
   * @return static instance of PropertiesLoader
   * @throws Exception
   */
  public static PropertiesLoader i () throws Exception {

    if (instance == null) {

      instance = new PropertiesLoader();
    }

    return instance;
  }

  private static final String kPropPath = "conf/prop";
  private static final String kPropExt = ".prop";

  /**
   * loadProperties
   * loads all properties files (with extension .prop) from ./conf/prop
   * @throws Exception
   */
  private synchronized void loadProperties() throws Exception {

    // lock for loading/reloading properties
    this.loadLatch = new CountDownLatch(1);

    // wait for in-processing get requests to finish
    while (this.getLatch.get() > 0) {

      try {

        this.getLatch.wait();
      } catch (InterruptedException e) {
      }
    }

    try {

      Map<String, Properties> tempMap = new HashMap<String, Properties>();

      File [] propFiles =
        FileLoaderInl.loadFiles(
          kPropPath,
          kPropExt);

      InputStream input;
      Properties prop;

      for (File propFile : propFiles) {

        input = new FileInputStream(propFile.getAbsolutePath() );

        prop = new Properties();
        prop.load(input);

        tempMap.put(
          propFile.getName().substring(
            0,
            propFile.getName().length() - kPropExt.length() ),
          prop);
      }

      this.propertiesFiles.clear();
      this.propertiesFiles.putAll(tempMap);
    } catch (Exception e) {

      throw e;
    } finally {

      // release loading/reloading lock
      this.loadLatch.countDown();
    }
  }

  /**
   * getProperty
   * @param fileName: properties file's name
   * @param propName: property's name
   * @param defaultValue: default value to return if not found here
   * @return string value of the property
   * @throws Exception
   */
  public String getProperty (
    String fileName,
    String propName,
    String defaultValue) throws Exception {

    if (defaultValue == null) {

      throw new CodeException(
        152,
        1,
        "Invalid property, properties file ["
        + fileName
        + "] doesn't have property ["
        + propName +
        "]",
        ExceptionClass.MISSING_ITEM);
    }

    // wait for loading/reloading to finish
    try {

      this.loadLatch.await();
    } catch (InterruptedException e) {
    }

    // increment get latch
    this.getLatch.incrementAndGet();

    try {

      Properties prop =
        propertiesFiles.get(fileName);

      if (prop == null) {

        return defaultValue;
      }

      return
        prop.getProperty(
          propName,
          defaultValue);
    } catch (Exception e) {

      throw e;
    } finally {

      // decrement get latch and notify it
      this.getLatch.decrementAndGet();

      synchronized (this.getLatch) {

        this.getLatch.notify();
      }
    }
  }

  /**
   * reload
   * used to reload properties file
   * @throws Exception
   */
  public void reload () throws Exception {

    this.loadProperties();
  }
}


text:
package com.jevetools.unmarshal.python.api;

/**
 * Copyright (c) 2013, jEVETools.
 *
 * All rights reserved.
 *
 * @version 0.0.1
 * @since 0.0.1
 */
public interface PyCRowSet
    extends PyBaseObjectEx, Iterable<PyBase>
{

}


text: void combineFITS( const QStringList & inputFilenames, const QString & outputFileName );


text:
////////////////////////////////////////////////////////////////////////////
//
// Copyright 2016 Realm Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////

#include <realm/object-store/impl/realm_coordinator.hpp>
#include "test_utils.hpp"

#include <realm/util/base64.hpp>
#include <realm/util/file.hpp>
#include <realm/string_data.hpp>

#include <external/json/json.hpp>

#include <iostream>
#include <sys/stat.h>
#ifndef _WIN32
#include <unistd.h>
#include <sys/types.h>
#endif

#if REALM_PLATFORM_APPLE
#include <sys/mount.h>
#include <sys/param.h>
#endif

namespace realm {

bool create_dummy_realm(std::string path)
{
    Realm::Config config;
    config.path = path;
    try {
        _impl::RealmCoordinator::get_coordinator(path)->get_realm(config, none);
        REQUIRE_REALM_EXISTS(path);
        return true;
    }
    catch (std::exception&) {
        return false;
    }
}

void reset_test_directory(const std::string& base_path)
{
    util::try_remove_dir_recursive(base_path);
    util::make_dir(base_path);
}

std::vector<char> make_test_encryption_key(const char start)
{
    std::vector<char> vector;
    vector.reserve(64);
    for (int i = 0; i < 64; i++) {
        vector.emplace_back((start + i) % 128);
    }
    return vector;
}

// FIXME: Catch2 limitation on old compilers (currently our android CI)
// https://github.com/catchorg/Catch2/blob/master/docs/limitations.md#clangg----skipping-leaf-sections-after-an-exception
void catch2_ensure_section_run_workaround(bool did_run_a_section, std::string section_name,
                                          util::FunctionRef<void()> func)
{
    if (did_run_a_section) {
        func();
    }
    else {
        std::cout << "Skipping test section '" << section_name << "' on this run." << std::endl;
    }
}

std::string encode_fake_jwt(const std::string& in, util::Optional<int64_t> exp, util::Optional<int64_t> iat)
{
    // by default make a valid expiry time so that the sync session pre check
    // doesn't trigger a token refresh on first open
    using namespace std::chrono_literals;
    if (!exp) {
        std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
        exp = std::chrono::system_clock::to_time_t(now + 30min);
    }
    if (!iat) {
        std::chrono::system_clock::time_point now = std::chrono::system_clock::now();
        iat = std::chrono::system_clock::to_time_t(now - 1s);
    }

    std::string unencoded_prefix = nlohmann::json({"alg", "HS256"}).dump();
    std::string unencoded_body =
        nlohmann::json(
            {{"user_data", {{"token", in}}}, {"exp", *exp}, {"iat", *iat}, {"access", {"download", "upload"}}})
            .dump();

    std::string encoded_prefix, encoded_body;
    encoded_prefix.resize(util::base64_encoded_size(unencoded_prefix.size()));
    encoded_body.resize(util::base64_encoded_size(unencoded_body.size()));
    util::base64_encode(unencoded_prefix.data(), unencoded_prefix.size(), &encoded_prefix[0], encoded_prefix.size());
    util::base64_encode(unencoded_body.data(), unencoded_body.size(), &encoded_body[0], encoded_body.size());
    std::string suffix = "Et9HFtf9R3GEMA0IICOfFMVXY7kkTX1wr4qCyhIf58U";
    return encoded_prefix + "." + encoded_body + "." + suffix;
}

static bool file_is_on_exfat(const std::string& path)
{
#if REALM_PLATFORM_APPLE
    if (path.empty())
        return false;

    struct statfs fsbuf;
    int ret = statfs(path.c_str(), &fsbuf);
    REALM_ASSERT_RELEASE(ret == 0);
    // The documentation and headers helpfully don't list any of the values of
    // f_type or provide constants for them
    return fsbuf.f_type == 28 /* exFAT */;
#else
    static_cast<void>(path);
    return false;
#endif
}

bool chmod_supported(const std::string& path)
{
#ifndef _WIN32
    if (getuid() == 0) {
        return false; // running as root
    }
    if (file_is_on_exfat(path)) {
        return false;
    }
    return true;
#else
    static_cast<void>(path);
    return false;
#endif
}

int get_permissions(const std::string& path)
{
    int perms = 0;
#ifndef _WIN32
    REALM_ASSERT(!path.empty());
    struct stat statbuf;
    int ret = ::stat(path.c_str(), &statbuf);
    REALM_ASSERT_EX(ret == 0, ret, errno);
    perms = statbuf.st_mode;
#else
    static_cast<void>(path);
#endif
    return perms;
}

void chmod(const std::string& path, int permissions)
{
#ifndef _WIN32
    int ret = ::chmod(path.c_str(), permissions);
    REALM_ASSERT_EX(ret == 0, ret, errno);
#else
    static_cast<void>(path);
    static_cast<void>(permissions);
#endif
}

} // namespace realm


text:
[![Actions Build Status](https://github.com/neo4j-contrib/neovis.js/workflows/CI/badge.svg?branch=master)]()[![npm version](https://badge.fury.io/js/neovis.js.svg)](https://badge.fury.io/js/neovis.js)

Graph visualizations powered by vis.js with data from Neo4j.

![](img/example-viz.png)

## Features

- [x] Connect to Neo4j instance to get live data
- [x] User specified labels and property to be displayed
- [x] User specified Cypher query to populate
- [x] Specify node property for url of image for node
- [x] Specify edge property for edge thickness
- [x] Specify node property for community / clustering
- [x] Specify node property for node size
- [x] Configure popover

## Install

Neovis.js can be installed via npm:

```
npm install --save neovis.js
```

you can also obtain neovis.js via CDN:

## CDN

For ease of use Neovis.js can be obtained from Neo4jLabs CDN:

*Most recent release*

```
<script src="https://cdn.neo4jlabs.com/neovis.js/v1.5.0/neovis.js"></script>
```

*Version without neo4j-driver dependency*

```
<script src="https://cdn.neo4jlabs.com/neovis.js/v1.5.0/neovis-without-dependencies.js"></script>
```

## Quickstart Example

Let's go through the steps to reproduce this visualization:

![](img/example-viz.png)

### Prepare Neo4j

Start with a blank Neo4j instance, or spin up a blank [Neo4j Sandbox](https://neo4jsandbox.com). We'll load the Game of Thrones dataset, run:

```
LOAD CSV WITH HEADERS FROM "https://raw.githubusercontent.com/mathbeveridge/asoiaf/master/data/asoiaf-all-edges.csv" AS row
MERGE (src:Character {name: row.Source})
MERGE (tgt:Character {name: row.Target})
MERGE (src)-[r:INTERACTS]->(tgt) ON CREATE SET r.weight = toInteger(row.weight)
```

We've pre-calculated PageRank and ran a community detection algorithm to assign community ids for each Character. Let's load those next:

```
LOAD CSV WITH HEADERS FROM "https://raw.githubusercontent.com/johnymontana/neovis.js/master/examples/data/got-centralities.csv" AS row
MATCH (c:Character {name: row.name})
SET c.community = toInteger(row.community),
    c.pagerank  = toFloat(row.pagerank)
```

Our graph now consists of `Character` nodes that are connected by an `INTERACTS` relationships. We can visualize the whole graph in Neo4j Browser by running:

```
MATCH p=(:Character)-[:INTERACTS]->(:Character)
RETURN p
```

![](/img/got-neo4j-browser.png)

We can see characters that are connected and with the help of the force directed layout we can begin to see clusters in the graph. However, we want to visualize the centralities (PageRank) and community detection results that we also imported.

Specifically we would like:

* Node size to be proportional to the Character's `pagerank` score. This will allow us to quickly identify important nodes in the network.
* Node color to determined by the `community` property. This will allow us to visualize clusters.
* Relationship thickeness should be proportional to the `weight` property on the `INTERACTS` relationship.

Neovis.js, by combining the JavaScript driver for Neo4j and the vis.js visualization library will allow us to build this visualization.

### index.html

Create a new html file:

``` html
<!doctype html>
<html>
    <head>
        <title>Neovis.js Simple Example</title>
        <style type="text/css">
            html, body {
                font: 16pt arial;
            }

            #viz {
                width: 900px;
                height: 700px;
                border: 1px solid lightgray;
                font: 22pt arial;
            }
        </style>
    </head>
    <body onload="draw()">
        <div id="viz"></div>
    </body>
</html>

```

We define some basic CSS to specify the boundaries of a `div` and then create a single `div` in the body. We also specify `onload="draw()"` so that the `draw()` function is called as soon as the body is loaded.

We need to pull in `neovis.js`:

``` html
<script src="https://rawgit.com/neo4j-contrib/neovis.js/master/dist/neovis.js"></script>
```

And define our draw() function:

```
<script type="text/javascript">

        var viz;

        function draw() {
            var config = {
                container_id: "viz",
                server_url: "bolt://localhost:7687",
                server_user: "neo4j",
                server_password: "sorts-swims-burglaries",
                labels: {
                    "Character": {
                        "caption": "name",
                        "size": "pagerank",
                        "community": "community",
                        "title_properties": [
                            "name",
                            "pagerank"
                        ]
                    }
                },
                relationships: {
                    "INTERACTS": {
                        "thickness": "weight",
                        "caption": false
                    }
                },
                initial_cypher: "MATCH (n)-[r:INTERACTS]->(m) RETURN *"
            };

            viz = new NeoVis.default(config);
            viz.render();
        }
    </script>
```

This function creates a `config` object that specifies how to connect to Neo4j, what data to fetch, and how to configure the visualization.

![](/img/example-viz.png)

See [simple-example.html](/examples/simple-example.html) for the full code.

### module usage
you can also use it as module, but it would require you have a way to import css files

```javascript
import NeoVis from 'neovis.js';
```

or you can import the version with bundled dependency

```javascript
import NeoVis from 'neovis.js/dist/neovis.js';
```

## Build

This project uses git submodules to include the dependencies for neo4j-driver and vis.js. This project uses webpack to build a bundle that includes all project dependencies. `webpack.config.js` contains the configuration for webpack. After cloning the repo:

```
npm install
npm run build
```

will build `dist/neovis.js` and `dist/neovis-without-dependencies.js`

## Documentation

* `Neovis.default(config)`
* `Neovis.clearNetwork()`
* `Neovis.reinit(config)`
* `Neovis.reload()`
* `Neovis.stabilize()`
* `Neovis.renderWithCypher(statement)`
* `Neovis.updateWithCypher(statement)`

* `config`

### `Neovis.default(config)`

Constructor for Neovis. Creates new Neovis object, given configuration. See [config](#config)

### `Neovis.clearNetwork()`

Clears network visualization

### `Neovis.reinit(config)`

Reinitializes the network visualization with a new `config` object. See [config](#config)

### `Neovis.reload()`

Reload the visualization. Will fetch data again from Neo4j per `initial_cypher` in the config object.

### `Neovis.stabilize()`

Stop the physics simulation.

### `Neovis.renderWithCypher(statement)`

Render a new visualization with results from a Cypher statement. Any `Node` and `Relationship` objects returned in the Cypher query will be rendered in the visualization. Paths are not currently supported.

### `Neovis.updateWithCypher(statement)`

Update the current visualization with results from a Cypher statement. Any `Node` and `Relationship` objects returned in the Cypher query will be rendered in the visualization. Paths are not currently supported.

### `config`

A configuration object that defines:

* how to connect to Neo4j (required)
* an initial Cypher query for loading data for the visualization (optional)
* the DOM element in which the visualization should be rendered (required)
* how to style elements of the visualization (`labels` and `relationships`) (required)

Example:

~~~ js
var config = {
                container_id: "viz",
                server_url: "bolt://localhost:7687",
                server_user: "neo4j",
                server_password: "sorts-swims-burglaries",
                labels: {
                    //"Character": "name",
                    "Character": {
                        "caption": "name",
                        "size": "pagerank",
                        "community": "community",
                        //"image": 'https://visjs.org/images/visjs_logo.png',
                        "title_properties": [
                            "name",
                            "pagerank"
                        ],
                        //"sizeCypher": "MATCH (n) WHERE id(n) = {id} MATCH (n)-[r]-() RETURN sum(r.weight) AS c"
                    },
                    [NeoVis.NEOVIS_DEFAULT_CONFIG]: {
                         "caption": "defaultCaptionProperty",
                         "size": "defaultPagerank",
                         "community": "defaultCommunity"
                         //"sizeCypher": "defaultSizeCypher"

                    }
                },
                relationships: {
                    "INTERACTS": {
                        "thickness": "weight",
                        "caption": false
                    },
                    [NeoVis.NEOVIS_DEFAULT_CONFIG]: {
                         "thickness": "defaultThicknessProperty",
                         "caption": "defaultCaption"
                    }
                },
                initial_cypher: "MATCH (n)-[r:INTERACTS]->(m) RETURN n,r,m"
            };

            viz = new NeoVis.default(config);
            viz.render();
~~~

#### `config.container_id`

#### `config.server_url`

#### `config.server_user`

#### `config.server_password`

#### `config.server_database`
(Only for Neo4j 4 and above) The name of database connecting to.

### `NeoVis.NEOVIS_DEFAULT_CONFIG`
For both `config.labels` and `config.relationships` NeoVis.NEOVIS_DEFAULT_CONFIG
symbol can be added, which have the have the same properties as the normal config for specific
label/relationshipType
you can use it by either `NeoVis.NEOVIS_DEFAULT_CONFIG` if you are using NeoVis as global
or you can use it by
~~~js
import { NEOVIS_DEFAULT_CONFIG } from 'neovis.js';
~~~
#### `config.labels`

```
"Character": {
    "caption": "name",
    "size": "pagerank",
    "community": "community",
    "image": 'https://visjs.org/images/visjs_logo.png',
    "font": {
        "size":26,
        "color":"#000000"
    },
    "title_properties": [
        "name",
        "pagerank"
    ],
    sizeCypher: "MATCH (n) WHERE id(n) = $id MATCH (n)-[r]-() RETURN sum(r.weight) AS c"
}
```
##### `config.labels.caption`
`String`: The property name to use as node caption.
or `Function`: A function takes a [Neo4j node](https://neo4j.com/docs/api/javascript-driver/current/class/src/graph-types.js~Node.html) and return a string caption.
Default to no caption.
##### `config.labels.size`
`String`: The property name to use as node size.
Default to `1`.
##### `config.labels.community`
`String`: The property name to use as community (color).  \
Default to color by label.
##### `config.labels.image`
`String`. The url of image to display.
Default to no image (a dot will be displayed).
##### `config.labels.font`
`Object`: If `font` is supplied, the default font size configuration for node captions will be overwritten by the customized config. See `font` in [vis-network - nodes](https://visjs.github.io/vis-network/docs/network/nodes.html) for all available configuration for fonts.
##### `config.labels.title_properties`
`Array`: If `title_properties` is supplied, only the attributes listed in it are displayed in the tooltip.
         Otherwise, all attributes are present in the tooltip.
##### `config.labels.sizeCypher`
`String`: A Cypher query to get the size of the node. `$id` represent the id of the node to query size, and the Cypher query should return a Neo4j `Number`.
`config.labels.sizeCypher` has a higher priority than `config.labels.size`, i.e. `sizeCypher` will overwrite the resulting size from `size` field.
#### `config.relationships`

```
{
    "INTERACTS": {
        "thickness": "weight",
        "caption": false
    }
}
```
##### `config.relationships.thickness`
`String`: The property name to use as edge thickness.
Default to `1`.
##### `config.relationships.caption`
`Boolean`: If set to true, the relationship type will be displayed as edge caption.
or `String`: The property name to use as edge caption.
#### `config.arrows`

Boolean. Defaults to false.

#### `config.hierarchical`

Boolean. Default to false.

#### `config.hierarchical_sort_method`

When hierarchical layout is enabled you may choose betweeen `"hubsize"` (default) and `"directed"`.

#### `config.initial_cypher`

#### `config.console_debug`

Defaults to not showing console messages

#### `config.encrypted`

`"ENCRYPTION_OFF"` (default) or `"ENCRYPTION_ON"`

This must be set to `"ENCRYPTION_ON"` when using a secure WebSocket connection, such as with Neo4j Aura.

#### `config.trust`

`"TRUST_ALL_CERTIFICATES"` (default) or `"TRUST_SYSTEM_CA_SIGNED_CERTIFICATES"`


text:
package shell

import (
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestDetectBash(t *testing.T) {
	originalShell := os.Getenv("SHELL")
	os.Setenv("SHELL", "/bin/bash")
	defer os.Setenv("SHELL", originalShell)
	shell, err := Detect()
	assert.Nil(t, err)
	assert.Equal(t, "bash", shell)
}

func TestDetectFish(t *testing.T) {
	originalShell := os.Getenv("SHELL")
	os.Setenv("SHELL", "/bin/bash")
	defer os.Setenv("SHELL", originalShell)
	originalFishdir := os.Getenv("__fish_bin_dir")
	os.Setenv("__fish_bin_dir", "/usr/local/Cellar/fish/2.2.0/bin")
	defer os.Setenv("__fish_bin_dir", originalFishdir)
	shell, err := Detect()
	assert.Nil(t, err)
	assert.Equal(t, "fish", shell)
}


text:
@implementation MainSearchBar

-(instancetype)initWithFrame:(CGRect)frame
{
    self = [super initWithFrame:frame];
    if (self) {

        self.font = [UIFont systemFontOfSize:15];
        self.placeholder = @"请输入搜索条件";
        self.background = [UIImage imageNamed:@""];

        //通过init来创建初始化绝大部分控件,控件都是没有尺寸
        UIImageView *searchIcon = [[UIImageView alloc]init];
        searchIcon.image = [UIImage imageNamed:@""];
        searchIcon.width = 30;
        searchIcon.height = 30;
        searchIcon.contentMode = UIViewContentModeCenter;
        self.leftView = searchIcon;
        self.leftViewMode = UITextFieldViewModeAlways;

    }

    return self;
}

+(instancetype)searchBar
{

    return [[self alloc]init];

}

@end