text:
import logging
import itertools
import re

from itertools import imap
from operator import itemgetter

from django.utils.translation import ugettext as _

from desktop.lib import thrift_util
from desktop.conf import get_ldap_password, LDAP_USERNAME
from desktop.conf import DEFAULT_USER
from hadoop import cluster

from TCLIService import TCLIService
from TCLIService.ttypes import TOpenSessionReq, TGetTablesReq, TFetchResultsReq,\
  TStatusCode, TGetResultSetMetadataReq, TGetColumnsReq, TTypeId,\
  TExecuteStatementReq, TGetOperationStatusReq, TFetchOrientation,\
  TCloseSessionReq, TGetSchemasReq, TGetLogReq, TCancelOperationReq,\
  TCloseOperationReq, TFetchResultsResp, TRowSet, TProtocolVersion

from beeswax import conf as beeswax_conf
from beeswax import hive_site
from beeswax.hive_site import hiveserver2_use_ssl
from beeswax.models import Session, HiveServerQueryHandle, HiveServerQueryHistory
from beeswax.server.dbms import Table, NoSuchObjectException, DataTable,\
                                QueryServerException

LOG = logging.getLogger(__name__)

IMPALA_RESULTSET_CACHE_SIZE = 'impala.resultset.cache.size'
DEFAULT_USER = DEFAULT_USER.get()

class HiveServerTable(Table):
  """
  We are parsing DESCRIBE EXTENDED text as the metastore API like GetColumns() misses most of the information.
  Impala only supports a simple DESCRIBE.
  """
  def __init__(self, table_results, table_schema, desc_results, desc_schema):
    if beeswax_conf.THRIFT_VERSION.get() >= 7:
      if not table_results.columns:
        raise NoSuchObjectException()
      self.table = table_results.columns
    else: # Deprecated. To remove in Hue 4.
      if not table_results.rows:
        raise NoSuchObjectException()
      self.table = table_results.rows and table_results.rows[0] or ''

    self.table_schema = table_schema
    self.desc_results = desc_results
    self.desc_schema = desc_schema

  @property
  def name(self):
    return HiveServerTRow(self.table, self.table_schema).col('TABLE_NAME')

  @property
  def is_view(self):
    return HiveServerTRow(self.table, self.table_schema).col('TABLE_TYPE') == 'VIEW' # Used to be VIRTUAL_VIEW

  @property
  def partition_keys(self):
    describe = self.extended_describe
    # Parses a list of: partitionKeys:[FieldSchema(name:baz, type:string, comment:null), FieldSchema(name:boom, type:string, comment:null)]
    match = re.search('partitionKeys:\[([^\]]+)\]', describe)
    if match is not None:
      match = match.group(1)
      return [PartitionKeyCompatible(*partition)
          for partition in re.findall('FieldSchema\(name:(.+?), type:(.+?), comment:(.+?)\)', match)]
    else:
      return []

  @property
  def path_location(self):
    try:
      describe = self.extended_describe
      match = re.search('location:([^,]+)', describe)
      if match is not None:
        match = match.group(1)
      return match
    except:
      # Impala does not have extended_describe
      return None

  @property
  def parameters(self):
    # Parses a list of: parameters:{serialization.format=1}),... parameters:{numPartitions=2, EXTERNAL=TRUE}
    describe = self.extended_describe
    params = re.findall('parameters:\{([^\}]+?)\}', describe)
    if params:
      params_list = ', '.join(params).split(', ')
      return dict([param.split('=')for param in params_list])
    else:
      return {}

  @property
  def cols(self):
    cols = HiveServerTTableSchema(self.desc_results, self.desc_schema).cols()
    try:
      end_cols_index = map(itemgetter('col_name'), cols).index('') # Truncate below extended describe
      return cols[0:end_cols_index]
    except:
      try:
        # Spark SQL: does not have an empty line in extended describe
        try:
          end_cols_index = map(itemgetter('col_name'), cols).index('# Partition Information')
        except:
          end_cols_index = map(itemgetter('col_name'), cols).index('Detailed Table Information')
        return cols[0:end_cols_index]
      except:
        # Impala: uses non extended describe and 'col' instead of 'col_name'
        return cols

  @property
  def comment(self):
    return HiveServerTRow(self.table, self.table_schema).col('REMARKS')

  @property
  def extended_describe(self):
    # Just keep rows after 'Detailed Table Information'
    rows = HiveServerTTableSchema(self.desc_results, self.desc_schema).cols()
    detailed_row_index = map(itemgetter('col_name'), rows).index('Detailed Table Information')
    # Hack because of bad delimiter escaping in LazySimpleSerDe in HS2: parameters:{serialization.format=})
    describe_text = rows[detailed_row_index]['data_type']
    try:
      # LazySimpleSerDe case, also add full next row
      return describe_text + rows[detailed_row_index + 1]['col_name'] + rows[detailed_row_index + 1]['data_type']
    except:
      return describe_text

  @property
  def properties(self):
    # Ugly but would need a recursive parsing to be clean
    no_table = re.sub('\)$', '', re.sub('^Table\(', '', self.extended_describe))
    properties = re.sub(', sd:StorageDescriptor\(cols.+?\]', '', no_table).split(', ')
    props = []

    for prop in properties:
      key_val = prop.rsplit(':', 1)
      if len(key_val) == 1:
        key_val = key_val[0].rsplit('=', 1)
      if len(key_val) == 2:
        props.append(key_val)

    return props

class HiveServerTRowSet2:
  def __init__(self, row_set, schema):
    self.row_set = row_set
    self.rows = row_set.rows
    self.schema = schema
    self.startRowOffset = row_set.startRowOffset

  def is_empty(self):
    return not self.row_set.columns or not HiveServerTColumnValue2(self.row_set.columns[0]).val

  def cols(self, col_names):
    cols_rows = []

    rs = HiveServerTRow2(self.row_set.columns, self.schema)
    cols = [rs.full_col(name) for name in col_names]

    for cols_row in itertools.izip(*cols):
      cols_rows.append(dict(itertools.izip(col_names, cols_row)))

    return cols_rows

  def __iter__(self):
    return self

  def next(self):
    if self.row_set.columns:
      return HiveServerTRow2(self.row_set.columns, self.schema)
    else:
      raise StopIteration

class HiveServerTRow2:
  def __init__(self, cols, schema):
    self.cols = cols
    self.schema = schema

  def col(self, colName):
    pos = self._get_col_position(colName)
    return HiveServerTColumnValue2(self.cols[pos]).val[0] # Return only first element

  def full_col(self, colName):
    pos = self._get_col_position(colName)
    return HiveServerTColumnValue2(self.cols[pos]).val # Return the full column and its values

  def _get_col_position(self, column_name):
    return filter(lambda (i, col): col.columnName == column_name, enumerate(self.schema.columns))[0][0]

  def fields(self):
    try:
      return [HiveServerTColumnValue2(field).val.pop(0) for field in self.cols]
    except IndexError:
      raise StopIteration

class HiveServerTColumnValue2:
  def __init__(self, tcolumn_value):
    self.column_value = tcolumn_value

  @property
  def val(self):
    # Could directly get index from schema but would need to cache the schema
    if self.column_value.stringVal:
      return self._get_val(self.column_value.stringVal)
    elif self.column_value.i16Val is not None:
      return self._get_val(self.column_value.i16Val)
    elif self.column_value.i32Val is not None:
      return self._get_val(self.column_value.i32Val)
    elif self.column_value.i64Val is not None:
      return self._get_val(self.column_value.i64Val)
    elif self.column_value.doubleVal is not None:
      return self._get_val(self.column_value.doubleVal)
    elif self.column_value.boolVal is not None:
      return self._get_val(self.column_value.boolVal)
    elif self.column_value.byteVal is not None:
      return self._get_val(self.column_value.byteVal)
    elif self.column_value.binaryVal is not None:
      return self._get_val(self.column_value.binaryVal)

  @classmethod
  def _get_val(cls, column):
    column.values = cls.set_nulls(column.values, column.nulls)
    column.nulls = '' # Clear the null values for not re-marking again the column with nulls at the next call
    return column.values

  @classmethod
  def mark_nulls(cls, values, bytestring):
    mask = bytearray(bytestring)

    for n in mask:
      yield n & 0x01
      yield n & 0x02
      yield n & 0x04
      yield n & 0x08

      yield n & 0x10
      yield n & 0x20
      yield n & 0x40
      yield n & 0x80

  @classmethod
  def set_nulls(cls, values, bytestring):
    if bytestring == '' or re.match('^(\x00)+$', bytestring): # HS2 has just \x00 or '', Impala can have \x00\x00...
      return values
    else:
      return [None if is_null else value for value, is_null in itertools.izip(values, cls.mark_nulls(values, bytestring))]

class HiveServerDataTable(DataTable):
  def __init__(self, results, schema, operation_handle, query_server):
    self.schema = schema and schema.schema
    self.row_set = HiveServerTRowSet(results.results, schema)
    self.operation_handle = operation_handle
    if query_server['server_name'] == 'impala':
      self.has_more = results.hasMoreRows
    else:
      self.has_more = not self.row_set.is_empty()    # Should be results.hasMoreRows but always True in HS2
    self.startRowOffset = self.row_set.startRowOffset    # Always 0 in HS2

  @property
  def ready(self):
    return True

  def cols(self):
    if self.schema:
      return [HiveServerTColumnDesc(col) for col in self.schema.columns]
    else:
      return []

  def rows(self):
    for row in self.row_set:
      yield row.fields()

class HiveServerTTableSchema:
  def __init__(self, columns, schema):
    self.columns = columns
    self.schema = schema

  def cols(self):
    try:
      return HiveServerTRowSet(self.columns, self.schema).cols(('col_name', 'data_type', 'comment'))
    except:
      # Impala API is different
      cols = HiveServerTRowSet(self.columns, self.schema).cols(('name', 'type', 'comment'))
      for col in cols:
        col['col_name'] = col.pop('name')
        col['col_type'] = col.pop('type')
      return cols

  def col(self, colName):
    pos = self._get_col_position(colName)
    return HiveServerTColumnDesc(self.columns[pos]).val

  def _get_col_position(self, column_name):
    return filter(lambda (i, col): col.columnName == column_name, enumerate(self.schema.columns))[0][0]

if beeswax_conf.THRIFT_VERSION.get() >= 7:
  HiveServerTRow = HiveServerTRow2
  HiveServerTRowSet = HiveServerTRowSet2
else:
  # Deprecated. To remove in Hue 4.
  class HiveServerTRow:
    def __init__(self, row, schema):
      self.row = row
      self.schema = schema

    def col(self, colName):
      pos = self._get_col_position(colName)
      return HiveServerTColumnValue(self.row.colVals[pos]).val

    def _get_col_position(self, column_name):
      return filter(lambda (i, col): col.columnName == column_name, enumerate(self.schema.columns))[0][0]

    def fields(self):
      return [HiveServerTColumnValue(field).val for field in self.row.colVals]

  class HiveServerTRowSet:
    def __init__(self, row_set, schema):
      self.row_set = row_set
      self.rows = row_set.rows
      self.schema = schema
      self.startRowOffset = row_set.startRowOffset

    def is_empty(self):
      return len(self.rows) == 0

    def cols(self, col_names):
      cols_rows = []
      for row in self.rows:
        row = HiveServerTRow(row, self.schema)
        cols = {}
        for col_name in col_names:
          cols[col_name] = row.col(col_name)
        cols_rows.append(cols)
      return cols_rows

    def __iter__(self):
      return self

    def next(self):
      if self.rows:
        return HiveServerTRow(self.rows.pop(0), self.schema)
      else:
        raise StopIteration

class HiveServerTColumnValue:
  def __init__(self, tcolumn_value):
    self.column_value = tcolumn_value

  @property
  def val(self):
    if self.column_value.boolVal is not None:
      return self.column_value.boolVal.value
    elif self.column_value.byteVal is not None:
      return self.column_value.byteVal.value
    elif self.column_value.i16Val is not None:
      return self.column_value.i16Val.value
    elif self.column_value.i32Val is not None:
      return self.column_value.i32Val.value
    elif self.column_value.i64Val is not None:
      return self.column_value.i64Val.value
    elif self.column_value.doubleVal is not None:
      return self.column_value.doubleVal.value
    elif self.column_value.stringVal is not None:
      return self.column_value.stringVal.value

class HiveServerTColumnDesc:
  def __init__(self, column):
    self.column = column

  @property
  def name(self):
    return self.column.columnName

  @property
  def comment(self):
    return self.column.comment

  @property
  def type(self):
    return self.get_type(self.column.typeDesc)

  @classmethod
  def get_type(self, typeDesc):
    for ttype in typeDesc.types:
      if ttype.primitiveEntry is not None:
        return TTypeId._VALUES_TO_NAMES[ttype.primitiveEntry.type]
      elif ttype.mapEntry is not None:
        return ttype.mapEntry
      elif ttype.unionEntry is not None:
        return ttype.unionEntry
      elif ttype.arrayEntry is not None:
        return ttype.arrayEntry
      elif ttype.structEntry is not None:
        return ttype.structEntry
      elif ttype.userDefinedTypeEntry is not None:
        return ttype.userDefinedTypeEntry

class HiveServerClient:
  HS2_MECHANISMS = {'KERBEROS': 'GSSAPI', 'NONE': 'PLAIN', 'NOSASL': 'NOSASL', 'LDAP': 'PLAIN'}

  def __init__(self, query_server, user):
    self.query_server = query_server
    self.user = user

    use_sasl, mechanism, kerberos_principal_short_name, impersonation_enabled, ldap_username, ldap_password = self.get_security()
    LOG.info('use_sasl=%s, mechanism=%s, kerberos_principal_short_name=%s, impersonation_enabled=%s' % (
             use_sasl, mechanism, kerberos_principal_short_name, impersonation_enabled))

    self.use_sasl = use_sasl
    self.kerberos_principal_short_name = kerberos_principal_short_name
    self.impersonation_enabled = impersonation_enabled

    if self.query_server['server_name'] == 'impala':
      from impala import conf as impala_conf

      ssl_enabled = impala_conf.SSL.ENABLED.get()
      ca_certs = impala_conf.SSL.CACERTS.get()
      keyfile = impala_conf.SSL.KEY.get()
      certfile = impala_conf.SSL.CERT.get()
      validate = impala_conf.SSL.VALIDATE.get()
      timeout = impala_conf.SERVER_CONN_TIMEOUT.get()
    else:
      ssl_enabled = hiveserver2_use_ssl()
      ca_certs = beeswax_conf.SSL.CACERTS.get()
      keyfile = beeswax_conf.SSL.KEY.get()
      certfile = beeswax_conf.SSL.CERT.get()
      validate = beeswax_conf.SSL.VALIDATE.get()
      timeout = beeswax_conf.SERVER_CONN_TIMEOUT.get()

    if ldap_username:
      username = ldap_username
      password = ldap_password
    else:
      username = user.username
      password = None

    self._client = thrift_util.get_client(TCLIService.Client,
                                          query_server['server_host'],
                                          query_server['server_port'],
                                          service_name=query_server['server_name'],
                                          kerberos_principal=kerberos_principal_short_name,
                                          use_sasl=use_sasl,
                                          mechanism=mechanism,
                                          username=username,
                                          password=password,
                                          timeout_seconds=timeout,
                                          use_ssl=ssl_enabled,
                                          ca_certs=ca_certs,
                                          keyfile=keyfile,
                                          certfile=certfile,
                                          validate=validate,
                                          transport_mode=query_server.get('transport_mode', 'socket'),
                                          http_url=query_server.get('http_url', '')
    )

  def get_security(self):
    principal = self.query_server['principal']
    impersonation_enabled = False
    ldap_username = None
    ldap_password = get_ldap_password()

    if ldap_password is not None: # Pass-through LDAP authentication
      ldap_username = LDAP_USERNAME.get()

    if principal:
      kerberos_principal_short_name = principal.split('/', 1)[0]
    else:
      kerberos_principal_short_name = None

    if self.query_server['server_name'] == 'impala':
      if ldap_password: # Force LDAP auth if ldap_password is provided
        use_sasl = True
        mechanism = HiveServerClient.HS2_MECHANISMS['NONE']
      else:
        cluster_conf = cluster.get_cluster_conf_for_job_submission()
        use_sasl = cluster_conf is not None and cluster_conf.SECURITY_ENABLED.get()
        mechanism = HiveServerClient.HS2_MECHANISMS['KERBEROS']
      impersonation_enabled = self.query_server['impersonation_enabled']
    else:
      hive_mechanism = hive_site.get_hiveserver2_authentication()
      if hive_mechanism not in HiveServerClient.HS2_MECHANISMS:
        raise Exception(_('%s server authentication not supported. Valid are %s.' % (hive_mechanism, HiveServerClient.HS2_MECHANISMS.keys())))
      use_sasl = hive_mechanism in ('KERBEROS', 'NONE', 'LDAP')
      mechanism = HiveServerClient.HS2_MECHANISMS[hive_mechanism]
      impersonation_enabled = hive_site.hiveserver2_impersonation_enabled()

    return use_sasl, mechanism, kerberos_principal_short_name, impersonation_enabled, ldap_username, ldap_password

  def open_session(self, user):
    kwargs = {
        'client_protocol': beeswax_conf.THRIFT_VERSION.get() - 1,
        'username': user.username, # If SASL or LDAP, it gets the username from the authentication mechanism" since it dependents on it.
        'configuration': {},
    }

    if self.impersonation_enabled:
      kwargs.update({'username': DEFAULT_USER})

      if self.query_server['server_name'] == 'impala': # Only when Impala accepts it
        kwargs['configuration'].update({'impala.doas.user': user.username})

    if self.query_server['server_name'] == 'beeswax': # All the time
      kwargs['configuration'].update({'hive.server2.proxy.user': user.username})

    req = TOpenSessionReq(**kwargs)
    res = self._client.OpenSession(req)

    if res.status is not None and res.status.statusCode not in (TStatusCode.SUCCESS_STATUS,):
      if hasattr(res.status, 'errorMessage') and res.status.errorMessage:
        message = res.status.errorMessage
      else:
        message = ''
      raise QueryServerException(Exception('Bad status for request %s:\n%s' % (req, res)), message=message)

    sessionId = res.sessionHandle.sessionId
    LOG.info('Opening session %s' % sessionId)

    encoded_status, encoded_guid = HiveServerQueryHandle(secret=sessionId.secret, guid=sessionId.guid).get()

    return Session.objects.create(owner=user,
                                  application=self.query_server['server_name'],
                                  status_code=res.status.statusCode,
                                  secret=encoded_status,
                                  guid=encoded_guid,
                                  server_protocol_version=res.serverProtocolVersion)

  def call(self, fn, req, status=TStatusCode.SUCCESS_STATUS):
    session = Session.objects.get_session(self.user, self.query_server['server_name'])

    if session is None:
      session = self.open_session(self.user)

    if hasattr(req, 'sessionHandle') and req.sessionHandle is None:
      req.sessionHandle = session.get_handle()

    res = fn(req)

    # Not supported currently in HS2 and Impala: TStatusCode.INVALID_HANDLE_STATUS
    if res.status.statusCode == TStatusCode.ERROR_STATUS and \
        re.search('Invalid SessionHandle|Invalid session|Client session expired', res.status.errorMessage or '', re.I):
      LOG.info('Retrying with a new session because for %s of %s' % (self.user, res))

      session = self.open_session(self.user)
      req.sessionHandle = session.get_handle()

      # Get back the name of the function to call
      res = getattr(self._client, fn.attr)(req)

    if status is not None and res.status.statusCode not in (
        TStatusCode.SUCCESS_STATUS, TStatusCode.SUCCESS_WITH_INFO_STATUS, TStatusCode.STILL_EXECUTING_STATUS):
      if hasattr(res.status, 'errorMessage') and res.status.errorMessage:
        message = res.status.errorMessage
      else:
        message = ''
      raise QueryServerException(Exception('Bad status for request %s:\n%s' % (req, res)), message=message)
    else:
      return res

  def close_session(self, sessionHandle):
    req = TCloseSessionReq(sessionHandle=sessionHandle)
    return self._client.CloseSession(req)

  def get_databases(self):
    # GetCatalogs() is not implemented in HS2
    req = TGetSchemasReq()
    res = self.call(self._client.GetSchemas, req)

    results, schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT)
    self.close_operation(res.operationHandle)

    col = 'TABLE_SCHEM'
    return HiveServerTRowSet(results.results, schema.schema).cols((col,))

  def get_tables(self, database, table_names):
    req = TGetTablesReq(schemaName=database, tableName=table_names)
    res = self.call(self._client.GetTables, req)

    results, schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=5000)
    self.close_operation(res.operationHandle)

    return HiveServerTRowSet(results.results, schema.schema).cols(('TABLE_NAME',))

  def get_table(self, database, table_name):
    req = TGetTablesReq(schemaName=database, tableName=table_name)
    res = self.call(self._client.GetTables, req)

    table_results, table_schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT)
    self.close_operation(res.operationHandle)

    if self.query_server['server_name'] == 'impala':
      # Impala does not supported extended
      query = 'DESCRIBE %s.%s' % (database, table_name)
    else:
      query = 'DESCRIBE EXTENDED %s.%s' % (database, table_name)
    (desc_results, desc_schema), operation_handle = self.execute_statement(query, max_rows=5000, orientation=TFetchOrientation.FETCH_NEXT)
    self.close_operation(operation_handle)

    return HiveServerTable(table_results.results, table_schema.schema, desc_results.results, desc_schema.schema)

  def execute_query(self, query, max_rows=1000):
    configuration = self._get_query_configuration(query)
    return self.execute_query_statement(statement=query.query['query'], max_rows=max_rows, configuration=configuration)

  def execute_query_statement(self, statement, max_rows=1000, configuration={}):
    (results, schema), operation_handle = self.execute_statement(statement=statement, max_rows=max_rows, configuration=configuration)
    return HiveServerDataTable(results, schema, operation_handle, self.query_server)

  def execute_async_query(self, query, statement=0):
    if statement == 0:
      # Impala just has settings currently
      if self.query_server['server_name'] == 'beeswax':
        for resource in query.get_configuration_statements():
          self.execute_statement(resource.strip())

    configuration = {}

    if self.query_server['server_name'] == 'impala' and self.query_server['querycache_rows'] > 0:
      configuration[IMPALA_RESULTSET_CACHE_SIZE] = str(self.query_server['querycache_rows'])

    # The query can override the default configuration
    configuration.update(self._get_query_configuration(query))
    query_statement = query.get_query_statement(statement)

    return self.execute_async_statement(statement=query_statement, confOverlay=configuration)

  def execute_statement(self, statement, max_rows=1000, configuration={}, orientation=TFetchOrientation.FETCH_FIRST):
    if self.query_server['server_name'] == 'impala' and self.query_server['QUERY_TIMEOUT_S'] > 0:
      configuration['QUERY_TIMEOUT_S'] = str(self.query_server['QUERY_TIMEOUT_S'])

    req = TExecuteStatementReq(statement=statement.encode('utf-8'), confOverlay=configuration)
    res = self.call(self._client.ExecuteStatement, req)

    return self.fetch_result(res.operationHandle, max_rows=max_rows, orientation=orientation), res.operationHandle

  def execute_async_statement(self, statement, confOverlay):
    if self.query_server['server_name'] == 'impala' and self.query_server['QUERY_TIMEOUT_S'] > 0:
      confOverlay['QUERY_TIMEOUT_S'] = str(self.query_server['QUERY_TIMEOUT_S'])

    req = TExecuteStatementReq(statement=statement.encode('utf-8'), confOverlay=confOverlay, runAsync=True)
    res = self.call(self._client.ExecuteStatement, req)

    return HiveServerQueryHandle(secret=res.operationHandle.operationId.secret,
                                 guid=res.operationHandle.operationId.guid,
                                 operation_type=res.operationHandle.operationType,
                                 has_result_set=res.operationHandle.hasResultSet,
                                 modified_row_count=res.operationHandle.modifiedRowCount)

  def fetch_data(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=1000):
    # Fetch until the result is empty dues to a HS2 bug instead of looking at hasMoreRows
    results, schema = self.fetch_result(operation_handle, orientation, max_rows)
    return HiveServerDataTable(results, schema, operation_handle, self.query_server)

  def cancel_operation(self, operation_handle):
    req = TCancelOperationReq(operationHandle=operation_handle)
    return self.call(self._client.CancelOperation, req)

  def close_operation(self, operation_handle):
    req = TCloseOperationReq(operationHandle=operation_handle)
    return self.call(self._client.CloseOperation, req)

  def get_columns(self, database, table):
    req = TGetColumnsReq(schemaName=database, tableName=table)
    res = self.call(self._client.GetColumns, req)

    res, schema = self.fetch_result(res.operationHandle, orientation=TFetchOrientation.FETCH_NEXT)
    self.close_operation(res.operationHandle)

    return res, schema

  def fetch_result(self, operation_handle, orientation=TFetchOrientation.FETCH_FIRST, max_rows=1000):
    if operation_handle.hasResultSet:
      fetch_req = TFetchResultsReq(operationHandle=operation_handle, orientation=orientation, maxRows=max_rows)
      res = self.call(self._client.FetchResults, fetch_req)
    else:
      res = TFetchResultsResp(results=TRowSet(startRowOffset=0, rows=[], columns=[]))

    if operation_handle.hasResultSet and TFetchOrientation.FETCH_FIRST: # Only fetch for the first call that should be with start_over
      meta_req = TGetResultSetMetadataReq(operationHandle=operation_handle)
      schema = self.call(self._client.GetResultSetMetadata, meta_req)
    else:
      schema = None

    return res, schema

  def fetch_log(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=1000):
    req = TFetchResultsReq(operationHandle=operation_handle, orientation=orientation, maxRows=max_rows, fetchType=1)
    res = self.call(self._client.FetchResults, req)

    if beeswax_conf.THRIFT_VERSION.get() >= 7:
      lines = res.results.columns[0].stringVal.values
    else:
      lines = imap(lambda r: r.colVals[0].stringVal.value, res.results.rows)

    return '\n'.join(lines)

  def get_operation_status(self, operation_handle):
    req = TGetOperationStatusReq(operationHandle=operation_handle)
    return self.call(self._client.GetOperationStatus, req)

  def explain(self, query):
    query_statement = query.get_query_statement(0)
    configuration = self._get_query_configuration(query)
    return self.execute_query_statement(statement='EXPLAIN %s' % query_statement, configuration=configuration)

  def get_log(self, operation_handle):
    try:
      req = TGetLogReq(operationHandle=operation_handle)
      res = self.call(self._client.GetLog, req)
      return res.log
    except:
      return 'Server does not support GetLog()'

  def get_partitions(self, database, table_name, max_parts):
    table = self.get_table(database, table_name)

    if max_parts is None or max_parts <= 0:
      max_rows = 10000
    else:
      max_rows = 1000 if max_parts <= 250 else max_parts

    partitionTable = self.execute_query_statement('SHOW PARTITIONS %s' % table_name, max_rows=max_rows) # DB prefix supported since Hive 0.13
    return [PartitionValueCompatible(partition, table) for partition in partitionTable.rows()][-max_parts:]

  def _get_query_configuration(self, query):
    return dict([(setting['key'], setting['value']) for setting in query.settings])

class HiveServerTableCompatible(HiveServerTable):
  """Same API as Beeswax"""

  def __init__(self, hive_table):
    self.table = hive_table.table
    self.table_schema = hive_table.table_schema
    self.desc_results = hive_table.desc_results
    self.desc_schema = hive_table.desc_schema

  @property
  def cols(self):
    return [type('Col', (object,), {'name': col.get('col_name', '').strip(),
                                    'type': col.get('data_type', col.get('col_type', '')).strip(), # Impala is col_type
                                    'comment': col.get('comment', '').strip() if col.get('comment') else '', }) for col in HiveServerTable.cols.fget(self)]

class ResultCompatible:

  def __init__(self, data_table):
    self.data_table = data_table
    self.rows = data_table.rows
    self.has_more = data_table.has_more
    self.start_row = data_table.startRowOffset
    self.ready = True

  @property
  def columns(self):
    return self.cols()

  def cols(self):
    return [col.name for col in self.data_table.cols()]

class PartitionKeyCompatible:

  def __init__(self, name, type, comment):
    self.name = name
    self.type = type
    self.comment = comment

  def __eq__(self, other):
    return isinstance(other, PartitionKeyCompatible) and \
        self.name == other.name and \
        self.type == other.type and \
        self.comment == other.comment

  def __repr__(self):
    return 'PartitionKey(name:%s, type:%s, comment:%s)' % (self.name, self.type, self.comment)

class PartitionValueCompatible:

  def __init__(self, partition, table):
    # Parses: ['datehour=2013022516'] or ['month=2011-07/dt=2011-07-01/hr=12']
    self.values = [val.split('=')[1] for part in partition for val in part.split('/')]
    self.sd = type('Sd', (object,), {'location': '%s/%s' % (table.path_location, ','.join(partition)),})

class ExplainCompatible:

  def __init__(self, data_table):
    self.textual = '\n'.join([line[0] for line in data_table.rows()])

class ResultMetaCompatible:

  def __init__(self):
    self.in_tablename = True

class HiveServerClientCompatible(object):
  """Same API as Beeswax"""

  def __init__(self, client):
    self._client = client
    self.user = client.user
    self.query_server = client.query_server

  def query(self, query, statement=0):
    return self._client.execute_async_query(query, statement)

  def get_state(self, handle):
    operationHandle = handle.get_rpc_handle()
    res = self._client.get_operation_status(operationHandle)
    return HiveServerQueryHistory.STATE_MAP[res.operationState]

  def get_operation_status(self, handle):
    operationHandle = handle.get_rpc_handle()
    return self._client.get_operation_status(operationHandle)

  def use(self, query):
    data = self._client.execute_query(query)
    self._client.close_operation(data.operation_handle)
    return data

  def explain(self, query):
    data_table = self._client.explain(query)
    data = ExplainCompatible(data_table)
    self._client.close_operation(data_table.operation_handle)
    return data

  def fetch(self, handle, start_over=False, max_rows=None):
    operationHandle = handle.get_rpc_handle()
    if max_rows is None:
      max_rows = 1000

    if start_over and not (self.query_server['server_name'] == 'impala' and self.query_server['querycache_rows'] == 0): # Backward compatibility for impala
      orientation = TFetchOrientation.FETCH_FIRST
    else:
      orientation = TFetchOrientation.FETCH_NEXT

    data_table = self._client.fetch_data(operationHandle, orientation=orientation, max_rows=max_rows)

    return ResultCompatible(data_table)

  def cancel_operation(self, handle):
    operationHandle = handle.get_rpc_handle()
    return self._client.cancel_operation(operationHandle)

  def close(self, handle):
    return self.close_operation(handle)

  def close_operation(self, handle):
    operationHandle = handle.get_rpc_handle()
    return self._client.close_operation(operationHandle)

  def close_session(self, session):
    operationHandle = session.get_handle()
    return self._client.close_session(operationHandle)

  def dump_config(self):
    return 'Does not exist in HS2'

  def get_log(self, handle, start_over=True):
    operationHandle = handle.get_rpc_handle()

    if beeswax_conf.USE_GET_LOG_API.get() or self.query_server['server_name'] == 'impala':
      return self._client.get_log(operationHandle)
    else:
      if start_over:
        orientation = TFetchOrientation.FETCH_FIRST
      else:
        orientation = TFetchOrientation.FETCH_NEXT

      return self._client.fetch_log(operationHandle, orientation=orientation, max_rows=-1)

  def get_databases(self):
    col = 'TABLE_SCHEM'
    return [table[col] for table in self._client.get_databases()]

  def get_tables(self, database, table_names):
    tables = [table['TABLE_NAME'] for table in self._client.get_tables(database, table_names)]
    tables.sort()
    return tables

  def get_table(self, database, table_name):
    table = self._client.get_table(database, table_name)
    return HiveServerTableCompatible(table)

  def get_columns(self, database, table):
    return self._client.get_columns(database, table)

  def get_default_configuration(self, *args, **kwargs):
    return {}

  def get_results_metadata(self, handle):
    # We just need to mock
    return ResultMetaCompatible()

  def create_database(self, name, description): raise NotImplementedError()

  def get_database(self, *args, **kwargs): raise NotImplementedError()

  def alter_table(self, dbname, tbl_name, new_tbl): raise NotImplementedError()

  def open_session(self, user):
    return self._client.open_session(user)

  def add_partition(self, new_part): raise NotImplementedError()

  def get_partition(self, *args, **kwargs): raise NotImplementedError()

  def get_partitions(self, database, table_name, max_parts):
    return self._client.get_partitions(database, table_name, max_parts)

  def alter_partition(self, db_name, tbl_name, new_part): raise NotImplementedError()


text:
from ventas_app.models import *
from django.db.models  import *

def valor_o_cero(valor):
    if(valor == None): return 0;
    else: return valor;

def movimientos_recarga(fecha_inicio, fecha_fin):
    """ Suma de la cantidad de dinero recibido por recargas en un lapso de tiempo entre fecha_inicio y fecha_fin """
    return MovimientoVentas.objects.filter(factura__fecha__gte = fecha_inicio).filter(factura__fecha__lte = fecha_fin).filter(Q(tipo='R')).aggregate(r=Sum('cantidad'))['r']

def movimientos_caja(fecha_inicio, fecha_fin):
    """ Suma de la cantidad de dinero recibido por todas las ventas en un lapso de tiempo entre fecha_inicio y fecha_fin """
    mcaja_hoy = MovimientoCaja.objects.filter(fecha__gte = fecha_inicio).filter(fecha__lte = fecha_fin)
    return  valor_o_cero(mcaja_hoy.filter(Q(tipo='I')).aggregate(r=Sum('cantidad'))['r']) - valor_o_cero(mcaja_hoy.filter(Q(tipo='Z')).aggregate(r=Sum('cantidad'))['r'])

def movimientos_ventas(fecha_inicio, fecha_fin):
    """ Suma de la cantidad de dinero  deducido de todos los movimientos de caja en un lapso de tiempo entre fecha_inicio y fecha_fin """
    return MovimientoVentas.objects.filter(factura__fecha__gte = fecha_inicio).filter(factura__fecha__lte = fecha_fin).filter(Q(tipo='E')).aggregate(r=Sum('cantidad'))['r']

def movimientos_ajuste(fecha_inicio, fecha_fin):
    """ Suma de la cantidad de dinero recibido por todas las ventas en un lapso de tiempo entre fecha_inicio y fecha_fin """
    mcaja_hoy = MovimientoCaja.objects.filter(fecha__gte = fecha_inicio).filter(fecha__lte = fecha_fin)
    return  valor_o_cero(mcaja_hoy.filter(Q(tipo='P')).aggregate(r=Sum('cantidad'))['r']) + valor_o_cero(mcaja_hoy.filter(Q(tipo='N')).aggregate(r=Sum('cantidad'))['r'])

redpajama_set_name: RedPajamaGithub


text:
NS_ASSUME_NONNULL_BEGIN

@interface NSError (ZKAdd)

+ (NSError *)errorWithDomain:(NSString *)domain code:(NSInteger)code description:(NSString *)description failureReason:(NSString *)failReason;
+ (NSError *)errorWithDomain:(NSString *)domain code:(NSInteger)code description:(NSString *)description;

@end

NS_ASSUME_NONNULL_END


text:
<!doctype html>

    <meta charset="UTF-8">

    <link rel="stylesheet" href="lib/onsen/css/onsenui.css">
    <link rel="stylesheet" href="styles/onsen-css-components-purple-theme.css">
    <link rel="stylesheet" href="styles/slider-menu	.css">
    <link rel="stylesheet" href="styles/list-style.css">

	<ons-sliding-menu main-page="page1.html" menu-page="menu.html" side="left" max-slide-distance="250px" var="menu">
	</ons-sliding-menu>

<ons-template id="page1.html">
  <ons-page>
    <ons-toolbar>

        <ons-toolbar-button ng-click="menu.toggleMenu()"><ons-icon icon="ion-navicon" style="font-size: 32px; width: 1em;"></ons-icon></ons-toolbar-button>

Page 1

    </ons-toolbar>

Page1 Contents

  </ons-page>
</ons-template>

<ons-template id="page2.html">
  <ons-page>
    <ons-toolbar>

        <ons-toolbar-button onclick="menu.toggleMenu()"><ons-icon icon="ion-navicon" style="font-size: 32px; width: 1em;"></ons-icon></ons-toolbar-button>

Page 2

    </ons-toolbar>

Page2 Contents

  </ons-page>
</ons-template>

<ons-template id="page3.html">
  <ons-page>
    <ons-toolbar>

        <ons-toolbar-button onclick="menu.toggleMenu()"><ons-icon icon="ion-navicon" style="font-size: 32px; width: 1em;"></ons-icon></ons-toolbar-button>

Page 3

    </ons-toolbar>

Page3 Contents

  </ons-page>
</ons-template>

<ons-template id="page4.html">
  <ons-page>
    <ons-toolbar>

        <ons-toolbar-button onclick="menu.toggleMenu()"><ons-icon icon="ion-navicon" style="font-size: 32px; width: 1em;"></ons-icon></ons-toolbar-button>

Page 4

    </ons-toolbar>

This is a card.

This card shows some content on the front and some on the back, and can be flipped by hovering on a desktop, or by swiping across on a touch device.

The back of the card is intentionally long, so that we can show both an example where the card is constrained to a square and where it is as long as the content on either side requires.

This card shows some content on the front and some on the back, and can be flipped by hovering on a desktop, or by swiping across on a touch device.

The back of the card is intentionally long, so that we can show both an example where the card is constrained to a square and where it is as long as the content on either side requires.

  </ons-page>
</ons-template>

<ons-template id="menu.html">
<ons-navigator title="Navigator" var="myNavigator">
  <ons-list>
	<ons-list-item>
      <ons-back-button>Home</ons-back-button>
    </ons-list-item>
    <ons-list-item modifier="chevron"  onclick="menu.setMainPage('page1.html', {closeMenu: true})">
      page1.html
    </ons-list-item>
    <ons-list-item modifier="chevron" onclick="menu.setMainPage('page2.html', {closeMenu: true})">
      page2.html
    </ons-list-item>
	<ons-list-item modifier="chevron" onclick="menu.setMainPage('page3.html', {closeMenu: true})">
      page3.html
    </ons-list-item>
	<ons-list-item modifier="chevron" onclick="menu.setMainPage('page4.html', {closeMenu: true})">
      page4.html
    </ons-list-item>
  </ons-list>
  </ons-navigator>
</ons-template>


text:
#ifndef FontConfigDirect_DEFINED
#define FontConfigDirect_DEFINED

#include <map>
#include <string>

#include "SkThread.h"
#include "SkFontHost_fontconfig_impl.h"

class FontConfigDirect : public FontConfigInterface {
  public:
    FontConfigDirect();

    // FontConfigInterface implementation. Thread safe.
    virtual bool Match(std::string* result_family, unsigned* result_fileid,
                       bool fileid_valid, unsigned fileid,
                       const std::string& family, bool* is_bold, bool* is_italic);
    virtual int Open(unsigned fileid);

  private:
    SkMutex mutex_;
    std::map<unsigned, std::string> fileid_to_filename_;
    std::map<std::string, unsigned> filename_to_fileid_;
    unsigned next_file_id_;
};

#endif  // FontConfigDirect_DEFINED


text:
package org.eclipse.deeplearning4j.dl4jcore.nn.conf.weightnoise;

import lombok.AllArgsConstructor;
import lombok.Data;
import org.deeplearning4j.BaseDL4JTest;
import org.eclipse.deeplearning4j.dl4jcore.TestUtils;
import org.deeplearning4j.datasets.iterator.ExistingDataSetIterator;
import org.deeplearning4j.nn.api.Layer;
import org.deeplearning4j.nn.conf.ComputationGraphConfiguration;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.distribution.NormalDistribution;
import org.deeplearning4j.nn.conf.layers.BaseLayer;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.conf.weightnoise.DropConnect;
import org.deeplearning4j.nn.conf.weightnoise.IWeightNoise;
import org.deeplearning4j.nn.conf.weightnoise.WeightNoise;
import org.deeplearning4j.nn.graph.ComputationGraph;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.junit.jupiter.api.Test;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.buffer.DataType;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.impl.reduce.longer.MatchCondition;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.conditions.Conditions;
import org.nd4j.linalg.schedule.ScheduleType;
import org.nd4j.linalg.schedule.SigmoidSchedule;
import org.deeplearning4j.nn.workspace.LayerWorkspaceMgr;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import static org.junit.jupiter.api.Assertions.*;

public class TestWeightNoise extends BaseDL4JTest {

    @Test
    public void testWeightNoiseConfigJson() {
        IWeightNoise[] weightNoises = new IWeightNoise[]{
                new DropConnect(0.5),
                new DropConnect(new SigmoidSchedule(ScheduleType.ITERATION, 0.5, 0.5, 100)),
                new WeightNoise(new NormalDistribution(0, 0.1))
        };

        for (IWeightNoise wn : weightNoises) {
            MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                    .weightNoise(wn)
                    .list()
                    .layer(new DenseLayer.Builder().nIn(10).nOut(10).build())
                    .layer(new DenseLayer.Builder().nIn(10).nOut(10).weightNoise(new DropConnect(0.25)).build())
                    .layer(new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build())
                    .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();

            assertEquals(wn, ((BaseLayer) net.getLayer(0).conf().getLayer()).getWeightNoise());
            assertEquals(new DropConnect(0.25), ((BaseLayer) net.getLayer(1).conf().getLayer()).getWeightNoise());
            assertEquals(wn, ((BaseLayer) net.getLayer(2).conf().getLayer()).getWeightNoise());

            TestUtils.testModelSerialization(net);

            ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder()
                    .weightNoise(wn)
                    .graphBuilder()
                    .addInputs("in")
                    .layer("0", new DenseLayer.Builder().nIn(10).nOut(10).build(), "in")
                    .layer("1", new DenseLayer.Builder().nIn(10).nOut(10).weightNoise(new DropConnect(0.25)).build(), "0")
                    .layer("2", new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build(), "1")
                    .setOutputs("2")
                    .build();

            ComputationGraph graph = new ComputationGraph(conf2);
            graph.init();

            assertEquals(wn, ((BaseLayer) graph.getLayer(0).conf().getLayer()).getWeightNoise());
            assertEquals(new DropConnect(0.25), ((BaseLayer) graph.getLayer(1).conf().getLayer()).getWeightNoise());
            assertEquals(wn, ((BaseLayer) graph.getLayer(2).conf().getLayer()).getWeightNoise());

            TestUtils.testModelSerialization(graph);

            graph.fit(new DataSet(Nd4j.create(1,10), Nd4j.create(1,10)));
        }
    }

    @Test
    public void testCalls() {

        List<DataSet> trainData = new ArrayList<>();
        trainData.add(new DataSet(Nd4j.rand(5, 10), Nd4j.rand(5, 10)));
        trainData.add(new DataSet(Nd4j.rand(5, 10), Nd4j.rand(5, 10)));
        trainData.add(new DataSet(Nd4j.rand(5, 10), Nd4j.rand(5, 10)));

        List<List<WeightNoiseCall>> expCalls = new ArrayList<>();
        for (int i = 0; i < 3; i++) {
            List<WeightNoiseCall> expCallsForLayer = new ArrayList<>();
            expCallsForLayer.add(new WeightNoiseCall(i, "W", 0, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "b", 0, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "W", 1, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "b", 1, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "W", 2, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "b", 2, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "W", 3, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "b", 3, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "W", 4, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "b", 4, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "W", 5, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(i, "b", 5, 1, true));

            //2 test calls
            expCallsForLayer.add(new WeightNoiseCall(i, "W", 6, 2, false));
            expCallsForLayer.add(new WeightNoiseCall(i, "b", 6, 2, false));

            expCalls.add(expCallsForLayer);
        }

        CustomWeightNoise wn1 = new CustomWeightNoise();
        CustomWeightNoise wn2 = new CustomWeightNoise();
        CustomWeightNoise wn3 = new CustomWeightNoise();

        List<CustomWeightNoise> list = Arrays.asList(wn1, wn2, wn3);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .list()
                .layer(new DenseLayer.Builder().nIn(10).nOut(10).weightNoise(wn1).build())
                .layer(new DenseLayer.Builder().nIn(10).nOut(10).weightNoise(wn2).build())
                .layer(new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).weightNoise(wn3).build())
                .build();

        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        net.fit(new ExistingDataSetIterator(trainData.iterator()));
        net.fit(new ExistingDataSetIterator(trainData.iterator()));
        net.output(trainData.get(0).getFeatures());

        for (int i = 0; i < 3; i++) {
            assertEquals(expCalls.get(i), list.get(i).getAllCalls());
        }

        wn1 = new CustomWeightNoise();
        wn2 = new CustomWeightNoise();
        wn3 = new CustomWeightNoise();
        list = Arrays.asList(wn1, wn2, wn3);

        ComputationGraphConfiguration conf2 = new NeuralNetConfiguration.Builder()
                .graphBuilder()
                .addInputs("in")
                .layer("0", new DenseLayer.Builder().nIn(10).nOut(10).weightNoise(wn1).build(), "in")
                .layer("1", new DenseLayer.Builder().nIn(10).nOut(10).weightNoise(wn2).build(), "0")
                .layer("2", new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).weightNoise(wn3).build(), "1")
                .setOutputs("2")
                .build();

        ComputationGraph graph = new ComputationGraph(conf2);
        graph.init();

        int[] layerIdxs = new int[]{graph.getLayer(0).getIndex(), graph.getLayer(1).getIndex(), graph.getLayer(2).getIndex()};

        expCalls.clear();
        for (int i = 0; i < 3; i++) {
            List<WeightNoiseCall> expCallsForLayer = new ArrayList<>();
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "W", 0, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "b", 0, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "W", 1, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "b", 1, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "W", 2, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "b", 2, 0, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "W", 3, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "b", 3, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "W", 4, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "b", 4, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "W", 5, 1, true));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "b", 5, 1, true));

            //2 test calls
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "W", 6, 2, false));
            expCallsForLayer.add(new WeightNoiseCall(layerIdxs[i], "b", 6, 2, false));

            expCalls.add(expCallsForLayer);
        }

        graph.fit(new ExistingDataSetIterator(trainData.iterator()));
        graph.fit(new ExistingDataSetIterator(trainData.iterator()));
        graph.output(trainData.get(0).getFeatures());

        for (int i = 0; i < 3; i++) {
            assertEquals(expCalls.get(i), list.get(i).getAllCalls(), String.valueOf(i));
        }

    }

    @Data
    private static class CustomWeightNoise implements IWeightNoise {

        private List<WeightNoiseCall> allCalls = new ArrayList<>();

        @Override
        public INDArray getParameter(Layer layer, String paramKey, int iteration, int epoch, boolean train, LayerWorkspaceMgr workspaceMgr) {
            allCalls.add(new WeightNoiseCall(layer.getIndex(), paramKey, iteration, epoch, train));
            return layer.getParam(paramKey);
        }

        @Override
        public IWeightNoise clone() {
            return new CustomWeightNoise();
        }
    }

    @AllArgsConstructor
    @Data
    private static class WeightNoiseCall {
        private int layerIdx;
        private String paramKey;
        private int iter;
        private int epoch;
        private boolean train;
    }

    @Test
    public void testDropConnectValues() {
        Nd4j.getRandom().setSeed(12345);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                .weightInit(WeightInit.ONES)
                .list()
                .layer(new OutputLayer.Builder().nIn(10).nOut(10).activation(Activation.SOFTMAX).build())
                .build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        Layer l = net.getLayer(0);
        DropConnect d = new DropConnect(0.5);

        INDArray outTest = d.getParameter(l, "W", 0, 0, false, LayerWorkspaceMgr.noWorkspaces());
        assertTrue(l.getParam("W") == outTest);    //Should be same object
        INDArray outTrain = d.getParameter(l, "W", 0, 0, true, LayerWorkspaceMgr.noWorkspaces());
        assertNotEquals(l.getParam("W"), outTrain);

        assertEquals(l.getParam("W"), Nd4j.ones(DataType.FLOAT, 10, 10));

        int countZeros = Nd4j.getExecutioner().exec(new MatchCondition(outTrain, Conditions.equals(0))).getInt(0);
        int countOnes = Nd4j.getExecutioner().exec(new MatchCondition(outTrain, Conditions.equals(1))).getInt(0);

        assertEquals(100, countZeros + countOnes);  //Should only be 0 or 2
        //Stochastic, but this should hold for most cases
        assertTrue(countZeros >= 25 && countZeros <= 75);
        assertTrue(countOnes >= 25 && countOnes <= 75);
    }

}


text:
package com.tigertext.lucene.ext;

import java.util.logging.Logger;

import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.ext.ExtensionQuery;
import org.apache.lucene.queryParser.ext.ParserExtension;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.function.CustomScoreQuery;
import org.apache.lucene.search.function.ValueSource;
import org.apache.lucene.search.function.ValueSourceQuery;

import com.tigertext.lucene.DocumentTranslator;

/**
 * Extension to run ".erlang" queries. These queries should be in the form
 * Field.erlang:Mod:Fun:Args where -spec Mod:Fun(Args, string()) -> false |
 * float(). If Mod:Fun(Args..., Value) returns false, the document doesn't match
 * Otherwise the result is the score. Args will be parsed on the Erlang node
 * using {ok, A, _} = erl_scan:string(Args++"."). {ok, B} =
 * erl_parse:parse_exprs(A). {value, Arguments, _} = erl_eval:exprs(B, []). So
 * the final call will be erlang:apply(Mod, Fun, Arguments ++ [Value]).
 *
 * @author Fernando Benavides <elbrujohalcon@inaka.net>
 */
public class ErlangParserExtension extends ParserExtension {
	private static final Logger	jlog	= Logger.getLogger(ErlangParserExtension.class
												.getName());
	private DocumentTranslator	translator;

	/**
	 * Creates a new parser extension
	 *
	 * @param translator
	 *            Used to determine the type of fields
	 */
	public ErlangParserExtension(DocumentTranslator translator) {
		super();
		this.translator = translator;
	}

	@Override
	public Query parse(ExtensionQuery extQuery) throws ParseException {
		String key = extQuery.getField();
		String[] modFun = extQuery.getRawQueryString().split(":");
		if (modFun.length < 2) {
			throw new ParseException(
					"erlang queries expect values in <mod>:<fun> or <mod>:<fun>:<args> format");
		} else {
			String mod = modFun[0], fun = modFun[1], args;
			if (modFun.length == 2) {
				args = "[]";
			} else if (modFun.length == 3) {
				args = modFun[2];
			} else {
				args = "";
				String colon = "";
				for (int i = 2; i < modFun.length; i++) {
					args += colon + modFun[i];
					colon = ":";
				}
			}

			jlog.fine("erlang query using " + key + " and " + mod + ":" + fun
					+ "(" + args + "...");

			Filter filter = new ErlangFilter(mod, fun, args, key,
					this.translator.getFieldType(key));

			ValueSource valSrc = new ErlangValueSource((ErlangFilter) filter);

			return new CustomScoreQuery(new ConstantScoreQuery(filter),
					new ValueSourceQuery(valSrc));
		}
	}

}


text:
using NMF.Collections.Generic;
using NMF.Collections.ObjectModel;
using NMF.Expressions;
using NMF.Expressions.Linq;
using NMF.Models;
using NMF.Models.Collections;
using NMF.Models.Expressions;
using NMF.Models.Meta;
using NMF.Models.Repository;
using NMF.Serialization;
using NMF.Utilities;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.ComponentModel;
using System.Diagnostics;
using System.Linq;

namespace NMF.Interop.Ecore
{

    /// <summary>
    /// The default implementation of the EAnnotation class
    /// </summary>
    [XmlNamespaceAttribute("http://www.eclipse.org/emf/2002/Ecore")]
    [XmlNamespacePrefixAttribute("ecore")]
    [ModelRepresentationClassAttribute("http://www.eclipse.org/emf/2002/Ecore#//EAnnotation/")]
    public class EAnnotation : EModelElement, IEAnnotation, IModelElement
    {

        /// <summary>
        /// The backing field for the Source property
        /// </summary>
        private string _source;

        /// <summary>
        /// The backing field for the Details property
        /// </summary>
        private ObservableCompositionOrderedSet<IEStringToStringMapEntry> _details;

        /// <summary>
        /// The backing field for the Contents property
        /// </summary>
        private ObservableCompositionOrderedSet<IModelElement> _contents;

        /// <summary>
        /// The backing field for the References property
        /// </summary>
        private ObservableAssociationOrderedSet<IModelElement> _references;

        private static IClass _classInstance;

        public EAnnotation()
        {
            this._details = new ObservableCompositionOrderedSet<IEStringToStringMapEntry>(this);
            this._details.CollectionChanging += this.DetailsCollectionChanging;
            this._details.CollectionChanged += this.DetailsCollectionChanged;
            this._contents = new ObservableCompositionOrderedSet<IModelElement>(this);
            this._contents.CollectionChanging += this.ContentsCollectionChanging;
            this._contents.CollectionChanged += this.ContentsCollectionChanged;
            this._references = new ObservableAssociationOrderedSet<IModelElement>();
            this._references.CollectionChanging += this.ReferencesCollectionChanging;
            this._references.CollectionChanged += this.ReferencesCollectionChanged;
        }

        /// <summary>
        /// The source property
        /// </summary>
        [XmlElementNameAttribute("source")]
        [XmlAttributeAttribute(true)]
        public virtual string Source
        {
            get
            {
                return this._source;
            }
            set
            {
                if ((this._source != value))
                {
                    string old = this._source;
                    ValueChangedEventArgs e = new ValueChangedEventArgs(old, value);
                    this.OnSourceChanging(e);
                    this.OnPropertyChanging("Source", e);
                    this._source = value;
                    this.OnSourceChanged(e);
                    this.OnPropertyChanged("Source", e);
                }
            }
        }

        /// <summary>
        /// The details property
        /// </summary>
        [DesignerSerializationVisibilityAttribute(DesignerSerializationVisibility.Content)]
        [XmlElementNameAttribute("details")]
        [XmlAttributeAttribute(false)]
        [ContainmentAttribute()]
        [ConstantAttribute()]
        public virtual IOrderedSetExpression<IEStringToStringMapEntry> Details
        {
            get
            {
                return this._details;
            }
        }

        /// <summary>
        /// The eModelElement property
        /// </summary>
        [XmlElementNameAttribute("eModelElement")]
        [DesignerSerializationVisibilityAttribute(DesignerSerializationVisibility.Hidden)]
        [XmlAttributeAttribute(true)]
        [XmlOppositeAttribute("eAnnotations")]
        public virtual IEModelElement EModelElement
        {
            get
            {
                return ModelHelper.CastAs<IEModelElement>(this.Parent);
            }
            set
            {
                this.Parent = value;
            }
        }

        /// <summary>
        /// The contents property
        /// </summary>
        [DesignerSerializationVisibilityAttribute(DesignerSerializationVisibility.Content)]
        [XmlElementNameAttribute("contents")]
        [XmlAttributeAttribute(false)]
        [ContainmentAttribute()]
        [ConstantAttribute()]
        public virtual IOrderedSetExpression<IModelElement> Contents
        {
            get
            {
                return this._contents;
            }
        }

        /// <summary>
        /// The references property
        /// </summary>
        [DesignerSerializationVisibilityAttribute(DesignerSerializationVisibility.Content)]
        [XmlElementNameAttribute("references")]
        [XmlAttributeAttribute(true)]
        [ConstantAttribute()]
        public virtual IOrderedSetExpression<IModelElement> References
        {
            get
            {
                return this._references;
            }
        }

        /// <summary>
        /// Gets the child model elements of this model element
        /// </summary>
        public override IEnumerableExpression<IModelElement> Children
        {
            get
            {
                return base.Children.Concat(new EAnnotationChildrenCollection(this));
            }
        }

        /// <summary>
        /// Gets the referenced model elements of this model element
        /// </summary>
        public override IEnumerableExpression<IModelElement> ReferencedElements
        {
            get
            {
                return base.ReferencedElements.Concat(new EAnnotationReferencedElementsCollection(this));
            }
        }

        /// <summary>
        /// Gets the Class model for this type
        /// </summary>
        public new static IClass ClassInstance
        {
            get
            {
                if ((_classInstance == null))
                {
                    _classInstance = ((IClass)(MetaRepository.Instance.Resolve("http://www.eclipse.org/emf/2002/Ecore#//EAnnotation/")));
                }
                return _classInstance;
            }
        }

        /// <summary>
        /// Gets fired before the Source property changes its value
        /// </summary>
        public event System.EventHandler<ValueChangedEventArgs> SourceChanging;

        /// <summary>
        /// Gets fired when the Source property changed its value
        /// </summary>
        public event System.EventHandler<ValueChangedEventArgs> SourceChanged;

        /// <summary>
        /// Gets fired before the EModelElement property changes its value
        /// </summary>
        public event System.EventHandler<ValueChangedEventArgs> EModelElementChanging;

        /// <summary>
        /// Gets fired when the EModelElement property changed its value
        /// </summary>
        public event System.EventHandler<ValueChangedEventArgs> EModelElementChanged;

        /// <summary>
        /// Raises the SourceChanging event
        /// </summary>
        /// <param name="eventArgs">The event data</param>
        protected virtual void OnSourceChanging(ValueChangedEventArgs eventArgs)
        {
            System.EventHandler<ValueChangedEventArgs> handler = this.SourceChanging;
            if ((handler != null))
            {
                handler.Invoke(this, eventArgs);
            }
        }

        /// <summary>
        /// Raises the SourceChanged event
        /// </summary>
        /// <param name="eventArgs">The event data</param>
        protected virtual void OnSourceChanged(ValueChangedEventArgs eventArgs)
        {
            System.EventHandler<ValueChangedEventArgs> handler = this.SourceChanged;
            if ((handler != null))
            {
                handler.Invoke(this, eventArgs);
            }
        }

        /// <summary>
        /// Forwards CollectionChanging notifications for the Details property to the parent model element
        /// </summary>
        /// <param name="sender">The collection that raised the change</param>
        /// <param name="e">The original event data</param>
        private void DetailsCollectionChanging(object sender, System.Collections.Specialized.NotifyCollectionChangedEventArgs e)
        {
            this.OnCollectionChanging("Details", e);
        }

        /// <summary>
        /// Forwards CollectionChanged notifications for the Details property to the parent model element
        /// </summary>
        /// <param name="sender">The collection that raised the change</param>
        /// <param name="e">The original event data</param>
        private void DetailsCollectionChanged(object sender, System.Collections.Specialized.NotifyCollectionChangedEventArgs e)
        {
            this.OnCollectionChanged("Details", e);
        }

        /// <summary>
        /// Raises the EModelElementChanging event
        /// </summary>
        /// <param name="eventArgs">The event data</param>
        protected virtual void OnEModelElementChanging(ValueChangedEventArgs eventArgs)
        {
            System.EventHandler<ValueChangedEventArgs> handler = this.EModelElementChanging;
            if ((handler != null))
            {
                handler.Invoke(this, eventArgs);
            }
        }

        /// <summary>
        /// Gets called when the parent model element of the current model element is about to change
        /// </summary>
        /// <param name="oldParent">The old parent model element</param>
        /// <param name="newParent">The new parent model element</param>
        protected override void OnParentChanging(IModelElement newParent, IModelElement oldParent)
        {
            IEModelElement oldEModelElement = ModelHelper.CastAs<IEModelElement>(oldParent);
            IEModelElement newEModelElement = ModelHelper.CastAs<IEModelElement>(newParent);
            ValueChangedEventArgs e = new ValueChangedEventArgs(oldEModelElement, newEModelElement);
            this.OnEModelElementChanging(e);
            this.OnPropertyChanging("EModelElement");
        }

        /// <summary>
        /// Raises the EModelElementChanged event
        /// </summary>
        /// <param name="eventArgs">The event data</param>
        protected virtual void OnEModelElementChanged(ValueChangedEventArgs eventArgs)
        {
            System.EventHandler<ValueChangedEventArgs> handler = this.EModelElementChanged;
            if ((handler != null))
            {
                handler.Invoke(this, eventArgs);
            }
        }

        /// <summary>
        /// Gets called when the parent model element of the current model element changes
        /// </summary>
        /// <param name="oldParent">The old parent model element</param>
        /// <param name="newParent">The new parent model element</param>
        protected override void OnParentChanged(IModelElement newParent, IModelElement oldParent)
        {
            IEModelElement oldEModelElement = ModelHelper.CastAs<IEModelElement>(oldParent);
            IEModelElement newEModelElement = ModelHelper.CastAs<IEModelElement>(newParent);
            if ((oldEModelElement != null))
            {
                oldEModelElement.EAnnotations.Remove(this);
            }
            if ((newEModelElement != null))
            {
                newEModelElement.EAnnotations.Add(this);
            }
            ValueChangedEventArgs e = new ValueChangedEventArgs(oldEModelElement, newEModelElement);
            this.OnEModelElementChanged(e);
            this.OnPropertyChanged("EModelElement", e);
            base.OnParentChanged(newParent, oldParent);
        }

        /// <summary>
        /// Forwards CollectionChanging notifications for the Contents property to the parent model element
        /// </summary>
        /// <param name="sender">The collection that raised the change</param>
        /// <param name="e">The original event data</param>
        private void ContentsCollectionChanging(object sender, System.Collections.Specialized.NotifyCollectionChangedEventArgs e)
        {
            this.OnCollectionChanging("Contents", e);
        }

        /// <summary>
        /// Forwards CollectionChanged notifications for the Contents property to the parent model element
        /// </summary>
        /// <param name="sender">The collection that raised the change</param>
        /// <param name="e">The original event data</param>
        private void ContentsCollectionChanged(object sender, System.Collections.Specialized.NotifyCollectionChangedEventArgs e)
        {
            this.OnCollectionChanged("Contents", e);
        }

        /// <summary>
        /// Forwards CollectionChanging notifications for the References property to the parent model element
        /// </summary>
        /// <param name="sender">The collection that raised the change</param>
        /// <param name="e">The original event data</param>
        private void ReferencesCollectionChanging(object sender, System.Collections.Specialized.NotifyCollectionChangedEventArgs e)
        {
            this.OnCollectionChanging("References", e);
        }

        /// <summary>
        /// Forwards CollectionChanged notifications for the References property to the parent model element
        /// </summary>
        /// <param name="sender">The collection that raised the change</param>
        /// <param name="e">The original event data</param>
        private void ReferencesCollectionChanged(object sender, System.Collections.Specialized.NotifyCollectionChangedEventArgs e)
        {
            this.OnCollectionChanged("References", e);
        }

        /// <summary>
        /// Gets the relative URI fragment for the given child model element
        /// </summary>
        /// <returns>A fragment of the relative URI</returns>
        /// <param name="element">The element that should be looked for</param>
        protected override string GetRelativePathForNonIdentifiedChild(IModelElement element)
        {
            int detailsIndex = ModelHelper.IndexOfReference(this.Details, element);
            if ((detailsIndex != -1))
            {
                return ModelHelper.CreatePath("details", detailsIndex);
            }
            int contentsIndex = ModelHelper.IndexOfReference(this.Contents, element);
            if ((contentsIndex != -1))
            {
                return ModelHelper.CreatePath("contents", contentsIndex);
            }
            return base.GetRelativePathForNonIdentifiedChild(element);
        }

        /// <summary>
        /// Resolves the given URI to a child model element
        /// </summary>
        /// <returns>The model element or null if it could not be found</returns>
        /// <param name="reference">The requested reference name</param>
        /// <param name="index">The index of this reference</param>
        protected override IModelElement GetModelElementForReference(string reference, int index)
        {
            if ((reference == "DETAILS"))
            {
                if ((index < this.Details.Count))
                {
                    return this.Details[index];
                }
                else
                {
                    return null;
                }
            }
            if ((reference == "CONTENTS"))
            {
                if ((index < this.Contents.Count))
                {
                    return this.Contents[index];
                }
                else
                {
                    return null;
                }
            }
            return base.GetModelElementForReference(reference, index);
        }

        /// <summary>
        /// Resolves the given attribute name
        /// </summary>
        /// <returns>The attribute value or null if it could not be found</returns>
        /// <param name="attribute">The requested attribute name</param>
        /// <param name="index">The index of this attribute</param>
        protected override object GetAttributeValue(string attribute, int index)
        {
            if ((attribute == "SOURCE"))
            {
                return this.Source;
            }
            return base.GetAttributeValue(attribute, index);
        }

        /// <summary>
        /// Gets the Model element collection for the given feature
        /// </summary>
        /// <returns>A non-generic list of elements</returns>
        /// <param name="feature">The requested feature</param>
        protected override System.Collections.IList GetCollectionForFeature(string feature)
        {
            if ((feature == "DETAILS"))
            {
                return this._details;
            }
            if ((feature == "CONTENTS"))
            {
                return this._contents;
            }
            if ((feature == "REFERENCES"))
            {
                return this._references;
            }
            return base.GetCollectionForFeature(feature);
        }

        /// <summary>
        /// Sets a value to the given feature
        /// </summary>
        /// <param name="feature">The requested feature</param>
        /// <param name="value">The value that should be set to that feature</param>
        protected override void SetFeature(string feature, object value)
        {
            if ((feature == "EMODELELEMENT"))
            {
                this.EModelElement = ((IEModelElement)(value));
                return;
            }
            if ((feature == "SOURCE"))
            {
                this.Source = ((string)(value));
                return;
            }
            base.SetFeature(feature, value);
        }

        /// <summary>
        /// Gets the property expression for the given attribute
        /// </summary>
        /// <returns>An incremental property expression</returns>
        /// <param name="attribute">The requested attribute in upper case</param>
        protected override NMF.Expressions.INotifyExpression<object> GetExpressionForAttribute(string attribute)
        {
            if ((attribute == "EMODELELEMENT"))
            {
                return new EModelElementProxy(this);
            }
            return base.GetExpressionForAttribute(attribute);
        }

        /// <summary>
        /// Gets the property expression for the given reference
        /// </summary>
        /// <returns>An incremental property expression</returns>
        /// <param name="reference">The requested reference in upper case</param>
        protected override NMF.Expressions.INotifyExpression<NMF.Models.IModelElement> GetExpressionForReference(string reference)
        {
            if ((reference == "EMODELELEMENT"))
            {
                return new EModelElementProxy(this);
            }
            return base.GetExpressionForReference(reference);
        }

        /// <summary>
        /// Gets the Class for this model element
        /// </summary>
        public override IClass GetClass()
        {
            if ((_classInstance == null))
            {
                _classInstance = ((IClass)(MetaRepository.Instance.Resolve("http://www.eclipse.org/emf/2002/Ecore#//EAnnotation/")));
            }
            return _classInstance;
        }

        /// <summary>
        /// The collection class to to represent the children of the EAnnotation class
        /// </summary>
        public class EAnnotationChildrenCollection : ReferenceCollection, ICollectionExpression<IModelElement>, ICollection<IModelElement>
        {

            private EAnnotation _parent;

            /// <summary>
            /// Creates a new instance
            /// </summary>
            public EAnnotationChildrenCollection(EAnnotation parent)
            {
                this._parent = parent;
            }

            /// <summary>
            /// Gets the amount of elements contained in this collection
            /// </summary>
            public override int Count
            {
                get
                {
                    int count = 0;
                    count = (count + this._parent.Details.Count);
                    count = (count + this._parent.Contents.Count);
                    return count;
                }
            }

            protected override void AttachCore()
            {
                this._parent.Details.AsNotifiable().CollectionChanged += this.PropagateCollectionChanges;
                this._parent.Contents.AsNotifiable().CollectionChanged += this.PropagateCollectionChanges;
            }

            protected override void DetachCore()
            {
                this._parent.Details.AsNotifiable().CollectionChanged -= this.PropagateCollectionChanges;
                this._parent.Contents.AsNotifiable().CollectionChanged -= this.PropagateCollectionChanges;
            }

            /// <summary>
            /// Adds the given element to the collection
            /// </summary>
            /// <param name="item">The item to add</param>
            public override void Add(IModelElement item)
            {
                IEStringToStringMapEntry detailsCasted = item.As<IEStringToStringMapEntry>();
                if ((detailsCasted != null))
                {
                    this._parent.Details.Add(detailsCasted);
                }
                this._parent.Contents.Add(item);
            }

            /// <summary>
            /// Clears the collection and resets all references that implement it.
            /// </summary>
            public override void Clear()
            {
                this._parent.Details.Clear();
                this._parent.Contents.Clear();
            }

            /// <summary>
            /// Gets a value indicating whether the given element is contained in the collection
            /// </summary>
            /// <returns>True, if it is contained, otherwise False</returns>
            /// <param name="item">The item that should be looked out for</param>
            public override bool Contains(IModelElement item)
            {
                if (this._parent.Details.Contains(item))
                {
                    return true;
                }
                if (this._parent.Contents.Contains(item))
                {
                    return true;
                }
                return false;
            }

            /// <summary>
            /// Copies the contents of the collection to the given array starting from the given array index
            /// </summary>
            /// <param name="array">The array in which the elements should be copied</param>
            /// <param name="arrayIndex">The starting index</param>
            public override void CopyTo(IModelElement[] array, int arrayIndex)
            {
                IEnumerator<IModelElement> detailsEnumerator = this._parent.Details.GetEnumerator();
                try
                {
                    for (
                    ; detailsEnumerator.MoveNext();
                    )
                    {
                        array[arrayIndex] = detailsEnumerator.Current;
                        arrayIndex = (arrayIndex + 1);
                    }
                }
                finally
                {
                    detailsEnumerator.Dispose();
                }
                this._parent.Contents.CopyTo(array, arrayIndex);
                arrayIndex = (arrayIndex + this._parent.Contents.Count);
            }

            /// <summary>
            /// Removes the given item from the collection
            /// </summary>
            /// <returns>True, if the item was removed, otherwise False</returns>
            /// <param name="item">The item that should be removed</param>
            public override bool Remove(IModelElement item)
            {
                IEStringToStringMapEntry eStringToStringMapEntryItem = item.As<IEStringToStringMapEntry>();
                if (((eStringToStringMapEntryItem != null)
                            && this._parent.Details.Remove(eStringToStringMapEntryItem)))
                {
                    return true;
                }
                if (this._parent.Contents.Remove(item))
                {
                    return true;
                }
                return false;
            }

            /// <summary>
            /// Gets an enumerator that enumerates the collection
            /// </summary>
            /// <returns>A generic enumerator</returns>
            public override IEnumerator<IModelElement> GetEnumerator()
            {
                return Enumerable.Empty<IModelElement>().Concat(this._parent.Details).Concat(this._parent.Contents).GetEnumerator();
            }
        }

        /// <summary>
        /// The collection class to to represent the children of the EAnnotation class
        /// </summary>
        public class EAnnotationReferencedElementsCollection : ReferenceCollection, ICollectionExpression<IModelElement>, ICollection<IModelElement>
        {

            private EAnnotation _parent;

            /// <summary>
            /// Creates a new instance
            /// </summary>
            public EAnnotationReferencedElementsCollection(EAnnotation parent)
            {
                this._parent = parent;
            }

            /// <summary>
            /// Gets the amount of elements contained in this collection
            /// </summary>
            public override int Count
            {
                get
                {
                    int count = 0;
                    count = (count + this._parent.Details.Count);
                    if ((this._parent.EModelElement != null))
                    {
                        count = (count + 1);
                    }
                    count = (count + this._parent.Contents.Count);
                    count = (count + this._parent.References.Count);
                    return count;
                }
            }

            protected override void AttachCore()
            {
                this._parent.Details.AsNotifiable().CollectionChanged += this.PropagateCollectionChanges;
                this._parent.EModelElementChanged += this.PropagateValueChanges;
                this._parent.Contents.AsNotifiable().CollectionChanged += this.PropagateCollectionChanges;
                this._parent.References.AsNotifiable().CollectionChanged += this.PropagateCollectionChanges;
            }

            protected override void DetachCore()
            {
                this._parent.Details.AsNotifiable().CollectionChanged -= this.PropagateCollectionChanges;
                this._parent.EModelElementChanged -= this.PropagateValueChanges;
                this._parent.Contents.AsNotifiable().CollectionChanged -= this.PropagateCollectionChanges;
                this._parent.References.AsNotifiable().CollectionChanged -= this.PropagateCollectionChanges;
            }

            /// <summary>
            /// Adds the given element to the collection
            /// </summary>
            /// <param name="item">The item to add</param>
            public override void Add(IModelElement item)
            {
                IEStringToStringMapEntry detailsCasted = item.As<IEStringToStringMapEntry>();
                if ((detailsCasted != null))
                {
                    this._parent.Details.Add(detailsCasted);
                }
                if ((this._parent.EModelElement == null))
                {
                    IEModelElement eModelElementCasted = item.As<IEModelElement>();
                    if ((eModelElementCasted != null))
                    {
                        this._parent.EModelElement = eModelElementCasted;
                        return;
                    }
                }
                this._parent.Contents.Add(item);
            }

            /// <summary>
            /// Clears the collection and resets all references that implement it.
            /// </summary>
            public override void Clear()
            {
                this._parent.Details.Clear();
                this._parent.EModelElement = null;
                this._parent.Contents.Clear();
                this._parent.References.Clear();
            }

            /// <summary>
            /// Gets a value indicating whether the given element is contained in the collection
            /// </summary>
            /// <returns>True, if it is contained, otherwise False</returns>
            /// <param name="item">The item that should be looked out for</param>
            public override bool Contains(IModelElement item)
            {
                if (this._parent.Details.Contains(item))
                {
                    return true;
                }
                if ((item == this._parent.EModelElement))
                {
                    return true;
                }
                if (this._parent.Contents.Contains(item))
                {
                    return true;
                }
                if (this._parent.References.Contains(item))
                {
                    return true;
                }
                return false;
            }

            /// <summary>
            /// Copies the contents of the collection to the given array starting from the given array index
            /// </summary>
            /// <param name="array">The array in which the elements should be copied</param>
            /// <param name="arrayIndex">The starting index</param>
            public override void CopyTo(IModelElement[] array, int arrayIndex)
            {
                IEnumerator<IModelElement> detailsEnumerator = this._parent.Details.GetEnumerator();
                try
                {
                    for (
                    ; detailsEnumerator.MoveNext();
                    )
                    {
                        array[arrayIndex] = detailsEnumerator.Current;
                        arrayIndex = (arrayIndex + 1);
                    }
                }
                finally
                {
                    detailsEnumerator.Dispose();
                }
                if ((this._parent.EModelElement != null))
                {
                    array[arrayIndex] = this._parent.EModelElement;
                    arrayIndex = (arrayIndex + 1);
                }
                this._parent.Contents.CopyTo(array, arrayIndex);
                arrayIndex = (arrayIndex + this._parent.Contents.Count);
                this._parent.References.CopyTo(array, arrayIndex);
                arrayIndex = (arrayIndex + this._parent.References.Count);
            }

            /// <summary>
            /// Removes the given item from the collection
            /// </summary>
            /// <returns>True, if the item was removed, otherwise False</returns>
            /// <param name="item">The item that should be removed</param>
            public override bool Remove(IModelElement item)
            {
                IEStringToStringMapEntry eStringToStringMapEntryItem = item.As<IEStringToStringMapEntry>();
                if (((eStringToStringMapEntryItem != null)
                            && this._parent.Details.Remove(eStringToStringMapEntryItem)))
                {
                    return true;
                }
                if ((this._parent.EModelElement == item))
                {
                    this._parent.EModelElement = null;
                    return true;
                }
                if (this._parent.Contents.Remove(item))
                {
                    return true;
                }
                if (this._parent.References.Remove(item))
                {
                    return true;
                }
                return false;
            }

            /// <summary>
            /// Gets an enumerator that enumerates the collection
            /// </summary>
            /// <returns>A generic enumerator</returns>
            public override IEnumerator<IModelElement> GetEnumerator()
            {
                return Enumerable.Empty<IModelElement>().Concat(this._parent.Details).Concat(this._parent.EModelElement).Concat(this._parent.Contents).Concat(this._parent.References).GetEnumerator();
            }
        }

        /// <summary>
        /// Represents a proxy to represent an incremental access to the source property
        /// </summary>
        private sealed class SourceProxy : ModelPropertyChange<IEAnnotation, string>
        {

            /// <summary>
            /// Creates a new observable property access proxy
            /// </summary>
            /// <param name="modelElement">The model instance element for which to create the property access proxy</param>
            public SourceProxy(IEAnnotation modelElement) :
                    base(modelElement, "source")
            {
            }

            /// <summary>
            /// Gets or sets the value of this expression
            /// </summary>
            public override string Value
            {
                get
                {
                    return this.ModelElement.Source;
                }
                set
                {
                    this.ModelElement.Source = value;
                }
            }
        }

        /// <summary>
        /// Represents a proxy to represent an incremental access to the eModelElement property
        /// </summary>
        private sealed class EModelElementProxy : ModelPropertyChange<IEAnnotation, IEModelElement>
        {

            /// <summary>
            /// Creates a new observable property access proxy
            /// </summary>
            /// <param name="modelElement">The model instance element for which to create the property access proxy</param>
            public EModelElementProxy(IEAnnotation modelElement) :
                    base(modelElement, "eModelElement")
            {
            }

            /// <summary>
            /// Gets or sets the value of this expression
            /// </summary>
            public override IEModelElement Value
            {
                get
                {
                    return this.ModelElement.EModelElement;
                }
                set
                {
                    this.ModelElement.EModelElement = value;
                }
            }
        }
    }
}


text:
require 'aws-sdk-core'
require 'aws-sigv4'

require_relative 'aws-sdk-timestreamquery/types'
require_relative 'aws-sdk-timestreamquery/client_api'
require_relative 'aws-sdk-timestreamquery/plugins/endpoints.rb'
require_relative 'aws-sdk-timestreamquery/client'
require_relative 'aws-sdk-timestreamquery/errors'
require_relative 'aws-sdk-timestreamquery/resource'
require_relative 'aws-sdk-timestreamquery/endpoint_parameters'
require_relative 'aws-sdk-timestreamquery/endpoint_provider'
require_relative 'aws-sdk-timestreamquery/endpoints'
require_relative 'aws-sdk-timestreamquery/customizations'

# This module provides support for Amazon Timestream Query. This module is available in the
# `aws-sdk-timestreamquery` gem.
#
# # Client
#
# The {Client} class provides one method for each API operation. Operation
# methods each accept a hash of request parameters and return a response
# structure.
#
#     timestream_query = Aws::TimestreamQuery::Client.new
#     resp = timestream_query.cancel_query(params)
#
# See {Client} for more information.
#
# # Errors
#
# Errors returned from Amazon Timestream Query are defined in the
# {Errors} module and all extend {Errors::ServiceError}.
#
#     begin
#       # do stuff
#     rescue Aws::TimestreamQuery::Errors::ServiceError
#       # rescues all Amazon Timestream Query API errors
#     end
#
# See {Errors} for more information.
#
# @!group service
module Aws::TimestreamQuery

  GEM_VERSION = '1.17.0'

end