text:
<!DOCTYPE html>

  <title>Semantic Checkboxes</title>
  <link rel="stylesheet" type="text/css" href="../../semantic/dist/semantic.min.css">

Checkboxes

  <input type="checkbox" name="example">
  Make my profile visible

  <input type="radio" name="radio" checked="checked">
  Radio choice

    How often do you use checkboxes?

        <input type="radio" name="frequency" checked="checked">
        Once a week

        <input type="radio" name="frequency">
        2-3 times a week

        <input type="radio" name="frequency">
        Once a day

        <input type="radio" name="frequency">
        Twice a day

    How often do you use checkboxes?

        <input type="radio" name="example2" checked="checked">
        Once a week

        <input type="radio" name="example2">
        2-3 times a week

        <input type="radio" name="example2">
        Once a day

        <input type="radio" name="example2">
        Twice a day

  <input type="checkbox" name="newsletter">
  Accept terms and conditions

  <input type="checkbox" name="public">
  Subscribe to weekly newsletter

  <input type="checkbox" checked="">
  Active

redpajama_set_name: RedPajamaGithub


text:
QUnit.test('options#ready', function (assert) {
  var done = assert.async();
  var util = window.Util;
  var image = util.createImage();

  assert.expect(1);

  return new Viewer(image, {
    inline: true,

    ready: function (e) {
      assert.ok(e.type === 'ready');

      done();
    }
  });
});


text:
package org.apache.spark.sql.execution.datasources.parquet

import java.math.{BigDecimal, BigInteger}
import java.nio.ByteOrder
import java.time.{ZoneId, ZoneOffset}

import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer

import org.apache.parquet.column.Dictionary
import org.apache.parquet.io.api.{Binary, Converter, GroupConverter, PrimitiveConverter}
import org.apache.parquet.schema.{GroupType, MessageType, OriginalType, Type}
import org.apache.parquet.schema.OriginalType.{INT_32, LIST, UTF8}
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.{BINARY, DOUBLE, FIXED_LEN_BYTE_ARRAY, INT32, INT64, INT96}

import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CaseInsensitiveMap, DateTimeUtils, GenericArrayData}
import org.apache.spark.sql.catalyst.util.DateTimeUtils.SQLTimestamp
import org.apache.spark.sql.catalyst.util.RebaseDateTime._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String

/**
 * A [[ParentContainerUpdater]] is used by a Parquet converter to set converted values to some
 * corresponding parent container. For example, a converter for a `StructType` field may set
 * converted values to a [[InternalRow]]; or a converter for array elements may append converted
 * values to an [[ArrayBuffer]].
 */
private[parquet] trait ParentContainerUpdater {
  /** Called before a record field is being converted */
  def start(): Unit = ()

  /** Called after a record field is being converted */
  def end(): Unit = ()

  def set(value: Any): Unit = ()
  def setBoolean(value: Boolean): Unit = set(value)
  def setByte(value: Byte): Unit = set(value)
  def setShort(value: Short): Unit = set(value)
  def setInt(value: Int): Unit = set(value)
  def setLong(value: Long): Unit = set(value)
  def setFloat(value: Float): Unit = set(value)
  def setDouble(value: Double): Unit = set(value)
}

/** A no-op updater used for root converter (who doesn't have a parent). */
private[parquet] object NoopUpdater extends ParentContainerUpdater

private[parquet] trait HasParentContainerUpdater {
  def updater: ParentContainerUpdater
}

/**
 * A convenient converter class for Parquet group types with a [[HasParentContainerUpdater]].
 */
private[parquet] abstract class ParquetGroupConverter(val updater: ParentContainerUpdater)
  extends GroupConverter with HasParentContainerUpdater

/**
 * Parquet converter for Parquet primitive types.  Note that not all Spark SQL atomic types
 * are handled by this converter.  Parquet primitive types are only a subset of those of Spark
 * SQL.  For example, BYTE, SHORT, and INT in Spark SQL are all covered by INT32 in Parquet.
 */
private[parquet] class ParquetPrimitiveConverter(val updater: ParentContainerUpdater)
  extends PrimitiveConverter with HasParentContainerUpdater {

  override def addBoolean(value: Boolean): Unit = updater.setBoolean(value)
  override def addInt(value: Int): Unit = updater.setInt(value)
  override def addLong(value: Long): Unit = updater.setLong(value)
  override def addFloat(value: Float): Unit = updater.setFloat(value)
  override def addDouble(value: Double): Unit = updater.setDouble(value)
  override def addBinary(value: Binary): Unit = updater.set(value.getBytes)
}

/**
 * A [[ParquetRowConverter]] is used to convert Parquet records into Catalyst [[InternalRow]]s.
 * Since Catalyst `StructType` is also a Parquet record, this converter can be used as root
 * converter.  Take the following Parquet type as an example:
 * {{{
 *   message root {
 *     required int32 f1;
 *     optional group f2 {
 *       required double f21;
 *       optional binary f22 (utf8);
 *     }
 *   }
 * }}}
 * 5 converters will be created:
 *
 * - a root [[ParquetRowConverter]] for [[MessageType]] `root`, which contains:
 *   - a [[ParquetPrimitiveConverter]] for required [[INT_32]] field `f1`, and
 *   - a nested [[ParquetRowConverter]] for optional [[GroupType]] `f2`, which contains:
 *     - a [[ParquetPrimitiveConverter]] for required [[DOUBLE]] field `f21`, and
 *     - a [[ParquetStringConverter]] for optional [[UTF8]] string field `f22`
 *
 * When used as a root converter, [[NoopUpdater]] should be used since root converters don't have
 * any "parent" container.
 *
 * @param schemaConverter A utility converter used to convert Parquet types to Catalyst types.
 * @param parquetType Parquet schema of Parquet records
 * @param catalystType Spark SQL schema that corresponds to the Parquet record type. User-defined
 *        types should have been expanded.
 * @param convertTz the optional time zone to convert to int96 data
 * @param rebaseDateTime true if need to rebase date/timestamp from Julian to Proleptic Gregorian
 *                       calendar
 * @param updater An updater which propagates converted field values to the parent container
 */
private[parquet] class ParquetRowConverter(
    schemaConverter: ParquetToSparkSchemaConverter,
    parquetType: GroupType,
    catalystType: StructType,
    convertTz: Option[ZoneId],
    rebaseDateTime: Boolean,
    updater: ParentContainerUpdater)
  extends ParquetGroupConverter(updater) with Logging {

  assert(
    parquetType.getFieldCount <= catalystType.length,
    s"""Field count of the Parquet schema is greater than the field count of the Catalyst schema:
       |
       |Parquet schema:
       |$parquetType
       |Catalyst schema:
       |${catalystType.prettyJson}
     """.stripMargin)

  assert(
    !catalystType.existsRecursively(_.isInstanceOf[UserDefinedType[_]]),
    s"""User-defined types in Catalyst schema should have already been expanded:
       |${catalystType.prettyJson}
     """.stripMargin)

  logDebug(
    s"""Building row converter for the following schema:
       |
       |Parquet form:
       |$parquetType
       |Catalyst form:
       |${catalystType.prettyJson}
     """.stripMargin)

  /**
   * Updater used together with field converters within a [[ParquetRowConverter]].  It propagates
   * converted filed values to the `ordinal`-th cell in `currentRow`.
   */
  private final class RowUpdater(row: InternalRow, ordinal: Int) extends ParentContainerUpdater {
    override def set(value: Any): Unit = row(ordinal) = value
    override def setBoolean(value: Boolean): Unit = row.setBoolean(ordinal, value)
    override def setByte(value: Byte): Unit = row.setByte(ordinal, value)
    override def setShort(value: Short): Unit = row.setShort(ordinal, value)
    override def setInt(value: Int): Unit = row.setInt(ordinal, value)
    override def setLong(value: Long): Unit = row.setLong(ordinal, value)
    override def setDouble(value: Double): Unit = row.setDouble(ordinal, value)
    override def setFloat(value: Float): Unit = row.setFloat(ordinal, value)
  }

  private[this] val currentRow = new SpecificInternalRow(catalystType.map(_.dataType))

  /**
   * The [[InternalRow]] converted from an entire Parquet record.
   */
  def currentRecord: InternalRow = currentRow

  // Converters for each field.
  private[this] val fieldConverters: Array[Converter with HasParentContainerUpdater] = {
    // (SPARK-31116) Use case insensitive map if spark.sql.caseSensitive is false
    // to prevent throwing IllegalArgumentException when searching catalyst type's field index
    val catalystFieldNameToIndex = if (SQLConf.get.caseSensitiveAnalysis) {
      catalystType.fieldNames.zipWithIndex.toMap
    } else {
      CaseInsensitiveMap(catalystType.fieldNames.zipWithIndex.toMap)
    }
    parquetType.getFields.asScala.map { parquetField =>
      val fieldIndex = catalystFieldNameToIndex(parquetField.getName)
      val catalystField = catalystType(fieldIndex)
      // Converted field value should be set to the `fieldIndex`-th cell of `currentRow`
      newConverter(parquetField, catalystField.dataType, new RowUpdater(currentRow, fieldIndex))
    }.toArray
  }

  // Updaters for each field.
  private[this] val fieldUpdaters: Array[ParentContainerUpdater] = fieldConverters.map(_.updater)

  override def getConverter(fieldIndex: Int): Converter = fieldConverters(fieldIndex)

  override def end(): Unit = {
    var i = 0
    while (i < fieldUpdaters.length) {
      fieldUpdaters(i).end()
      i += 1
    }
    updater.set(currentRow)
  }

  override def start(): Unit = {
    var i = 0
    val numFields = currentRow.numFields
    while (i < numFields) {
      currentRow.setNullAt(i)
      i += 1
    }
    i = 0
    while (i < fieldUpdaters.length) {
      fieldUpdaters(i).start()
      i += 1
    }
  }

  /**
   * Creates a converter for the given Parquet type `parquetType` and Spark SQL data type
   * `catalystType`. Converted values are handled by `updater`.
   */
  private def newConverter(
      parquetType: Type,
      catalystType: DataType,
      updater: ParentContainerUpdater): Converter with HasParentContainerUpdater = {

    catalystType match {
      case BooleanType | IntegerType | LongType | FloatType | DoubleType | BinaryType =>
        new ParquetPrimitiveConverter(updater)

      case ByteType =>
        new ParquetPrimitiveConverter(updater) {
          override def addInt(value: Int): Unit =
            updater.setByte(value.asInstanceOf[ByteType#InternalType])
        }

      case ShortType =>
        new ParquetPrimitiveConverter(updater) {
          override def addInt(value: Int): Unit =
            updater.setShort(value.asInstanceOf[ShortType#InternalType])
        }

      // For INT32 backed decimals
      case t: DecimalType if parquetType.asPrimitiveType().getPrimitiveTypeName == INT32 =>
        new ParquetIntDictionaryAwareDecimalConverter(t.precision, t.scale, updater)

      // For INT64 backed decimals
      case t: DecimalType if parquetType.asPrimitiveType().getPrimitiveTypeName == INT64 =>
        new ParquetLongDictionaryAwareDecimalConverter(t.precision, t.scale, updater)

      // For BINARY and FIXED_LEN_BYTE_ARRAY backed decimals
      case t: DecimalType
        if parquetType.asPrimitiveType().getPrimitiveTypeName == FIXED_LEN_BYTE_ARRAY ||
           parquetType.asPrimitiveType().getPrimitiveTypeName == BINARY =>
        new ParquetBinaryDictionaryAwareDecimalConverter(t.precision, t.scale, updater)

      case t: DecimalType =>
        throw new RuntimeException(
          s"Unable to create Parquet converter for decimal type ${t.json} whose Parquet type is " +
            s"$parquetType.  Parquet DECIMAL type can only be backed by INT32, INT64, " +
            "FIXED_LEN_BYTE_ARRAY, or BINARY.")

      case StringType =>
        new ParquetStringConverter(updater)

      case TimestampType if parquetType.getOriginalType == OriginalType.TIMESTAMP_MICROS =>
        if (rebaseDateTime) {
          new ParquetPrimitiveConverter(updater) {
            override def addLong(value: Long): Unit = {
              val rebased = rebaseJulianToGregorianMicros(value)
              updater.setLong(rebased)
            }
          }
        } else {
          new ParquetPrimitiveConverter(updater) {
            override def addLong(value: Long): Unit = {
              updater.setLong(value)
            }
          }
        }

      case TimestampType if parquetType.getOriginalType == OriginalType.TIMESTAMP_MILLIS =>
        if (rebaseDateTime) {
          new ParquetPrimitiveConverter(updater) {
            override def addLong(value: Long): Unit = {
              val micros = DateTimeUtils.millisToMicros(value)
              val rebased = rebaseJulianToGregorianMicros(micros)
              updater.setLong(rebased)
            }
          }
        } else {
          new ParquetPrimitiveConverter(updater) {
            override def addLong(value: Long): Unit = {
              updater.setLong(DateTimeUtils.millisToMicros(value))
            }
          }
        }

      // INT96 timestamp doesn't have a logical type, here we check the physical type instead.
      case TimestampType if parquetType.asPrimitiveType().getPrimitiveTypeName == INT96 =>
        new ParquetPrimitiveConverter(updater) {
          // Converts nanosecond timestamps stored as INT96
          override def addBinary(value: Binary): Unit = {
            assert(
              value.length() == 12,
              "Timestamps (with nanoseconds) are expected to be stored in 12-byte long binaries, " +
              s"but got a ${value.length()}-byte binary.")

            val buf = value.toByteBuffer.order(ByteOrder.LITTLE_ENDIAN)
            val timeOfDayNanos = buf.getLong
            val julianDay = buf.getInt
            val rawTime = DateTimeUtils.fromJulianDay(julianDay, timeOfDayNanos)
            val adjTime = convertTz.map(DateTimeUtils.convertTz(rawTime, _, ZoneOffset.UTC))
              .getOrElse(rawTime)
            updater.setLong(adjTime)
          }
        }

      case DateType =>
        if (rebaseDateTime) {
          new ParquetPrimitiveConverter(updater) {
            override def addInt(value: Int): Unit = {
              updater.set(rebaseJulianToGregorianDays(value))
            }
          }
        } else {
          new ParquetPrimitiveConverter(updater) {
            override def addInt(value: Int): Unit = {
              updater.set(value)
            }
          }
        }

      // A repeated field that is neither contained by a `LIST`- or `MAP`-annotated group nor
      // annotated by `LIST` or `MAP` should be interpreted as a required list of required
      // elements where the element type is the type of the field.
      case t: ArrayType if parquetType.getOriginalType != LIST =>
        if (parquetType.isPrimitive) {
          new RepeatedPrimitiveConverter(parquetType, t.elementType, updater)
        } else {
          new RepeatedGroupConverter(parquetType, t.elementType, updater)
        }

      case t: ArrayType =>
        new ParquetArrayConverter(parquetType.asGroupType(), t, updater)

      case t: MapType =>
        new ParquetMapConverter(parquetType.asGroupType(), t, updater)

      case t: StructType =>
        val wrappedUpdater = {
          // SPARK-30338: avoid unnecessary InternalRow copying for nested structs:
          // There are two cases to handle here:
          //
          //  1. Parent container is a map or array: we must make a deep copy of the mutable row
          //     because this converter may be invoked multiple times per Parquet input record
          //     (if the map or array contains multiple elements).
          //
          //  2. Parent container is a struct: we don't need to copy the row here because either:
          //
          //     (a) all ancestors are structs and therefore no copying is required because this
          //         converter will only be invoked once per Parquet input record, or
          //     (b) some ancestor is struct that is nested in a map or array and that ancestor's
          //         converter will perform deep-copying (which will recursively copy this row).
          if (updater.isInstanceOf[RowUpdater]) {
            // `updater` is a RowUpdater, implying that the parent container is a struct.
            updater
          } else {
            // `updater` is NOT a RowUpdater, implying that the parent container a map or array.
            new ParentContainerUpdater {
              override def set(value: Any): Unit = {
                updater.set(value.asInstanceOf[SpecificInternalRow].copy())  // deep copy
              }
            }
          }
        }
        new ParquetRowConverter(
          schemaConverter, parquetType.asGroupType(), t, convertTz, rebaseDateTime, wrappedUpdater)

      case t =>
        throw new RuntimeException(
          s"Unable to create Parquet converter for data type ${t.json} " +
            s"whose Parquet type is $parquetType")
    }
  }

  /**
   * Parquet converter for strings. A dictionary is used to minimize string decoding cost.
   */
  private final class ParquetStringConverter(updater: ParentContainerUpdater)
    extends ParquetPrimitiveConverter(updater) {

    private var expandedDictionary: Array[UTF8String] = null

    override def hasDictionarySupport: Boolean = true

    override def setDictionary(dictionary: Dictionary): Unit = {
      this.expandedDictionary = Array.tabulate(dictionary.getMaxId + 1) { i =>
        UTF8String.fromBytes(dictionary.decodeToBinary(i).getBytes)
      }
    }

    override def addValueFromDictionary(dictionaryId: Int): Unit = {
      updater.set(expandedDictionary(dictionaryId))
    }

    override def addBinary(value: Binary): Unit = {
      // The underlying `ByteBuffer` implementation is guaranteed to be `HeapByteBuffer`, so here we
      // are using `Binary.toByteBuffer.array()` to steal the underlying byte array without copying
      // it.
      val buffer = value.toByteBuffer
      val offset = buffer.arrayOffset() + buffer.position()
      val numBytes = buffer.remaining()
      updater.set(UTF8String.fromBytes(buffer.array(), offset, numBytes))
    }
  }

  /**
   * Parquet converter for fixed-precision decimals.
   */
  private abstract class ParquetDecimalConverter(
      precision: Int, scale: Int, updater: ParentContainerUpdater)
    extends ParquetPrimitiveConverter(updater) {

    protected var expandedDictionary: Array[Decimal] = _

    override def hasDictionarySupport: Boolean = true

    override def addValueFromDictionary(dictionaryId: Int): Unit = {
      updater.set(expandedDictionary(dictionaryId))
    }

    // Converts decimals stored as INT32
    override def addInt(value: Int): Unit = {
      addLong(value: Long)
    }

    // Converts decimals stored as INT64
    override def addLong(value: Long): Unit = {
      updater.set(decimalFromLong(value))
    }

    // Converts decimals stored as either FIXED_LENGTH_BYTE_ARRAY or BINARY
    override def addBinary(value: Binary): Unit = {
      updater.set(decimalFromBinary(value))
    }

    protected def decimalFromLong(value: Long): Decimal = {
      Decimal(value, precision, scale)
    }

    protected def decimalFromBinary(value: Binary): Decimal = {
      if (precision <= Decimal.MAX_LONG_DIGITS) {
        // Constructs a `Decimal` with an unscaled `Long` value if possible.
        val unscaled = ParquetRowConverter.binaryToUnscaledLong(value)
        Decimal(unscaled, precision, scale)
      } else {
        // Otherwise, resorts to an unscaled `BigInteger` instead.
        Decimal(new BigDecimal(new BigInteger(value.getBytes), scale), precision, scale)
      }
    }
  }

  private class ParquetIntDictionaryAwareDecimalConverter(
      precision: Int, scale: Int, updater: ParentContainerUpdater)
    extends ParquetDecimalConverter(precision, scale, updater) {

    override def setDictionary(dictionary: Dictionary): Unit = {
      this.expandedDictionary = Array.tabulate(dictionary.getMaxId + 1) { id =>
        decimalFromLong(dictionary.decodeToInt(id).toLong)
      }
    }
  }

  private class ParquetLongDictionaryAwareDecimalConverter(
      precision: Int, scale: Int, updater: ParentContainerUpdater)
    extends ParquetDecimalConverter(precision, scale, updater) {

    override def setDictionary(dictionary: Dictionary): Unit = {
      this.expandedDictionary = Array.tabulate(dictionary.getMaxId + 1) { id =>
        decimalFromLong(dictionary.decodeToLong(id))
      }
    }
  }

  private class ParquetBinaryDictionaryAwareDecimalConverter(
      precision: Int, scale: Int, updater: ParentContainerUpdater)
    extends ParquetDecimalConverter(precision, scale, updater) {

    override def setDictionary(dictionary: Dictionary): Unit = {
      this.expandedDictionary = Array.tabulate(dictionary.getMaxId + 1) { id =>
        decimalFromBinary(dictionary.decodeToBinary(id))
      }
    }
  }

  /**
   * Parquet converter for arrays.  Spark SQL arrays are represented as Parquet lists.  Standard
   * Parquet lists are represented as a 3-level group annotated by `LIST`:
   * {{{
   *   <list-repetition> group <name> (LIST) {            <-- parquetSchema points here
   *     repeated group list {
   *       <element-repetition> <element-type> element;
   *     }
   *   }
   * }}}
   * The `parquetSchema` constructor argument points to the outermost group.
   *
   * However, before this representation is standardized, some Parquet libraries/tools also use some
   * non-standard formats to represent list-like structures.  Backwards-compatibility rules for
   * handling these cases are described in Parquet format spec.
   *
   * @see https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists
   */
  private final class ParquetArrayConverter(
      parquetSchema: GroupType,
      catalystSchema: ArrayType,
      updater: ParentContainerUpdater)
    extends ParquetGroupConverter(updater) {

    private[this] val currentArray = ArrayBuffer.empty[Any]

    private[this] val elementConverter: Converter = {
      val repeatedType = parquetSchema.getType(0)
      val elementType = catalystSchema.elementType

      // At this stage, we're not sure whether the repeated field maps to the element type or is
      // just the syntactic repeated group of the 3-level standard LIST layout. Take the following
      // Parquet LIST-annotated group type as an example:
      //
      //    optional group f (LIST) {
      //      repeated group list {
      //        optional group element {
      //          optional int32 element;
      //        }
      //      }
      //    }
      //
      // This type is ambiguous:
      //
      // 1. When interpreted as a standard 3-level layout, the `list` field is just the syntactic
      //    group, and the entire type should be translated to:
      //
      //      ARRAY<STRUCT<element: INT>>
      //
      // 2. On the other hand, when interpreted as a non-standard 2-level layout, the `list` field
      //    represents the element type, and the entire type should be translated to:
      //
      //      ARRAY<STRUCT<element: STRUCT<element: INT>>>
      //
      // Here we try to convert field `list` into a Catalyst type to see whether the converted type
      // matches the Catalyst array element type. If it doesn't match, then it's case 1; otherwise,
      // it's case 2.
      val guessedElementType = schemaConverter.convertField(repeatedType)

      if (DataType.equalsIgnoreCompatibleNullability(guessedElementType, elementType)) {
        // If the repeated field corresponds to the element type, creates a new converter using the
        // type of the repeated field.
        newConverter(repeatedType, elementType, new ParentContainerUpdater {
          override def set(value: Any): Unit = currentArray += value
        })
      } else {
        // If the repeated field corresponds to the syntactic group in the standard 3-level Parquet
        // LIST layout, creates a new converter using the only child field of the repeated field.
        assert(!repeatedType.isPrimitive && repeatedType.asGroupType().getFieldCount == 1)
        new ElementConverter(repeatedType.asGroupType().getType(0), elementType)
      }
    }

    override def getConverter(fieldIndex: Int): Converter = elementConverter

    override def end(): Unit = updater.set(new GenericArrayData(currentArray.toArray))

    override def start(): Unit = currentArray.clear()

    /** Array element converter */
    private final class ElementConverter(parquetType: Type, catalystType: DataType)
      extends GroupConverter {

      private var currentElement: Any = _

      private[this] val converter =
        newConverter(parquetType, catalystType, new ParentContainerUpdater {
          override def set(value: Any): Unit = currentElement = value
        })

      override def getConverter(fieldIndex: Int): Converter = converter

      override def end(): Unit = currentArray += currentElement

      override def start(): Unit = currentElement = null
    }
  }

  /** Parquet converter for maps */
  private final class ParquetMapConverter(
      parquetType: GroupType,
      catalystType: MapType,
      updater: ParentContainerUpdater)
    extends ParquetGroupConverter(updater) {

    private[this] val currentKeys = ArrayBuffer.empty[Any]
    private[this] val currentValues = ArrayBuffer.empty[Any]

    private[this] val keyValueConverter = {
      val repeatedType = parquetType.getType(0).asGroupType()
      new KeyValueConverter(
        repeatedType.getType(0),
        repeatedType.getType(1),
        catalystType.keyType,
        catalystType.valueType)
    }

    override def getConverter(fieldIndex: Int): Converter = keyValueConverter

    override def end(): Unit = {
      // The parquet map may contains null or duplicated map keys. When it happens, the behavior is
      // undefined.
      // TODO (SPARK-26174): disallow it with a config.
      updater.set(
        new ArrayBasedMapData(
          new GenericArrayData(currentKeys.toArray),
          new GenericArrayData(currentValues.toArray)))
    }

    override def start(): Unit = {
      currentKeys.clear()
      currentValues.clear()
    }

    /** Parquet converter for key-value pairs within the map. */
    private final class KeyValueConverter(
        parquetKeyType: Type,
        parquetValueType: Type,
        catalystKeyType: DataType,
        catalystValueType: DataType)
      extends GroupConverter {

      private var currentKey: Any = _

      private var currentValue: Any = _

      private[this] val converters = Array(
        // Converter for keys
        newConverter(parquetKeyType, catalystKeyType, new ParentContainerUpdater {
          override def set(value: Any): Unit = currentKey = value
        }),

        // Converter for values
        newConverter(parquetValueType, catalystValueType, new ParentContainerUpdater {
          override def set(value: Any): Unit = currentValue = value
        }))

      override def getConverter(fieldIndex: Int): Converter = converters(fieldIndex)

      override def end(): Unit = {
        currentKeys += currentKey
        currentValues += currentValue
      }

      override def start(): Unit = {
        currentKey = null
        currentValue = null
      }
    }
  }

  private trait RepeatedConverter {
    private[this] val currentArray = ArrayBuffer.empty[Any]

    protected def newArrayUpdater(updater: ParentContainerUpdater) = new ParentContainerUpdater {
      override def start(): Unit = currentArray.clear()
      override def end(): Unit = updater.set(new GenericArrayData(currentArray.toArray))
      override def set(value: Any): Unit = currentArray += value
    }
  }

  /**
   * A primitive converter for converting unannotated repeated primitive values to required arrays
   * of required primitives values.
   */
  private final class RepeatedPrimitiveConverter(
      parquetType: Type,
      catalystType: DataType,
      parentUpdater: ParentContainerUpdater)
    extends PrimitiveConverter with RepeatedConverter with HasParentContainerUpdater {

    val updater: ParentContainerUpdater = newArrayUpdater(parentUpdater)

    private[this] val elementConverter: PrimitiveConverter =
      newConverter(parquetType, catalystType, updater).asPrimitiveConverter()

    override def addBoolean(value: Boolean): Unit = elementConverter.addBoolean(value)
    override def addInt(value: Int): Unit = elementConverter.addInt(value)
    override def addLong(value: Long): Unit = elementConverter.addLong(value)
    override def addFloat(value: Float): Unit = elementConverter.addFloat(value)
    override def addDouble(value: Double): Unit = elementConverter.addDouble(value)
    override def addBinary(value: Binary): Unit = elementConverter.addBinary(value)

    override def setDictionary(dict: Dictionary): Unit = elementConverter.setDictionary(dict)
    override def hasDictionarySupport: Boolean = elementConverter.hasDictionarySupport
    override def addValueFromDictionary(id: Int): Unit = elementConverter.addValueFromDictionary(id)
  }

  /**
   * A group converter for converting unannotated repeated group values to required arrays of
   * required struct values.
   */
  private final class RepeatedGroupConverter(
      parquetType: Type,
      catalystType: DataType,
      parentUpdater: ParentContainerUpdater)
    extends GroupConverter with HasParentContainerUpdater with RepeatedConverter {

    val updater: ParentContainerUpdater = newArrayUpdater(parentUpdater)

    private[this] val elementConverter: GroupConverter =
      newConverter(parquetType, catalystType, updater).asGroupConverter()

    override def getConverter(field: Int): Converter = elementConverter.getConverter(field)
    override def end(): Unit = elementConverter.end()
    override def start(): Unit = elementConverter.start()
  }
}

private[parquet] object ParquetRowConverter {
  def binaryToUnscaledLong(binary: Binary): Long = {
    // The underlying `ByteBuffer` implementation is guaranteed to be `HeapByteBuffer`, so here
    // we are using `Binary.toByteBuffer.array()` to steal the underlying byte array without
    // copying it.
    val buffer = binary.toByteBuffer
    val bytes = buffer.array()
    val start = buffer.arrayOffset() + buffer.position()
    val end = buffer.arrayOffset() + buffer.limit()

    var unscaled = 0L
    var i = start

    while (i < end) {
      unscaled = (unscaled << 8) | (bytes(i) & 0xff)
      i += 1
    }

    val bits = 8 * (end - start)
    unscaled = (unscaled << (64 - bits)) >> (64 - bits)
    unscaled
  }

  def binaryToSQLTimestamp(binary: Binary): SQLTimestamp = {
    assert(binary.length() == 12, s"Timestamps (with nanoseconds) are expected to be stored in" +
      s" 12-byte long binaries. Found a ${binary.length()}-byte binary instead.")
    val buffer = binary.toByteBuffer.order(ByteOrder.LITTLE_ENDIAN)
    val timeOfDayNanos = buffer.getLong
    val julianDay = buffer.getInt
    DateTimeUtils.fromJulianDay(julianDay, timeOfDayNanos)
  }
}


text:
import random
from PIL import Image, ImageDraw
import math

from inyourface.Animator import Animator

class EffectAnimator(Animator):

    name = "crying"

    def cry_frame(self, frame_image, faces, index):
        # Instantiates a client
        tear = Image.open(self.__class__.get_os_path('overlays/tear.png'))

        lowest = 0

        for face in faces:

            for side in ('left', 'right'):

                ((lcx, lcy), (ex, ey), (rcx, rcy)) = face.get_eye_coords(side)

                ew = int(1.25 * math.hypot(rcx - lcx, rcy - lcy))

                pasted = tear.resize((ew, ew), Image.BICUBIC).rotate(face.tilt_angle, Image.BICUBIC)
                left_y = int(lcy + (index * ew * 1.5) + (ew * .5))
                right_y = int(rcy + (index * ew * 1.5) + (ew * .75) )
                frame_image.paste(pasted, (int(lcx - ew/2), left_y), pasted)
                frame_image.paste(pasted, (int(rcx - ew/2), right_y), pasted)
                lowest = max(left_y, right_y)

        return lowest

    def bottom_fill(self, frame_image, index):

        draw = ImageDraw.Draw(frame_image)
        increment = int(frame_image.height / 20)
        draw.rectangle( [(0,frame_image.height - index*increment), (frame_image.width,frame_image.height)], fill="#69cbfe" )

    def manipulate_frame(self, frame_image, faces, index):
        for i in range(0, index):
            self.cry_frame(frame_image, faces, i)
        return frame_image


text:
using MvcMusicStore.Data.Repository.EntityFramework.Common;
using MvcMusicStore.Domain.Entities;
using MvcMusicStore.Domain.Interfaces.Repository;

namespace MvcMusicStore.Data.Repository.EntityFramework
{
    public class OrderDetailRepository : Repository<OrderDetail>, IOrderDetailRepository
    {
    }
}


text:
package org.encog.ml.factory;

import org.encog.Encog;
import org.encog.EncogError;
import org.encog.engine.network.activation.ActivationFunction;
import org.encog.plugin.EncogPluginBase;
import org.encog.plugin.EncogPluginService1;

public class MLActivationFactory {

	public static final String AF_BIPOLAR = "bipolar";
	public static final String AF_COMPETITIVE = "comp";
	public static final String AF_GAUSSIAN = "gauss";
	public static final String AF_LINEAR = "linear";
	public static final String AF_LOG = "log";
	public static final String AF_RAMP = "ramp";
	public static final String AF_SIGMOID = "sigmoid";
	public static final String AF_SIN = "sin";
	public static final String AF_SOFTMAX = "softmax";
	public static final String AF_STEP = "step";
	public static final String AF_TANH = "tanh";

	public ActivationFunction create(String fn) {

		for (EncogPluginBase plugin : Encog.getInstance().getPlugins()) {
			if (plugin instanceof EncogPluginService1) {
				ActivationFunction result = ((EncogPluginService1) plugin).createActivationFunction(fn);
				if (result != null) {
					return result;
				}
			}
		}
		return null;
	}
}


text:
module Hachioji::Taikikanshi
  class Ch4Client < BaseClient

    endpoint_path "/ch4hour.htm"
    measured_value_type :float

  end
end


text:
NSString *const kAMPLibrary = @"amplitude-ios";
NSString *const kAMPVersion = @"4.10.0";
NSString *const kAMPEventLogDomain = @"api.amplitude.com";
NSString *const kAMPEventLogUrl = @"https://api.amplitude.com/";
NSString *const kAMPDefaultInstance = @"$default_instance";
const int kAMPApiVersion = 3;
const int kAMPDBVersion = 3;
const int kAMPDBFirstVersion = 2; // to detect if DB exists yet

// for tvOS, upload events immediately, don't save too many events locally
#if TARGET_OS_TV
const int kAMPEventUploadThreshold = 1;
const int kAMPEventMaxCount = 100;
NSString *const kAMPPlatform = @"tvOS";
NSString *const kAMPOSName = @"tvos";
#else  // macOS/iOS
const int kAMPEventUploadThreshold = 30;
const int kAMPEventMaxCount = 1000;
#if TARGET_OS_MACCATALYST
NSString *const kAMPPlatform = @"mac";
NSString *const kAMPOSName = @"mac";
#else
NSString *const kAMPPlatform = @"iOS";
NSString *const kAMPOSName = @"ios";
#endif
#endif

const int kAMPEventUploadMaxBatchSize = 100;
const int kAMPEventRemoveBatchSize = 20;
const int kAMPEventUploadPeriodSeconds = 30; // 30 seconds
const long kAMPMinTimeBetweenSessionsMillis = 5 * 60 * 1000; // 5 minutes
const int kAMPMaxStringLength = 1024;
const int kAMPMaxPropertyKeys = 1000;

NSString *const IDENTIFY_EVENT = @"$identify";
NSString *const GROUP_IDENTIFY_EVENT = @"$groupidentify";
NSString *const AMP_OP_ADD = @"$add";
NSString *const AMP_OP_APPEND = @"$append";
NSString *const AMP_OP_CLEAR_ALL = @"$clearAll";
NSString *const AMP_OP_PREPEND = @"$prepend";
NSString *const AMP_OP_SET = @"$set";
NSString *const AMP_OP_SET_ONCE = @"$setOnce";
NSString *const AMP_OP_UNSET = @"$unset";

NSString *const AMP_REVENUE_PRODUCT_ID = @"$productId";
NSString *const AMP_REVENUE_QUANTITY = @"$quantity";
NSString *const AMP_REVENUE_PRICE = @"$price";
NSString *const AMP_REVENUE_REVENUE_TYPE = @"$revenueType";
NSString *const AMP_REVENUE_RECEIPT = @"$receipt";

NSString *const AMP_TRACKING_OPTION_CARRIER = @"carrier";
NSString *const AMP_TRACKING_OPTION_CITY = @"city";
NSString *const AMP_TRACKING_OPTION_COUNTRY = @"country";
NSString *const AMP_TRACKING_OPTION_DEVICE_MANUFACTURER = @"device_manufacturer";
NSString *const AMP_TRACKING_OPTION_DEVICE_MODEL = @"device_model";
NSString *const AMP_TRACKING_OPTION_DMA = @"dma";
NSString *const AMP_TRACKING_OPTION_IDFA = @"idfa";
NSString *const AMP_TRACKING_OPTION_IDFV = @"idfv";
NSString *const AMP_TRACKING_OPTION_IP_ADDRESS = @"ip_address";
NSString *const AMP_TRACKING_OPTION_LANGUAGE = @"language";
NSString *const AMP_TRACKING_OPTION_LAT_LNG = @"lat_lng";
NSString *const AMP_TRACKING_OPTION_OS_NAME = @"os_name";
NSString *const AMP_TRACKING_OPTION_OS_VERSION = @"os_version";
NSString *const AMP_TRACKING_OPTION_PLATFORM = @"platform";
NSString *const AMP_TRACKING_OPTION_REGION = @"region";
NSString *const AMP_TRACKING_OPTION_VERSION_NAME = @"version_name";


text:
package com.siyeh.igtest.portability;

import static java.lang.Math.abs;
import static java.util.AbstractMap.*;

public class StaticImport {

    public static void main(String[] args) {
        abs(3.0);
        Entry entry;
    }
}


text:
"""
    urlresolver XBMC Addon
    Copyright (C) 2011 t0mm0

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
import re
import json
import urllib
import urllib2
from urlresolver import common
from urlresolver.common import i18n
from lib import helpers
from urlresolver.resolver import UrlResolver, ResolverError

class VidUpMeResolver(UrlResolver):
    name = "vidup.me"
    domains = ["vidup.me"]
    pattern = '(?://|\.)(vidup\.me)/(?:embed-|download/)?([0-9a-zA-Z]+)'

    def __init__(self):
        self.net = common.Net()
        self.headers = {'User-Agent': common.SMU_USER_AGENT}

    def get_media_url(self, host, media_id):
        web_url = self.get_url(host, media_id)
        headers = {
            'Referer': web_url
        }
        headers.update(self.headers)
        html = self.net.http_GET(web_url, headers=headers).content
        sources = helpers.parse_sources_list(html)
        if sources:
            if len(sources) > 1:
                try: sources.sort(key=lambda x: int(re.sub("\D", '', x[0])), reverse=True)
                except: common.logger.log_debug('Scrape sources sort failed |int(re.sub(r"""\D""", '', x[0])|')
            try:
                vt = self.__auth_ip(media_id)
                if vt:
                    params = {'direct': 'false', 'ua': 1, 'vt': vt}
                    return helpers.pick_source(sources) + '?' + urllib.urlencode(params) + helpers.append_headers(self.headers)
            except urllib2.HTTPError:
                source = helpers.pick_source(sources)
                return source
        else:
            raise ResolverError('Video Token Missing')

    def __auth_ip(self, media_id):
        header = i18n('vidup_auth_header')
        line1 = i18n('auth_required')
        line2 = i18n('visit_link')
        line3 = i18n('click_pair') % ('https://vidup.me/pair')
        with common.kodi.CountdownDialog(header, line1, line2, line3) as cd:
            return cd.start(self.__check_auth, [media_id])

    def __check_auth(self, media_id):
        common.logger.log('Checking Auth: %s' % (media_id))
        url = 'https://vidup.me/pair?file_code=%s&check' % (media_id)
        try: js_result = json.loads(self.net.http_GET(url, headers=self.headers).content)
        except ValueError:
            raise ResolverError('Unusable Authorization Response')
        except urllib2.HTTPError as e:
            if e.code == 401:
                js_result = json.loads(e.read())
            else:
                raise

        common.logger.log('Auth Result: %s' % (js_result))
        if js_result.get('status'):
            return js_result.get('response', {}).get('vt')
        else:
            return {}

    def get_url(self, host, media_id):
        return self._default_get_url(host, media_id)


text:
FROM python:3.6.6

RUN set -ex && pip install pip pipenv --upgrade

# sphinxcontrib-spelling dependency
RUN apt-get update \
  && apt-get install -yqq libenchant-dev

COPY Pipfile Pipfile
COPY Pipfile.lock Pipfile.lock
RUN set -ex && pipenv install --system --deploy --dev
ENV PYTHONPATH=/usr/local/bin:/app

COPY setup.py README.rst /app/
COPY eemeter/ /app/eemeter/
RUN set -ex && pip install -e /app
RUN set -ex && cd /usr/local/lib/ && python /app/setup.py develop

WORKDIR /app


text:
{% capture version %}/{{page.version}}/{% endcapture %}
{% capture redirectto %}{{site.baseurl}}/{{ page.path | replace: "/latest/", version | replace: "index.md", "" }}{% endcapture %}
<!DOCTYPE html>
<meta charset="utf-8">
<title>Redirecting...</title>
<link rel="canonical" href="{{redirectto}}">
<meta http-equiv="refresh" content="0; url={{redirectto}}">

Redirecting...

version: {{version}}
from (os/latest/en) {{site.baseurl}}/{{page.path}}
to                  {{redirectto}}
Click here if you are not redirected.


text:
namespace nx
{

enum DrawType
{
    DRAW_TYPE_STATIC = GL_STATIC_DRAW,
    DRAW_TYPE_DYNAMIC = GL_DYNAMIC_DRAW,
    DRAW_TYPE_STREAM = GL_STREAM_DRAW
};

} // namespace nx

#endif // DRAWTYPE_H_INCLUDE


text:
package com.twu.biblioteca;

import org.junit.Before;
import org.junit.Test;

import java.util.List;

import static junit.framework.Assert.assertNull;
import static junit.framework.Assert.assertSame;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertTrue;

public class BookLibraryTest {
    BookLibrary bookLibrary;
    Book letusc = new Book("Let Us C", "Yashwant Kanetkar", "2000");
    Book galvin = new Book("Operating System", "Galvin", "2005");
    Book internetSec = new Book("Internet Security", "Ankit Fadia", "1995");

    @Before
    public void setUp() {
        bookLibrary = new BookLibrary();
    }

    @Test
    public void testAddBookShouldReturnReferenceOfAddedBook() {
        assertSame(letusc, bookLibrary.addItem(letusc));
        assertTrue(bookLibrary.isItemPresent(letusc.getTitle()));
    }

    @Test
    public void testAddBookShouldReturnNullIfAddedBookIsAlreadyPresent() {
        bookLibrary.addItem(letusc);
        assertNull(bookLibrary.addItem(letusc));
    }

    @Test
    public void testListOfBooks() {
        bookLibrary.addItem(letusc);
        bookLibrary.addItem(galvin);
        bookLibrary.addItem(internetSec);

        List<String> bookList = bookLibrary.getListOfAvailableItems();
        assertTrue(bookList.contains(letusc.getFormattedString()));
        assertTrue(bookList.contains(galvin.getFormattedString()));
        assertTrue(bookList.contains(internetSec.getFormattedString()));
    }

    @Test
    public void testRemoveBookShouldRemoveBookFromLibrary(){
        bookLibrary.addItem(letusc);
        bookLibrary.removeItem(letusc.getTitle());
        assertFalse(bookLibrary.isItemPresent(letusc.getTitle()));
    }

    @Test
    public void testRemoveBookShouldReturnReferenceOfRemovedBook() {
        bookLibrary.addItem(letusc);
        assertSame(letusc, bookLibrary.removeItem(letusc.getTitle()));
    }

    @Test
    public void testRemoveBookShouldReturnNullIfBookIsNotPresent() {
        assertNull(bookLibrary.removeItem(letusc.getTitle()));
    }
}


text: Sina Date test and analyse


text:
int main()
{
    int res = EXIT_SUCCESS;
    ews::set_up();

    try
    {
        const auto env = ews::test::environment();
        auto service = ews::service(env.server_uri, env.domain, env.username,
                                    env.password);

        ews::distinguished_folder_id contacts_folder =
            ews::standard_folder::contacts;
        auto restriction =
            ews::is_equal_to(ews::contact_property_path::email_address_1,
                             "superhero@ducktales.com");
        auto item_ids = service.find_item(contacts_folder, restriction);

        std::cout << "Found " << item_ids.size() << " item(s)\n";

        for (const auto& id : item_ids)
        {
            auto contact = service.get_contact(id);

            std::vector<ews::update> update_list;

            auto job_title_property = ews::property(
                ews::contact_property_path::job_title, "Superhero");
            update_list.emplace_back(job_title_property);

            ews::physical_address city(ews::physical_address::key::business, "",
                                       "Metropolis", "", "", "");
            auto business_city_property = ews::property(
                ews::contact_property_path::physical_address::business::city,
                city);
            update_list.emplace_back(business_city_property);

            ews::physical_address street(ews::physical_address::key::business,
                                         "SuperStreet", "", "", "", "");
            auto business_street_property = ews::property(
                ews::contact_property_path::physical_address::business::street,
                street);
            update_list.emplace_back(business_street_property);

            auto new_id = service.update_item(id, update_list);
            (void)new_id;

// TODO: check this!
#if 0
            std::cout << "Subject: " << contact.get_subject() << std::endl;
            std::cout << "GivenName: " << contact.get_given_name() << std::endl;
            std::cout << "EmailAddress1: " << contact.get_email_address_1() << std::endl;
            std::cout << "Surname: " << contact.get_surname() << std::endl;
#endif
        }
    }
    catch (std::exception& exc)
    {
        std::cout << exc.what() << std::endl;
        res = EXIT_FAILURE;
    }

    ews::tear_down();
    return res;
}


text:
bigdata = joinpath(dirname(@__FILE__),"..", "data", "big.txt")

vector_size = rand(100:300)
window = rand(1:5)
sample = [0, 1e-5][rand(1:2)]
hs = rand(0:1)
negative = rand(1:5)
threads = rand(1:16)
iter = rand(1:5)
min_count = rand(1:5)
alpha = 0.025
debug = 2
threshold = rand(50:100)
binary = 1
cbow = rand(0:1)
classes = rand(30:100)
save_vocab = "vocab.txt"
read_vocab = "vocab.txt"

word2phrase(bigdata, "bigphrase", min_count=min_count, threshold=threshold,
            debug=debug)

word2vec("bigphrase", "bigvecs.txt", size=vector_size, window=window, sample=sample,
         hs=hs, negative=negative, threads=threads, iter=iter,
         min_count=min_count, alpha=alpha, debug=debug, cbow=cbow,
         save_vocab=save_vocab)
word2vec(bigdata, "bigvecs.bin", size=vector_size, window=window,
         sample=sample, hs=hs, negative=negative, threads=threads,
         iter=iter, binary=binary, read_vocab = read_vocab)

rm("bigphrase")

model = wordvectors("bigvecs.txt")
len_vecs, num_words = size(model)
@test len_vecs == vector_size

word2clusters(bigdata, "bigclus.txt", classes, size=vector_size, window=window,
              sample=sample, hs=hs, negative=negative, threads=threads,
              iter=iter, min_count=min_count, alpha=alpha, debug=debug,
              cbow=cbow, read_vocab=read_vocab, save_vocab=save_vocab)

println("training passed test...")


text:
"""
Program to analyse the molecular radius of all solutes in a list
The solutes coordinates are taken from the Minnesota solvation database
"""

import argparse
import os

import numpy as np

import dblib

def _calc_radii(filename):

    with open(filename,"r") as f :
        data = [s.strip().split() for s in f.readlines()[3:]]
        data = np.array(data,dtype=float)[:,1:]
        len = data.max(axis=0)-data.min(axis=0)
        return np.max(len)

if __name__ == '__main__':

    argparser = argparse.ArgumentParser(description="Script to calculate molecular radii of all solutes in a list")
    argparser.add_argument('-db', '--db', help="the molecule database")
    argparser.add_argument('-solvent', '--solvent', help="the solvent")
    argparser.add_argument('-solutes','--solutes',help="the list of solutes")
    argparser.add_argument('-xyzdir','--xyzdir',help="the directory with all xyz files")
    args = argparser.parse_args()

    db = dblib.SolvDb(args.db,type="abs",filehandle="^0")
    solutes = [s.strip() for s in open(args.solutes,'r').readlines()]

    maxrad = -1000
    maxradsol = ""
    for entry in db.itersolutelist(args.solvent,solutes):
        rad = _calc_radii(os.path.join(args.xyzdir,entry.FileHandle+".xyz"))
        print "%-30s\t%.3f"%(entry.SoluteName,rad)
        if rad > maxrad :
            maxrad = rad
            maxradsol = entry.SoluteName

    print "\nMax:\n%-30s\t%.3f"%(maxradsol,maxrad)


text:
package org.wso2.carbon.apimgt.gateway.jwt;

import com.google.gson.Gson;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.wso2.carbon.apimgt.gateway.dto.RevokedJWTTokenDTO;
import org.wso2.carbon.apimgt.gateway.internal.ServiceReferenceHolder;
import org.wso2.carbon.apimgt.impl.APIConstants;
import org.wso2.carbon.apimgt.impl.dto.EventHubConfigurationDto;
import org.wso2.carbon.apimgt.impl.utils.APIUtil;

import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Timer;
import java.util.TimerTask;

/**
 * Class which is responsible to fetch the revoked JWT signatures via webservice database during startup
 */
public class RevokedJWTTokensRetriever extends TimerTask {

    private static final Log log = LogFactory.getLog(RevokedJWTTokensRetriever.class);
    private static final int revokedJWTTokensRetrievalTimeoutInSeconds = 15;
    private static final int revokedJWTTokensRetrievalRetries = 15;

    @Override
    public void run() {

        log.debug("Starting web service based revoked JWT tokens retrieving process.");
        loadRevokedJWTTokensFromWebService();
    }

    /**
     * This method will retrieve revoked JWT tokens by calling a web service.
     *
     * @return List of RevokedJWTTokensDTOs.
     */
    private RevokedJWTTokenDTO[] retrieveRevokedJWTTokensData() {

        try {
            // The resource resides in the throttle web app. Hence reading throttle configs
            String url = getEventHubConfiguration().getServiceUrl().concat(APIConstants.INTERNAL_WEB_APP_EP).concat(
                    "/revokedjwt");
            HttpGet method = new HttpGet(url);
            byte[] credentials = Base64.encodeBase64((getEventHubConfiguration().getUsername() + ":" +
                    getEventHubConfiguration().getPassword()).getBytes(StandardCharsets.UTF_8));
            method.setHeader("Authorization", "Basic " + new String(credentials, StandardCharsets.UTF_8));
            URL keyMgtURL = new URL(url);
            int keyMgtPort = keyMgtURL.getPort();
            String keyMgtProtocol = keyMgtURL.getProtocol();
            HttpClient httpClient = APIUtil.getHttpClient(keyMgtPort, keyMgtProtocol);
            HttpResponse httpResponse = null;
            int retryCount = 0;
            boolean retry;
            do {
                try {
                    httpResponse = httpClient.execute(method);
                    retry = false;
                } catch (IOException ex) {
                    retryCount++;
                    if (retryCount < revokedJWTTokensRetrievalRetries) {
                        retry = true;
                        log.warn("Failed retrieving revoked JWT token signatures from remote endpoint: " +
                                ex.getMessage() + ". Retrying after " + revokedJWTTokensRetrievalTimeoutInSeconds +
                                " seconds...");
                        Thread.sleep(revokedJWTTokensRetrievalTimeoutInSeconds * 1000);
                    } else {
                        throw ex;
                    }
                }
            } while (retry);

            String responseString = EntityUtils.toString(httpResponse.getEntity(), "UTF-8");
            if (responseString != null && !responseString.isEmpty()) {
                return new Gson().fromJson(responseString, RevokedJWTTokenDTO[].class);
            }
        } catch (IOException | InterruptedException e) {
            log.error("Exception when retrieving revoked JWT tokens from remote endpoint ", e);
        }
        return null;
    }

    private void loadRevokedJWTTokensFromWebService() {

        RevokedJWTTokenDTO[] revokedJWTTokenDTOS = retrieveRevokedJWTTokensData();
        if(revokedJWTTokenDTOS != null) {
            for (RevokedJWTTokenDTO revokedJWTToken : revokedJWTTokenDTOS) {
                RevokedJWTDataHolder.getInstance().addRevokedJWTToMap(revokedJWTToken.getSignature(),
                        revokedJWTToken.getExpiryTime());
                if(log.isDebugEnabled()) {
                    log.debug("JWT signature : " + revokedJWTToken.getSignature() + " added to the revoke map.");
                }
            }
        } else {
            log.debug("No revoked JWT tokens are retrieved via web service");
        }
    }

    /**
     *  Initiates the timer task to fetch data from the web service.
     *  Timer task will not run after the retry count is completed.
     */
    public void startRevokedJWTTokensRetriever() {
        //using same initDelay as in keytemplates,blocking conditions retriever
        new Timer().schedule(this, getEventHubConfiguration().getInitDelay());
    }

    protected EventHubConfigurationDto getEventHubConfiguration() {
        return ServiceReferenceHolder.getInstance().getAPIManagerConfiguration().getEventHubConfigurationDto();
    }

}


text:
using HarmonyLib;
using NetworkSkins.Skins;
// ReSharper disable InconsistentNaming

namespace NetworkSkins.Patches.PedestrianBridgeAI
{
    [HarmonyPatch(typeof(global::PedestrianBridgeAI), "GetNodeBuilding")]
    public class PedestrianBridgeAiGetNodeBuildingPatch
    {
        public static void Prefix(ref global::PedestrianBridgeAI __instance, ushort nodeID, out PedestrianBridgePillarPatcherState? __state)
        {
            var skin = NetworkSkinManager.NodeSkins[nodeID];
            if (skin != null)
            {
                __state = new PedestrianBridgePillarPatcherState(__instance);

                __instance.m_bridgePillarInfo = skin.m_bridgePillarInfo;
                __instance.m_bridgePillarInfos = skin.m_bridgePillarInfos;
            }
            else
            {
                __state = null;
            }
        }

        public static void Postfix(ref global::PedestrianBridgeAI __instance, ref PedestrianBridgePillarPatcherState? __state)
        {
            __state?.Restore(__instance);
        }
    }

    public struct PedestrianBridgePillarPatcherState
    {
        public readonly BuildingInfo BridgePillarInfo;
        public readonly BuildingInfo[] BridgePillarInfos;

        public PedestrianBridgePillarPatcherState(global::PedestrianBridgeAI netAi)
        {
            BridgePillarInfo = netAi.m_bridgePillarInfo;
            BridgePillarInfos = netAi.m_bridgePillarInfos;
        }

        public void Restore(global::PedestrianBridgeAI netAi)
        {
            netAi.m_bridgePillarInfo = BridgePillarInfo;
            netAi.m_bridgePillarInfos = BridgePillarInfos;
        }
    }
}


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.8.14"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>coda db: coda::db::row_iterator< ValueType, NonConst, RowType > Class Template Reference</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>

<link href="search/search.css" rel="stylesheet" type="text/css"/>

<link href="doxygen.css" rel="stylesheet" type="text/css" />

coda db

coda

db

row_iterator

Public Member Functions |
Protected Member Functions |
Protected Attributes

coda::db::row_iterator< ValueType, NonConst, RowType > Class Template Reference

#include <row.h>

Inheritance diagram for coda::db::row_iterator< ValueType, NonConst, RowType >:

  <map id="coda::db::row_5Fiterator_3C_20ValueType_2C_20NonConst_2C_20RowType_20_3E_map" name="coda::db::row_5Fiterator_3C_20ValueType_2C_20NonConst_2C_20RowType_20_3E_map">
</map>

Public Member Functions

 	row_iterator (const std::shared_ptr< RowType > &pRow, int position)

 	row_iterator (const row_iterator &other)

 	row_iterator (row_iterator &&other)

row_iterator & 	operator= (const row_iterator &other)

row_iterator & 	operator= (row_iterator &&other)

ValueType & 	operator* ()

ValueType * 	operator-> ()

ValueType 	operator[] (size_t nPosition)

row_iterator & 	operator++ ()

row_iterator 	operator++ (int)

row_iterator & 	operator-- ()

row_iterator 	operator-- (int)

row_iterator 	operator+ (int n)

row_iterator & 	operator+= (int n)

row_iterator 	operator- (int n)

row_iterator & 	operator-= (int n)

bool 	operator== (const row_iterator &other) const

bool 	operator!= (const row_iterator &other) const

bool 	operator< (const row_iterator &other) const

bool 	operator<= (const row_iterator &other) const

bool 	operator> (const row_iterator &other) const

bool 	operator>= (const row_iterator &other) const

int 	operator- (const row_iterator &other)

std::string 	name () const

Protected Member Functions

void 	set_current_value (size_t index)

Protected Attributes

std::shared_ptr< RowType > 	row_

int 	position_

NonConst 	currentValue_

Detailed Description

template<class ValueType, class NonConst, class RowType>

class coda::db::row_iterator< ValueType, NonConst, RowType >

iterator for columns in a row

The documentation for this class was generated from the following file:

row.h

<address class="footer">
Generated by

 1.8.14
</address>


text:
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Linq;
using System.Reflection;
using System.Text;

namespace System.Data.Entity.Metadata
{
    /// <summary>
    /// Entity metadata.
    /// </summary>
    public class EntityMetadata
    {
        private Dictionary<string, PropertyMetadata> _Properties;

        /// <summary>
        /// Initialize entity metadata.
        /// </summary>
        /// <param name="type"></param>
        public EntityMetadata(Type type)
        {
            if (type == null)
                throw new ArgumentNullException("type");

            Type = type;
            KeyType = type.GetProperty("Index").PropertyType;

            PropertyInfo[] properties = type.GetProperties().Where(t=>t.GetCustomAttribute<NotMappedAttribute>() == null).ToArray();
            Properties = properties.Select(t => new PropertyMetadata(t)).OrderBy(t => t.Order).ToArray();

            ViewProperties = Properties.Where(t => !t.IsHiddenOnView).ToArray();
            EditProperties = Properties.Where(t => !t.IsHiddenOnEdit && t.Property.GetSetMethod() != null).ToArray();
            SearchProperties = Properties.Where(t => t.Searchable).ToArray();
            DetailProperties = Properties.Where(t => !t.IsHiddenOnDetail).ToArray();

            _Properties = new Dictionary<string, PropertyMetadata>();
            for (int i = 0; i < Properties.Length; i++)
            {
                if (_Properties.ContainsKey(Properties[i].Property.Name))
                    continue;
                _Properties.Add(Properties[i].Property.Name, Properties[i]);
            }

            EntityAuthenticationAttribute authenticate = type.GetCustomAttribute<EntityAuthenticationAttribute>();
            if (authenticate == null)
            {
                AllowAnonymous = true;
                AddRoles = new string[0];
                EditRoles = new string[0];
                ViewRoles = new string[0];
                RemoveRoles = new string[0];
            }
            else
            {
                AllowAnonymous = authenticate.AllowAnonymous;
                AddRoles = authenticate.AddRolesRequired;
                EditRoles = authenticate.EditRolesRequired;
                ViewRoles = authenticate.ViewRolesRequired;
                RemoveRoles = authenticate.RemoveRolesRequired;
            }

            DisplayNameAttribute display = type.GetCustomAttribute<DisplayNameAttribute>();
            if (display != null)
                Name = display.DisplayName == null ? type.Name : display.DisplayName;
            else
                Name = type.Name;

            DisplayColumnAttribute displayColumn = type.GetCustomAttribute<DisplayColumnAttribute>();
            if (displayColumn != null)
            {
                DisplayProperty = GetProperty(displayColumn.DisplayColumn);
                if (displayColumn.SortColumn != null)
                {
                    SortProperty = Properties.SingleOrDefault(t => t.Property.Name == displayColumn.SortColumn);
                    SortDescending = displayColumn.SortDescending;
                }
            }
            else
                DisplayProperty = GetProperty("Index");
            ParentAttribute parent = type.GetCustomAttribute<ParentAttribute>();
            if (parent != null)
            {
                ParentProperty = Properties.SingleOrDefault(t => t.Property.Name == parent.PropertyName);
                ParentLevel = parent.Level;
            }
        }

        /// <summary>
        /// Get the system type of entity.
        /// </summary>
        public Type Type { get; private set; }

        /// <summary>
        /// Get the system type of key of entity.
        /// </summary>
        public Type KeyType { get; private set; }

        /// <summary>
        /// Get the display name of entity.
        /// </summary>
        public string Name { get; private set; }

        /// <summary>
        /// Get the display property of entity.
        /// </summary>
        public PropertyMetadata DisplayProperty { get; private set; }

        /// <summary>
        /// Get the sort property of entity.
        /// </summary>
        public PropertyMetadata SortProperty { get; private set; }

        /// <summary>
        /// Get the parent property of entity.
        /// </summary>
        public PropertyMetadata ParentProperty { get; private set; }

        /// <summary>
        /// Get the parent level of entity.
        /// </summary>
        public int ParentLevel { get; private set; }

        /// <summary>
        /// Get the sort mode of entity.
        /// </summary>
        public bool SortDescending { get; private set; }

        /// <summary>
        /// Get the properties of entity.
        /// </summary>
        public PropertyMetadata[] Properties { get; private set; }

        /// <summary>
        /// Get the properties of entity in viewlist.
        /// </summary>
        public PropertyMetadata[] ViewProperties { get; private set; }

        /// <summary>
        /// Get the properties of entity while edit.
        /// </summary>
        public PropertyMetadata[] EditProperties { get; private set; }

        /// <summary>
        /// Get the properties of entity while search.
        /// </summary>
        public PropertyMetadata[] SearchProperties { get; private set; }

        /// <summary>
        /// Get the properties of entity in detail.
        /// </summary>
        public PropertyMetadata[] DetailProperties { get; private set; }

        /// <summary>
        /// Get is entity allow anonymous operate.
        /// </summary>
        public bool AllowAnonymous { get; private set; }

        /// <summary>
        /// Get the roles to view entity.
        /// </summary>
        public string[] ViewRoles { get; private set; }

        /// <summary>
        /// Get the roles to add entity.
        /// </summary>
        public string[] AddRoles { get; private set; }

        /// <summary>
        /// Get the roles to edit entity.
        /// </summary>
        public string[] EditRoles { get; private set; }

        /// <summary>
        /// Get the roles to remove entity.
        /// </summary>
        public string[] RemoveRoles { get; private set; }

        /// <summary>
        /// Get the property of entity.
        /// </summary>
        /// <param name="name">Name of property.</param>
        /// <returns>Return property metadata. Return null if property doesn't exists.</returns>
        public PropertyMetadata GetProperty(string name)
        {
            if (!_Properties.ContainsKey(name))
                return null;
            return _Properties[name];
        }
    }
}