text:
TAGGER=./hindi-pos-tagger/bin/tnt -v0 -H hindi-pos-tagger/models/hindi
LEMMATIZER=./hindi-pos-tagger/bin/lemmatiser.py hindi-pos-tagger/models/hindi.lemma
TAG2VERT=./bin/tag2vert.py
NORMALIZE=./bin/normalize_vert.py
POSMOD=./bin/modify_pos.py
ADDDUMMY= ./bin/add_dummy_word.py
MERGE= ./bin/merge.py
CONVERT_NULL=./bin/convert_NULL.py
CONVERT_FORMAT=./bin/convert_format.py
TOKENIZER=./bin/unitok.py -l hindi -n
INPUT = ../../tmp/hindi.input.txt
OUTPUT = ../../tmp/hindi.output
RM = ../../tmp/*.tmp*

# Normalizer replaces some of the spellings with easy spellings on which the parser or MT systems work very well.

$(OUTPUT):
	# uncomment below line if you require a normalizer
	cat $(INPUT) | $(TOKENIZER) | sed -e 's/।/./g' | sed -e 's/^\.$$/.\n<\/s>\n<s>/g' |  $(NORMALIZE)  > $@.tmp.words
	# uncomment below line if you do not require a normalizer
	# cat $< | $(TOKENIZER) |  sed -e 's/।/./g' | sed -e 's/^\.$$/.\n<\/s>\n<s>/g'  > $@.tmp.words
	$(TAGGER) $@.tmp.words | sed -e 's/\t\+/\t/g' | $(LEMMATIZER) | $(TAG2VERT) | $(POSMOD) | cut -f1,2,3 > $@.tmp.tag
	echo python3 --version
	python3 bin/convert_format.py $@.tmp.tag $@.tmp.tag.conll
	java -jar bin/malt.jar -c test_complete -i $@.tmp.tag.conll -o $@.tmp.output -m parse
	python3 bin/convert_output.py $@.tmp.output $@
	rm  $(RM)
	echo "Output stored in $@"

clean:
	rm  $(RM)


text:
/* fonts.css from the YUI Library: developer.yahoo.com/yui/
refer to developer.yahoo.com/yui/3/cssfonts/ for font sizing percentages

there are three custom edits:
 * remove arial, helvetica from explicit font stack
 * we normalize monospace styles ourselves
 * table font-size is reset in the HTML5 reset above so there is no need to repeat
 */
body { font:13px/1.231 sans-serif; *font-size:small; } /* hack retained to preserve specificity */

select, input, textarea, button { font:99% sans-serif; }

/* normalize monospace sizing
 * en.wikipedia.org/wiki/MediaWiki_talk:Common.css/Archive_11#Teletype_style_fix_for_Chrome
 */
pre, code, kbd, samp { font-family: monospace, sans-serif; }

/*
 * minimal base styles
 */

body, select, input, textarea {
  color: #777;
}

body {
    background: black;
}

#main p {
  line-height: 1.4em;
}
#main p {
  line-height: 1.4em;
}
#main p, #main h2, #main h3, #main dl, #main ul, #main table {
  margin-bottom: 1em;
}
#main ul {
  list-style: disc;
  margin-left: 20px;
}

/* headers (h1,h2,etc) have no default font-size or margin,
you'll want to define those yourself. */
h1,h2,h3,h4,h5,h6 { font-weight: bold; }

h2, h3 { font-family: Trebuchet MS }

/* always force a scrollbar in non-IE: */
html { overflow-y: scroll; }

/* accessible focus treatment: people.opera.com/patrickl/experiments/keyboard/test */
a:hover, a:active { outline: none; }

a, a:active, a:visited { color: #31519A; }
a:hover { color: #036; }

ol { list-style-type: decimal; }

/* remove margins for navigation lists */
nav ul, nav li { margin: 0; }

small { font-size: 85%; }
strong, th { font-weight: bold; }

td { vertical-align: top; }

/* set sub, sup without affecting line-height: gist.github.com/413930*/
sub, sup { font-size: 75%; line-height: 0; position: relative; }
sup { top: -0.5em; }
sub { bottom: -0.25em; }

pre {
  padding: 15px;

  /* www.pathf.com/blogs/2008/05/formatting-quoted-code-in-blog-posts-css21-white-space-pre-wrap/ */
  white-space: pre; /* CSS2 */
  white-space: pre-wrap; /* CSS 2.1 */
  white-space: pre-line; /* CSS 3 (and 2.1 as well, actually) */
  word-wrap: break-word; /* IE */
}

textarea { overflow: auto; } /* www.sitepoint.com/blogs/2010/08/20/ie-remove-textarea-scrollbars/ */

.ie6 legend, .ie7 legend { margin-left: -7px; } /* thnx ivannikolic! */

/* align checkboxes, radios, text inputs with their label
by: Thierry Koblentz tjkdesign.com/ez-css/css/base.css  */
input[type="radio"] { vertical-align: text-bottom; }
input[type="checkbox"] { vertical-align: baseline; }
.ie7 input[type="checkbox"] { vertical-align: baseline; }
.ie6 input { vertical-align: text-bottom; }

/* hand cursor on clickable input elements */
label, input[type="button"], input[type="submit"], input[type="image"], button { cursor: pointer; }

/* webkit browsers add a 2px margin outside the chrome of form elements */
button, input, select, textarea { margin: 0; }

/* colors for form validity */
input:valid, textarea:valid   {  }
input:invalid, textarea:invalid {
  border-radius: 1px;
  -moz-box-shadow: 0px 0px 5px red;
  -webkit-box-shadow: 0px 0px 5px red;
  box-shadow: 0px 0px 5px red;
}
.no-boxshadow input:invalid,
.no-boxshadow textarea:invalid { background-color: #f0dddd; }

/* These selection declarations have to be separate.
No text-shadow: twitter.com/miketaylr/status/12228805301
Also: hot pink. */
::-moz-selection{ background: #0095da; color:#fff; text-shadow: none; }
::selection { background:#0095da; color:#fff; text-shadow: none; }

/*  j.mp/webkit-tap-highlight-color */
a:link { -webkit-tap-highlight-color: #FF5E99; }

/* make buttons play nice in IE:
www.viget.com/inspire/styling-the-button-element-in-internet-explorer/ */
button {  width: auto; overflow: visible; }

/* bicubic resizing for non-native sized IMG:
code.flickr.com/blog/2008/11/12/on-ui-quality-the-little-things-client-side-image-resizing/ */
.ie7 img { -ms-interpolation-mode: bicubic; }

span.price {
  font-size: 1.5em;
}
span.price.discounted {
  color: #f00;
  text-decoration: line-through;
}
span.price.selling {
  font-weight: bold;
}

h1{
  color: #D3D3D3;
  font: normal normal 3.5em Arvo;
  text-transform: uppercase;
  margin: 60px 0 20px 0;
  padding-bottom: 10px;
}

h1 a {
  margin-left: 20px;
  text-transform:lowercase;
}

h2{
  font: normal normal 2em Arvo;
  text-transform: uppercase;
  margin: 30px 0 10px 0;
  padding-bottom: 10px;
}

h3 {
  color: #777;
  font: normal normal 2em Arvo;
  margin-bottom: 10px;
}

redpajama_set_name: RedPajamaGithub


text:
using NationBuilderConnect.Model;
using Newtonsoft.Json;

namespace NationBuilderConnect.Client.Model.Responses
{
    /// <summary>
    ///     The response received from the server when pushing a person
    /// </summary>
    public class PushPersonResponse
    {
        /// <summary>
        /// The created or updated person
        /// </summary>
        [JsonProperty("person")]
        public Person Person { get; private set; }

        /// <summary>
        /// The person's precinct
        /// </summary>
        [JsonProperty("precinct")]
        public Precinct Precinct { get; private set; }

        /// <summary>
        ///     Whether or not the person was created. If false, an existing person was updated
        /// </summary>
        public bool WasNewPersonCreated { get; internal set; }
    }
}


text:
package atlas

import (
	"reflect"
)

/*
	The AtlasEntry is a declarative roadmap of what we should do for
	marshal and unmarshal of a single object, keyed by type.

	There are a lot of paths your mappings might want to take:

	  - For a struct type, you may simply want to specify some alternate keys, or some to leave out, etc.
	  - For an interface type, you probably want to specify one of our interface muxing strategies
	     with a mapping between enumstr:typeinfo (and, what to do if we get a struct we don't recognize).
	  - For a string, int, or other primitive, you don't need to say anything: defaults will DTRT.
	  - For a typedef'd string, int, or other primitive, you *still* don't need to say anything: but,
	     if you want custom behavior (say, transform the string to an int at the last second, and back again),
		 you can specify transformer functions for that.
	  - For a struct type that you want to turn into a whole different kind (like a string): use
	     those same transform functions.  (You'll no longer need a FieldMap.)
	  - For the most esoteric needs, you can fall all the way back to providing a custom MarshalMachine
	     (but avoid that; it's a lot of work, and one of these other transform methods should suffice).
*/
type AtlasEntry struct {
	// The reflect info of the type this morphism is regarding.
	Type reflect.Type

	// --------------------------------------------------------
	// The big escape valves: wanna map to some other kind completely?
	// --------------------------------------------------------

	// Transforms the value we reached by walking (the 'live' value -- which
	// must be of `this.Type`) into another value (the 'serialable' value --
	// which will be of `this.MarshalTransformTargetType`).
	//
	// The target type may be anything, even of a completely different Kind!
	//
	// This transform func runs first, then the resulting value is
	// serialized (by running through the path through Atlas again, so
	// chaining of transform funcs is supported, though not recommended).
	MarshalTransformFunc MarshalTransformFunc
	// The type of value we expect after using the MarshalTransformFunc.
	//
	// The match between transform func and target type should be checked
	// during construction of this AtlasEntry.
	MarshalTransformTargetType reflect.Type

	// Expects a different type (the 'serialable' value -- which will be of
	// 'this.UnmarshalTransformTargetType') than the value we reached by
	// walking (the 'live' value -- which must be of `this.Type`).
	//
	// The target type may be anything, even of a completely different Kind!
	//
	// The unmarshal of that target type will be run first, then the
	// resulting value is fed through this function to produce the real value,
	// which is then placed correctly into bigger mid-unmarshal object tree.
	//
	// For non-primitives, unmarshal of the target type will always target
	// an empty pointer or empty slice, roughly as per if it was
	// operating on a value produced by `TargetType.New()`.
	UnmarshalTransformFunc UnmarshalTransformFunc
	// The type of value we will manufacture an instance of and unmarshal
	// into, then when done provide to the UnmarshalTransformFunc.
	//
	// The match between transform func and target type should be checked
	// during construction of this AtlasEntry.
	UnmarshalTransformTargetType reflect.Type

	// --------------------------------------------------------
	// Standard options for how to map (varies by Kind)
	// --------------------------------------------------------

	// A "tag" to emit when marshalling this type of value;
	// and when unmarshalling, this tag will cause unmarshal to pick
	// this atlas (and if there's conflicting type info, error).
	Tag int
	// Flag for whether the Tag feature should be used (zero is a valid tag).
	Tagged bool

	// A mapping of fields in a struct to serial keys.
	// Only valid if `this.Type.Kind() == Struct`.
	StructMap *StructMap

	// Configuration for how to traverse a map kind.
	// Only valid if `this.Type.Kind() == Map`.
	MapMorphism *MapMorphism

	// Configuration for how to pick concrete types to fill a union interface.
	// Only valid if `this.Type.Kind() == Interface`.
	UnionKeyedMorphism *UnionKeyedMorphism

	// FUTURE: enum-ish primitives, multiplexers for interfaces,
	//  lots of such things will belong here.

	// --------------------------------------------------------
	// Hooks, validate helpers
	// --------------------------------------------------------

	// A validation function which will be called for the whole value
	// after unmarshalling reached the end of the object.
	// If it returns an error, the entire unmarshal will error.
	//
	// Not used in marshalling.
	// Not reachable if an UnmarshalTransform is set.
	ValidateFn func(v interface{}) error
}

func BuildEntry(typeHintObj interface{}) *BuilderCore {
	rt := reflect.TypeOf(typeHintObj)
	if rt.Kind() == reflect.Ptr {
		if rt.Elem().Kind() == reflect.Interface {
			rt = rt.Elem()
		} else {
			panic("invalid atlas build: use the bare object, not a pointer (refmt will handle pointers automatically)")
		}
	}
	return &BuilderCore{
		&AtlasEntry{Type: rt},
	}
}

/*
	Intermediate step in building an AtlasEntry: use `BuildEntry` to
	get one of these to start with, then call one of the methods
	on this type to get a specialized builder which has the methods
	relevant for setting up that specific kind of mapping.

	One full example of using this builder may look like the following:

		atlas.BuildEntry(Formula{}).StructMap().Autogenerate().Complete()

	Some intermediate manipulations may be performed on this object,
	for example setting the "tag" (if you want to use cbor tagging),
	before calling the specializer method.
	In this case, just keep chaining the configuration calls like so:

		atlas.BuildEntry(Formula{}).UseTag(4000)
			.StructMap().Autogenerate().Complete()

*/
type BuilderCore struct {
	entry *AtlasEntry
}

func (x *BuilderCore) UseTag(tag int) *BuilderCore {
	x.entry.Tagged = true
	x.entry.Tag = tag
	return x
}


text:
AVSystem commons library contains a typesafe, typeclass-based serialization framework, similar to other Scala
serialization libraries like [Circe](https://circe.github.io/circe/) or [uPickle](https://github.com/lihaoyi/upickle).
However, `GenCodec` is **not** a JSON library even though it has support for JSON serialization.

**[API reference](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/index.html)**

<!-- START doctoc generated TOC please keep comment here to allow auto update -->
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
## Table of Contents  *generated with [DocToc](https://github.com/thlorenz/doctoc)*

- [`GenCodec` typeclass](#gencodec-typeclass)
  - [Writing and reading simple (primitive) values](#writing-and-reading-simple-primitive-values)
  - [Writing and reading *lists*](#writing-and-reading-lists)
  - [Writing and reading *objects*](#writing-and-reading-objects)
    - [More about object field order](#more-about-object-field-order)
  - [Writing custom "native" types](#writing-custom-native-types)
  - [Reading metadata from an `Input`](#reading-metadata-from-an-input)
  - [Implementations of `Input` and `Output` available by default](#implementations-of-input-and-output-available-by-default)
- [Codecs available by default](#codecs-available-by-default)
- [`GenKeyCodec`](#genkeycodec)
- [Serializing and deserializing examples](#serializing-and-deserializing-examples)
- [Making your own types serializable](#making-your-own-types-serializable)
- [Simple wrappers](#simple-wrappers)
- [Case classes](#case-classes)
    - [Safe evolution and refactoring - summary](#safe-evolution-and-refactoring---summary)
  - [Case class like types](#case-class-like-types)
- [Singletons](#singletons)
- [Sealed hierarchies](#sealed-hierarchies)
  - [Nested format](#nested-format)
  - [Flat format](#flat-format)
  - [Customizing sealed hierarchy codecs](#customizing-sealed-hierarchy-codecs)
  - [Nested vs flat format](#nested-vs-flat-format)
- [Third party classes](#third-party-classes)
  - [Injecting additional implicits into `GenCodec` materialization](#injecting-additional-implicits-into-gencodec-materialization)
- [`GenObjectCodec`](#genobjectcodec)
- [Summary](#summary)
  - [Codec dependencies](#codec-dependencies)
  - [Types supported by automatic materialization](#types-supported-by-automatic-materialization)
  - [Recursive types, generic types and GADTs (generalized algebraic data types)](#recursive-types-generic-types-and-gadts-generalized-algebraic-data-types)
  - [Customizing annotations](#customizing-annotations)
  - [Safely introducing changes to serialized classes (retaining backwards compatibility)](#safely-introducing-changes-to-serialized-classes-retaining-backwards-compatibility)
- [Performance](#performance)

<!-- END doctoc generated TOC please keep comment here to allow auto update -->

Features:
* `GenCodec` is *generic* - it supports many serialization formats. There is a JSON backend provided by default but it can support any format structurally similar to JSON (one that supports writing simple values, lists and objects). Enabling support for a particular serialization format is a matter of providing adequate implementations of [Input](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Input.html) and [Output](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Output.html). This is particularly easy to do for any of the popular JSON AST representations (Circe, Play, uPickle, etc.). Even though `GenCodec` supports JSON serialization, it's not a *JSON library*. Therefore, it does not provide its own JSON AST representation.
* `GenCodec` is *typesafe* - it is typeclass-based, i.e. type `T` can be serialized/deserialized only when there is an implicit `GenCodec[T]` available. This is fundamental when it comes to type safety. Thanks to how typeclasses work in Scala, data types that the programmer wants to serialize are thoroughly validated during compilation to determine whether they can be serialized or not. For example `List[T]` is serializable only when `T` is serializable and a case class is serializable only if its fields are serializable. This validation goes arbitrary levels deep. Typeclass also helps making the serialization format more compact and platform independent, by avoiding usage of runtime reflection.
* `GenCodec` is *boilerplate-free* - it provides macros for automatic derivation of codecs for case classes (and case class like types) and sealed hierarchies. This includes complex types like recursively-defined case classes and GADTs. These macro generated codecs can be further customized with annotations.
* `GenCodec` is *fast* - the speed primarily comes from avoiding any intermediate representations during serialization and deserialization, like some JSON AST or [shapeless](https://github.com/milessabin/shapeless)' `Generic` used by many other Scala serialization libraries. See [Performance](#performance) for benchmark results.
* `GenCodec` works in *ScalaJS*. Macro-generated codecs compile to compact and fast JavaScript.

## [`GenCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html) typeclass

The central trait of the framework is the [`GenCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html) typeclass:

```scala
trait GenCodec[T] {
  def read(input: Input): T
  def write(output: Output, value: T): Unit
}
```

A `GenCodec[T]` can read a value of type `T` from an [`Input`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Input.html)
and write a value of type `T` to an [`Output`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Output.html).
[`Input`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Input.html) and [`Output`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Output.html) are abstract, raw, stream-like, mutable entities which perform the actual serialization and
deserialization using some  particular format hardwired into them, like JSON. Therefore, [`GenCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html) by itself is not
bound to any format. It only depends on the fact that this format is capable of serializing following types and structures:
* integer numbers, arbitrarily large (`Byte`s, `Short`s, `Int`s, `Long`s and `BigInt`s)
* decimal numbers, arbitrarily large (`Float`s, `Double`s and `BigDecimal`s)
* `Char`s, `String`s, `Boolean`s and `null`s
* arbitrary byte chunks
* millisecond-precision timestamps
* arbitrarily nested *lists*
* arbitrarily nested *objects*, i.e. sequences of *fields* - key-value mappings where each key is a string

Not all of these types must be supported "natively". Some of them may be represented using others, e.g. `Byte`s and
`Short`s using `Int`s or timestamps using `Long`s with number of milliseconds since Unix epoch. **Serialization format
is not required to keep type information**. It should only ensure that writing and reading the same types is consistent,
e.g. when an `Int` was written by a codec then it should be also able to successfully read an `Int`. However, it is the
responsibility of the codec itself to have writing and reading logic consistent with each other.

The only exception to this rule is `null` handling - every `Input` implementation must be able to distinguish `null`
value from any other values through its `readNull()` method. This means that e.g. empty string cannot be used as a
representation of `null` because it would be then indistinguishable from an actual empty string.
Every `Input`/`Output` pair must have a dedicated representation for `null`.

### Writing and reading simple (primitive) values

Every [`Output`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Output.html) exposes
a [`SimpleOutput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/SimpleOutput.html)
instance through its `writeSimple` method. `SimpleOutput` aggregates methods for writing simple/primitive values like
`Int`, `String`, etc. These values are considered "simple" because they can be written as a whole with a single method
call.

**NOTE**: `SimpleOutput`'s methods like `writeString` **MUST NEVER** be passed `null` as the value to write.
Instead, `GenCodec` must manually do null checking and use `Output`'s `writeNull()` method for explicit `null` values.
Fortunately, there are helpers for creating nullsafe codecs, e.g. `GenCodec.nullableSimple`.

Similarly, every [`Input`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/Output.html)
exposes a [`SimpleInput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/SimpleOutput.html)
instance through `readSimple` method which can be used to read simple values. `GenCodec` should always read the same type
as was written. For example, if `writeInt` is used to write then `readInt` should be used to read by the same codec
implementation.

If the codec wants to be nullsafe, then it can use `readNull()` on `Input`. Unlike other reading methods, `readNull()`
returns a boolean value which indicates if the value held by `Input` was actually a `null`. If `false` is returned
then the codec may proceed to read its actual type, e.g. with `readString`.

### Writing and reading *lists*

In order to write a *list*, which is an ordered sequence of arbitrary values, the codec must obtain a
[`ListOutput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/ListOutput.html)
by calling `writeList` on `Output`. Then, it may use its `writeElement()` to obtain a fresh `Output` for each consecutive
element. `Output` returned for each element must be fully written *before* obtaining `Output` for next element.
After all elements are written, the codec is required to call `finish()` on the `ListOutput`.

Reading a list from an `Input` is analogous. The codec must obtain a [`ListInput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/ListInput.html)
using `Input`'s `readList()` method. `ListInput` exposes a `hasNext` method to check whether there are more elements to read
and a `nextElement` method which provides an `Input` instance for next list element. This `Input` instance must be fully
read before `hasNext` or `nextElement` is called again. At the very end, the codec may want to call `skipRemaining()`
on then `ListInput` in order to skip any remaining elements (only necessary when list is not read until `hasNext` returns
`false` - e.g. when a fixed number of elements is expected).

Note that because serialization formats are not required to keep type information, lists may sometimes be
indistinguishable from some primitive values. For example, `JsonStringOutput` may use JSON arrays of numbers to represent
binary values (which are written through `SimpleInput`'s `writeBinary` method). In such situation, calling `readList()`
on the JSON input will succeed even if `writeBinary` was originally used on the JSON output.

### Writing and reading *objects*

An *object* is a mapping between non-null `String` keys and arbitrary values. Internally, each serialization format
may represent an object either as an ordered key-value pair sequence or as a random access key-value mapping (i.e.
it must *either* preserve order of fields *or* provide random access to fields by field name).

In order to write an object, `writeObject()` must be called on `Output` to obtain an
[`ObjectOutput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/ObjectOutput.html)
instance. The codec may then obtain an `Output` instance for each named field through `writeField` method.
This instance must be fully written before `writeField` can be called again. At the very end, `finish()` must be called
on the `ObjectOutput`.

In order to read an object, the codec must call `readObject()` on `Input` in order to obtain an
[`ObjectInput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/ObjectInput.html)
instance. Similarly to `ListInput`, `ObjectInput` provides a `hasNext` method to check whether there are any remaining
fields in the `ObjectInput`. In order to read each field, the codec may use `nextField()` method with returns a
[`FieldInput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/FieldInput.html) - a
subinterface of `Input` which additionally provides field name.

If serialization format implemented by particular `Input`/`Output` implementation preserves field order then `nextField`
is guaranteed to return fields in the same order as they were written by `ObjectOutput`. However, if the serialiaztion
format does **not** preserve field order then it must provide random field access instead. This is done by `peekField`
method. Codec implementations must be prepared for both situations.

If you need a custom object serialization for your type then your best bet is probably to implement appropriate `apply` and `unapply`
methods on companion object of your type. And even if you can't do it (e.g. because you're writing a codec for a third
party type) then you can still use [`fromApplyUnapplyProvider`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec$.html#fromApplyUnapplyProvider[T](applyUnapplyProvider:Any):com.avsystem.commons.serialization.GenCodec[T]) and avoid implementing your codec manually.

Finally, if you *really* need to write your object codec by hand then you can make it easier by using
`getNextNamedField` helper method on `ObjectInput` instead of `nextField` and `peekField` directly.

#### More about object field order

For most serialization formats, it's completely natural to retain object field order. For example, a JSON string naturally has object fields stored in the order that they
were written to `JsonObjectOutput`. For these formats it is required that an `ObjectInput` returns object fields in exactly the same order as they were written to
a corresponding `ObjectOutput`. This normally includes all serialization formats backed by strings, byte sequences, streams, etc.

> :bulb: It is generally recommended reading data with particular `Input` only when it was written using its
> corresponding `Output`. However, some intputs offer additional guarantees. For example `JsonStringInput` will accept
> fields in any order even though `JsonStringOutput` retains field order in the resulting JSON string. This way it's
> possible to use `JsonStringInput` to read JSON that was written by other tools or humans that may mix up field order.

However, there are also serialization formats that use memory representation where an object is usually backed by a hashtable. Such representations cannot retain field order.
`GenCodec` can still work with these but as an alternative to preserving field order, they must implement random field access (field-by-name access). This is done by
overriding [`peekField`](avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/ObjectInput.html#peekField(name:String):com.avsystem.commons.misc.Opt[com.avsystem.commons.serialization.FieldInput]) on `ObjectInput`.

To summarize, an `ObjectInput` is generally **not** guaranteed to retain order of fields as they were written to an `ObjectOutput` but if it doesn't then it must
provide random field access by field name. Also, note that out of all the default and macro-generated codecs provided, only [flat sealed hierarchy codecs](#flat-format) actually depends
on this requirement. All the other (non-custom) codecs ignore field order during deserialization so you can e.g. freely
reorder fields in your case classes if they use macro generated codecs.

### Writing custom "native" types

`Input` and `Output` interfaces provide some "backdoors" which make it possible for codec implementations to take advantage
of particular serialization format's additional capabilities which are not exposed in standard `Input`/`Output` APIs.
For example, [BSON](http://bsonspec.org/) used by MongoDB has special representation for Object IDs which we would like
to use. `Input` and `Output` implementations for BSON may support it through their `writeCustom` and `readCustom` methods.
See their documentation for more details.

### Reading metadata from an `Input`

As mentioned before, serialization formats in general are not required to keep any type information (except distinguishing
`null` values from others). This way they can be as simple and compact as possible. However, many common formats
naturally *do* keep some type information. For example, when reading a JSON we can easily check whether it contains an
object, an array, a string, a number, a boolean or a `null` value. Codecs could use this in order to optimize layout of
their serialized values and avoid manual writing of their own type information.

`Input` may expose type information through its `readMetadata` method. `Output` on the other hand may tell the codec
whether its corresponding `Input` is capable of keeping this type information - this is done by `Output`'s `keepsMetadata`
method. See their documentation for more details.

### Implementations of `Input` and `Output` available by default

Implementations of `Input` and `Output` provided by `scala-commons` by default are:
* `SimpleValueInput`/`SimpleValueOutput` - translate serialized values into simple Scala objects.
  Primitive types are represented by themselves, lists are represented by standard Scala `List[T]` values and objects
  are represented by standard Scala `Map[String, T]` values.
* `JsonStringInput`/`JsonStringOutput` - JSON implementations that work
directly on `String` values, without intermediate JSON AST
* `CborInput`/`CborOutput` - [Concise Binary Object Representation, CBOR](https://tools.ietf.org/html/rfc7049)
* `StreamInput`/`StreamOutput` - simple opaque binary format
* `HoconInput` (in `commons-hocon` module) - for reading from
  [Human Optimized Config Object Notation, HOCON](https://github.com/lightbend/config/blob/master/HOCON.md)
* `BsonReaderInput`/`BsonWriterOutput`,`BsonWriterNamedOutput`,`BsonValueOutput` (in `commons-mongo` module) - for
  reading and writing from [Binary JSON, BSON](http://bsonspec.org/), format used by MongoDB wire protocol.

## Codecs available by default

In order to serialize/deserialize value of some type, there needs to be an implicit value of type `GenCodec[T]` available.
The library by default provides codecs for common Scala and Java types:
* `Unit`, `Null`, `String`, `Symbol`, `Char`, `Boolean`, `Byte`, `Short`, `Int`, `Long`, `Float`, `Double`,
  `java.util.Date`, `Array[Byte]`, `BigInt`, `BigDecimal` and all their Java boxed counterparts
  (like `java.lang.Integer`).
* Any Scala tuple, provided that every tuple element type can be serialized. Tuples are serialized into lists.
* Any `Array[T]`, provided that `T` can be serialized
* Any Scala collection extending `scala.collection.Seq[T]` or `scala.collection.Set[T]`, provided that `T` can be
  serialized
  and there is an appropriate instance of `Factory` (e.g. `GenCodec[List[T]]` requires `Factory[T, List[T]]`).
  All standard library collections have this `Factory` instance so you only have to worry about it when dealing with
  custom collection implementations.
* Any `java.util.Collection[T]`, provided that `T` can be serialized
* Any `scala.collection.Map[K,V]` provided that `V` can be serialized and there is an implicit
  [`GenKeyCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenKeyCodec.html)
   available for `K` so that it can be converted to a string and used as object key.
  There must also be an appropriate instance of `Factory` for particular `Map` implementation (e.g. `HashMap[K,V]`
  requires `Factory[(K, V), HashMap[K, V]]`). All standard library map implementations have this `Factory` instance
  so you only have to worry about it when dealing with custom map implementations.
* Any `java.util.Map[K,V]`, with the same restrictions as for Scala maps (there must be `GenCodec` for `V` and
  `GenKeyCodec` for `K`)
* `Option[T]`, `Opt[T]`, `OptArg[T]`, `NOpt[T]`, `OptRef[T]`, provided that `T` can be serialized.
* `Either[A,B]`, provided that `A` and `B` can be serialized.
* [`NamedEnum`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/misc/NamedEnum.html)s
  whose companion object extends [`NamedEnumCompanion`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/misc/NamedEnumCompanion.html).
  This includes [`ValueEnum`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/misc/ValueEnum.html)s.
* Java enums

## [`GenKeyCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenKeyCodec.html)

For serialization of maps, there is an auxilliary typeclass - [`GenKeyCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenKeyCodec.html).
It provides the ability to translate values of some type into `String` keys that can be used as object keys by
[`GenCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html)s for Scala
and Java `Map` types. By default, following types have [`GenKeyCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenKeyCodec.html) provided:
* `String`
* `Boolean`, `Char`, `Byte`, `Short`, `Int`, `Long`
* `JBoolean`, `JCharacter`, `JByte`, `JShort`, `JInteger`, `JLong`
* [`NamedEnum`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/misc/NamedEnum.html)s
  whose companion object extends [`NamedEnumCompanion`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/misc/NamedEnumCompanion.html).
  This includes [`ValueEnum`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/misc/ValueEnum.html)s.
* Java enums

## Serializing and deserializing examples

To serialize and deserialize values, you can use the [`read`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html#read(input:com.avsystem.commons.serialization.Input):T)
and [`write`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html#write(output:com.avsystem.commons.serialization.Output,value:T):Unit)
methods directly. However, most of the *backends* (`Input`/`Output` implementations) provide some helper method which you can use instead of instantiating `Input` and `Output`s manually and passing them to `GenCodec`.

For example, the `JsonStringOutput` and `JsonStringInput` companion objects provide `read` and `write` functions that require an implicit `GenCodec` instance and convert directly between serialized types and JSON strings.

```scala
import com.avsystem.commons.serialization.json._

// primitive types like `Int` are represented by themselves
JsonStringOutput.write[Int](123) // JSON: 123
JsonStringInput.read[Int]("123") // 123

// `Option`, `Opt`, `NOpt`, `OptRef` and `OptArg` are represented either as `null` (when empty) or directly
// as the underlying value (when non-empty). `Some(null)` should not be used - it is indistinguishable from
// `None` unless the codec for the type wrapped in `Option` has some special `null` handling.
val raw = JsonStringOutput.write[Option[String]](None) // JSON: null
JsonStringInput.read[Option[String]]("null") // None

val raw = JsonStringOutput.write[Option[String]](Some("sth")) // "sth"
JsonStringInput.read[Option[String]]("sth") // Some("sth")

// all collections are represented as lists
val raw = JsonStringOutput.write(Set(1,2,3)) // JSON: [1,2,3]
JsonStringInput.read[Set[Int]](raw) // Set(1,2,3)

// maps are represented as objects
val raw = JsonStringOutput.write(Map("1" -> 1, "2" -> 2)) // JSON: {"1":1,"2":2}
JsonStringInput.read[Map[String,Int]](raw) // Map("1" -> 1, "2" -> 2)

// maps without GenKeyCodec instance for key type are represented as lists of key-value pairs
val raw = JsonStringOutput.write(Map(1.0 -> 1, 2.0 -> 2)) // JSON: [{"k":1.0,"v":1},{"k":2.0,"v":2}]
JsonStringInput.read[Map[Double,Int]](raw) // Map(1.0 -> 1, 2.0 -> 2)

// tuples are represented as heterogeneous lists
val raw = JsonStringOutput.write((1, "sth", 2.0)) // JSON: [1,"sth",2.0]
JsonStringInput.read[(Int,String,Double)](raw) // (1, "sth", 2.0)
```

## Making your own types serializable

In order to make your own (or third-party) classes and types serializable, you need to provide an instance of
[`GenCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html)
for it. You can implement it manually, but in most cases you'll probably rely on one of the predefined
codecs (primitive types, collections, standard library classes, etc.) or materialize it automatically by making
the companion object of your type extend
[`HasGenCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/HasGenCodec.html)
or by using the
[`GenCodec.materialize`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec$.html#materialize[T]:com.avsystem.commons.serialization.GenCodec[T])
macro directly.

## Simple wrappers

When your type is a simple wrapper over a type that already has a `GenCodec` instance, the easiest way to provide a codec for the wrapper type is to use `TransparentWrapperCompanion` as base class for its companion object:

```scala
case class BinaryData(bytes: Array[Byte]) extends AnyVal
object BinaryData extends TransparentWrapperCompanion[Array[Byte], BinaryData]
```

NOTE: if your type wraps `String`, `Int` or `Long` then you can use `StringWrapperCompanion`, `IntWrapperCompanion` and `LongWrapperCompanion` for brevity.

Using `TransparentWrapperCompanion` not only derives a `GenCodec` instance from the wrapped type's instance but also other typeclass instances, e.g. [`GenKeyCodec`](#genkeycodec). This mechanism is extensible and can be used by any typeclass that wants to "understand" wrapping. Therefore, `TransparentWrapperCompanion` is a generic tool, not specifically bound in any way to `GenCodec`.

## Case classes

```scala
case class Person(name: String, birthYear: Int)
object Person extends HasGenCodec[Person]
```

By making companion object of `Person` extend `HasGenCodec[Person]` you're making the compiler automatically
materialize an instance of `GenCodec[Person]` and inject it into the companion object.
The same works for case classes, case class like types (i.e. ones that have appropriate `apply` and `unapply` methods
in their companion object) and sealed hierarchies.

There are also other flavors of `HasGenCodec`. For example, `HasPolyGenCodec` handles parameterized (generic) data types
and `HasGenCodecWithDeps` allows injecting additional implicits into macro materialization.
Even if none of the available base companion classes fits your needs, it should be relatively painless to write your own.

The macro-materialized codec for case class serializes it into an object where field names serve as keys and field
values as associated values. For example, `Person("Fred", 1990)` would be represented (using `JsonStringOutput`)
as `{"name":"Fred","birthYear":1990}`.

The macro will only compile if it can find a `GenCodec` instance for every field of your case class. Otherwise, you'll get a compilation error telling you that some field can't be serialized because no implicit `GenCodec` is defined for its type. This way the macro will fully validate your case class. This is good - you'll never serialize any type by accident and if you forget to make any type serializable, the compiler will tell you about it. This way you avoid problems usually associated with runtime reflection based serialization, particularly popular in Java ecosystem.

In general, the serialization framework requires that the serialized representation retains order of object fields and during deserialization supplies them in exactly the same order as they were written during serialization. This is usually a reasonable assumption because most serialization formats are either textual, binary or stream-like (the word "serialization" itself indicates a sequential order). If field order is not preserved, serialization format must support random field access instead - see [object field order](#object-field-order).

The codec materialized for case class guarantees that the fields are written in the order of their declaration in constructor. However, during deserialization the codec is lenient and does not require that the order of fields is the same as during serialization. It will successfully deserialize the case class as long as all the fields are present in the serialized format (in any order) or have a default value defined (either as Scala-level default parameter value or with [`@whenAbsent`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/whenAbsent.html) annotation). Any superfluous fields will be simply ignored. This allows the programmer to refactor the case class without breaking compatibility with serialization format - fields may be reordered and removed. New fields may also be added, as long as they have a default value defined.

The way macro materializes the codec may be customized with annotations. All annotations are governed by the same
[annotation processing](Annotations.md) rules.

* Using [`@name`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/name.html) you can change the raw field name used for serialization of each case class field.

  ```scala
  case class Entity(@name("_id") id: String, data: Int)
  ```

  This is useful in particular when you want to refactor your case class and change the name of some field without changing the serialized format (in order to remain backwards compatible). Note that the annotation (like all other annotations used to customize serialization) may also be inherited from implemented/overridden member.

* Using [`@whenAbsent`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/whenAbsent.html) you can provide a fallback value for case class field. This value
  is used during deserialization when the field is missing in the encoded data. Alternatively to using `@whenAbsent`, you can simply provide Scala-level default parameter value and it
  will also be picked up as fallback value for deserialization. However, `@whenAbsent` is better when you want the default value to be used *only* during deserialization, without
  affecting programming interface.

* Using [`@generated`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/generated.html) annotation on some additional members of your case class you can instruct the codec to serialize some additional fields into the resulting format. This can be useful when some case class field has been removed or converted to a regular `val` or `def` but we want the serialized format to be backwards compatible. Sometimes it may also be necessary to generate additional fields for the purpose of database indexing.

  ```scala
  case class Person(name: String, birthYear: Int) {
    @generated def upperName: String = name.toUpper
  }
  ```

  Generated members may also be customized with `@name` annotation. During serialization, generated fields are emitted after all the "regular" fields have been written. Unlike for the regular fields, there is no guarantee about the order of generated fields in the serialized format. During deserialization, generated fields are simply ignored.

* If one of the fields in your case class has a default value (in Scala or with `@whenAbsent`), you might want to not serialize that field if its value is the default one. To instruct the codec to omit default values, [`@transientDefault`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/transientDefault.html) annotation can be used.

  ```scala
  case class Person(name: String, birthYear: Int, @transientDefault planet: String = "Earth")
  ```

  This comes in handy especially when your field might not have a value at all. You can express it using [`Opt`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/misc/Opt.html) which is serialized either as `null` (when empty) or directly as the value inside it (when non-empty). By specifying `Opt.Empty` as default value and applying `@transientDefault` annotation, you can completely avoid emitting the field when there is no value.

  ```scala
  case class Person(name: String, birthYear: Int, @transientDefault maidenName: Opt[String] = Opt.Empty)
  ```

  Note that the absence of a field with default value in the serialized data does not cause any errors during deserialization - if the field is missing, the codec will simply use its default value (it works even without `@transientDefault` annotation).

* If your case class has exactly one field, you might want to avoid it being wrapped in an object and simply serialize the value of that field. This way your class would be a "transparent wrapper" over some type. Wrapping a primitive type into nicely named wrapper is a common technique to increase readability and type safety. In Scala, value classes are often utilized for this purpose.
  If your case class has exactly one field, you can annotate is as [`@transparent`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/transparent.html) and the macro materialized codec will simply serialize the wrapped value.

  ```scala
  @transparent case class DatabaseId(raw: String) extends AnyVal
  ```

#### Safe evolution and refactoring - summary

Following changes can be made to a case class while keeping it backwards compatible with the old format (the old format will successfully deserialize to the new case class):

* renaming the case class or moving to a different package, object, etc. - case class name is not serialized (unless it's a part of a [sealed hierarchiy](#sealed-hierarchies))
* reordering fields
* removing a field
* renaming a field, as long as the old name is provided using `@name` annotation
* adding a field, as long as it has default value defined
* changing the type of a field, as long as the old and new type serialize to the same format
* adding generated fields
* adding implicit parameters (they must be available at the codec materialization site and will be embedded in the codec itself)
* case class may also be safely lifted to a sealed hierarchy (when using `@flatten` and `@defaultCase` - see [sealed hierarchies](#sealed-hierarchies) for details)

### Case class like types

If, for whatever reason, your class can't be a case class, but you still want it to be serialized like a case class would be, you can make it look like a case class. In order to do this, simply provide your own implementations of `apply` and `unapply` methods in the companion object of your trait/class. For case classes, these methods are generated automatically by the compiler.

```scala
class Person(val name: String, val birthYear: Int)
object Person extends HasGenCodec[Person] {
  def apply(name: String, birthYear: Int): Person = new Person(name, birthYear)
  def unapply(person: Person): Option[(String, Int)] = Some((person.name, person.birthYear))
}
```

**NOTE**: if `apply` method takes a repeated (varargs) parameter, then there must be an `unapplySeq` method instead of `unapply` and the repeated parameter should correspond to a `Seq` in the `unapplySeq` return type.

**NOTE**: the `Option` in return type of `unapply`/`unapplySeq` may be replaced with other similar types, e.g. `Opt`, `NOpt`, etc. thanks to the [name based extractors](https://hseeberger.wordpress.com/2013/10/04/name-based-extractors-in-scala-2-11/) feature in Scala. This way you may avoid boxing associated with `Option`s.

You can use all the customization features available for regular case classes - `@name`, `@transientDefault` (applied on `apply` parameters), `@generated`, `@transparent`, etc.

## Singletons

`GenCodec.materialize` macro is also able to generate (trivial) codecs for singletons, i.e. `object`s or types like `this.type`.
Singletons always serialize into empty object (unless `@generated` fields are defined). When using [`SimpleValueOutput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/SimpleValueOutput.html),
empty object is represented by empty `Map`.

```scala
object SomeObject {
  implicit val codec: GenCodec[SomeObject.type] = GenCodec.materialize[SomeObject.type]
}
```

Just like case classes, singletons might define or inherit members annotated as [`@generated`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/generated.html) - their values will be serialized as the only object fields and ignored during deserialization.

Singletons will successfully deserialize from any object, ignoring all its fields.

Singleton codecs may not seem very useful as standalone codes - they're primarily used when the object is a part of a [sealed hierarchy](#sealed-hierarchies).

## Sealed hierarchies

`HasGenCodec` base companion class and all its variants can also be used to derive a `GenCodec` for a sealed trait or
class. There are two possible serialization formats for sealed hierarchies: *nested* (the default one) and *flat*
(enabled using [`@flatten`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/flatten.html)
annotation). The nested format is the default one for historical reasons - it is generally recommended to use the flat
format as it's more robust and customizable. The advantage of nested format is that it does not depend on the order of
object fields.

### Nested format

```scala
sealed trait Timeout
case class FiniteTimeout(seconds: Int) extends Timeout
case object InfiniteTimeout extends Timeout
object Timeout extends HasGenCodec[Timeout]
```

In nested format, values of sealed traits or classes are serialized into objects with just one field. The name of that
field is the name of actual class being serialized. The value of that field will be the serialized class itself, using
its own codec. For example, `FiniteTimeout(60)` would be represented (using [`JsonStringOutput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/json/JsonStringOutput.html))
as `{"FiniteTimeout":{"seconds":60}}`

`GenCodec` for each case class/object may be provided explicitly or left for the macro to materialize. In other words, the `materialize` macro called for sealed trait *will* descend into its subtypes and materialize their codecs recursively. However, it will still *not* descend into any case class fields.

### Flat format

The other format is called "flat" because it does not introduce the intermediate single-field object. It is enabled by annotating your sealed hierarchy root with [`@flatten`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/flatten.html) annotation, e.g.

```scala
@flatten sealed trait Timeout
case class FiniteTimeout(seconds: Int) extends Timeout
case object InfiniteTimeout extends Timeout
object Timeout extends HasGenCodec[Timeout]
```

Instead of creating a single-field object, now the `materialize` macro will assume that every case class/object serializes to an object (e.g. JSON object) and will use this object as a representation of the entire sealed type. In order to differentiate between case classes during deserialization, an additional discriminator field containing class name is added at the beginning of resulting object. For example, `FiniteTimeout(60)` would be represented (using [`JsonStringOutput`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/json/JsonStringOutput.html)) as `{"_case":"FiniteTimeout","seconds":60}`

#### Transparent wrappers in flat sealed hierarchies

It is possible for a case class in a flat sealed hierarchy to be a _transparent wrapper_. To do this, the wrapped
type must have an implicit `ApplyUnapplyCodec` (a subtype of `GenCodec` specific for case classes and case class like types).
The wrapping case itself must have a companion that extends `TransparentWrapperCompanion`.

```scala
case class NumericalData(num: Int)
object NumericalData extends HasApplyUnapplyCodec[NumericalData]
case class TextualData(text: String)
object TextualData extends HasApplyUnapplyCodec[TextualData]

@flat sealed trait DataUnion
case class NumericalCase(data: NumericalData) extends DataUnion
object NumericalCase extends TransparentWrapperCompanion[NumericalData, NumericalCase]
case class TextualCase(data: TextualData) extends DataUnion
object TextualCase extends TransparentWrapperCompanion[TextualData, TextualCase]
object DataUnion extends HasGenCodec[DataUnion]
```

Serializing `NumericalCase(NumericalData(42))` using `JsonStringOutput` would yield
JSON string `{"_case": "NumericalCase", "num": 42}`.

`NumericalData` and `TextualData` would typically be defined in a different file than the sealed hierarchy itself.
That's the typical motivation for using transparent wrappers in a sealed hierarchy - you can't have these types
in the same compilation unit as the sealed trait.

### Customizing sealed hierarchy codecs

Similarly to case classes, sealed hierarchy codecs may be customized with annotations. All annotations are governed by
the same [annotation processing](Annotations.md) rules.

* Using [`@name`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/name.html) you can change the class name saved as marker field name in nested format or as marker field value in flat format.

   ```scala
   sealed trait Tree
   @name("L") case class Leaf(value: Int) extends Tree
   @name("B") case class Branch(left: Tree, right: Tree) extends Tree
   object Tree extends HasGenCodec[Tree]
   ```

* When using flat format, name of the marker field (`_case` by default) may be customized by passing it as an argument to `@flatten` annotation, e.g. `@flatten("type")`.

* When using flat format, one of the case classes may be annotated as [`@defaultCase`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/defaultCase.html). When marker field is missing during deserialization, the codec will assume that it's deserializing the case class annotated as `@defaultCase`. This mechanism is useful to retain backwards compatibility when refactoring a case class into a sealed hierarchy with multiple case classes.

* It's important to remember that deserialization of the flat format relies on the preservation of field order by serialization backend (or random field access):
  In particular, the marker field must be known to the codec before it reads other fields so that it knows which class to create and how to deserialize the rest of the fields.
  There is one escape hatch from this requirement - a field present in one or more of case classes in the sealed hierarchy may be marked as [`@outOfOrder`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/outOfOrder.html). See the documentation of this annotation for more details. The direct motivation for introducing this annotation was to support deserialization of `_id` field in MongoDB documents - the database server always serves documents with `_id` being the very first field.

### Nested vs flat format

Advantages of nested format:
* Codec materialized for sealed hierarchy may reuse already existing codecs for its case classes
* Each case class may serialize to arbitrary representation while flat format requires every case class to serialize to an object
* It does not rely on object field order

Advantages of flat format:
* When some field is present in more than one case class, it may be extracted from serialized form in uniform way, regardless of which case class it comes from. This may greatly simplify querying and indexing databases used to store sealed hierarchies.
* Case class serialized with flat sealed hierarchy codec may be safely deserialized using codec of the case class itself.
* Using `@defaultCase` annotation, a case class may be safely refactored into a sealed hierarchy.

In other words, when the serialized form is opaque and you don't care about it as long as it deserializes properly to the same value then the nested format should be better. If you care about how the serialized form looks like and you want to retain it through refactorings then probably the flat format is easier to maintain.

## Third party classes

When you need to serialize a type that comes from a third party library, you must implement a `GenCodec` for it, put somewhere in your codebase and remember to import it when needed. You must import it because it's not possible to put it in companion object of the type being serialized. However, you can still use all the goodness of macro materialization only if you can make the third party type "look like" a case class by defining a "fake" companion for that type and passing it explicitly to `GenCodec.fromApplyUnapplyProvider`. For example, here's an easy way to make a typical Java bean class serializable with `GenCodec`:

This is the third party Java class:

```java
public class JavaPerson {
    private String name;
    private int birthYear;

    public String getName() { return name; }
    public void setName(String name) { this.name = name; }
    public int getBirthYear() { return birthYear; }
    public void setBirthYear(int birthYear) { this.birthYear = birthYear; }
}
```

This is your Scala code to make it serializable:

```scala
object JavaPersonFakeCompanion {
  def apply(name: String, birthYear: Int): JavaPerson = {
    val result = new JavaPerson
    result.setName(name)
    result.setBirthYear(birthYear)
    result
  }
  def unapply(javaPerson: JavaPerson): Option[(String, Int)] =
    Some((javaPerson.getName, javaPerson.getBirthYear))

  implicit val javaPersonCodec: GenCodec[JavaPerson] =
    GenCodec.fromApplyUnapplyProvider[JavaPerson](JavaPersonFakeCompanion)
}

```

Now, as long as `JavaPersonFakeCompanion.javaPersonCodec` is in scope, `JavaPerson` instances will serialize just as if it was a regular Scala case class. The macro derives serialization format from signatures of `apply` and `unapply` methods and uses them to create and deconstruct `JavaPerson` instances.

### Injecting additional implicits into `GenCodec` materialization

When possible, you should keep all your `GenCodec` instances (and other implicits) in companion objects of your data types.
This way they will be effectively globally visible and won't need to be imported. However, this is not always possible -
a `GenCodec` instance for a third party class is the most common example of such situation.

Of course, you can always import third party implicits explicitly into the scope. However, this is problematic because it
doesn't always work (presumably due to compiler bugs) and IDEs like IntelliJ IDEA tend to recognize such imports as unused.
Because of this, `GenCodec` comes with a handy base companion class, similar to `HasGenCodec` but capable of injecting additional implicits. It is called `HasGenCodecWithDeps`.

```scala
object MyAdditionalImplicits {
  implicit val javaPersonCodec: GenCodec[JavaPerson] = ...
  ...
}

case class UsesJavaPerson(javaPerson: JavaPerson)
object UsesJavaPerson extends HasGenCodecWithDeps[MyAdditionalImplicits.type, JavaPerson]
```

In order to reduce boilerplate, you can make your own base companion class which automatically injects desired implicits:

```scala
abstract class HasCustomizedGenCodec[T](
  implicit macroCodec: MacroInstances[MyAdditionalImplicits.type, () => GenCodec[T]]
) extends HasGenCodecWithDeps[MyAdditionalImplicits.type, T]
```

## [`GenObjectCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenKeyCodec.html)

`GenObjectCodec` is a subinterface of `GenCodec` which directly exposes methods that write to `ObjectOutput` and read
from `ObjectInput`. You can use this typeclass instead of plain `GenCodec` if you want to be sure that some particular
type serializes to an *object*. `GenObjectCodec` instances are provided similarly to `GenCodec` instances - there is
a `GenObjectCodec.materialize` macro and `HasGenObjectCodec` companion base class.

## Summary

### Codec dependencies

The `materialize` macro will only generate [`GenCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html)
implementation for case class or sealed hierarchy if all fields of case classes are already serializable
(i.e. their types have their own [`GenCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html)
instances). For example, the following code will not compile:

```scala
case class Address(city: String, zipcode: String)
case class Person(name: String, address: Address)
object Person {
  implicit val codec: GenCodec[Person] = GenCodec.materialize[Person] // error!
}
```

The `materialize` macro does not descend into case class fields and will therefore refuse to generate codec for `Person`
because it doesn't have a codec for `Address`. This behavior is intentional and serves to avoid making types serializable
by accident. However, there is an alternative macro which *does* descend into dependencies,
[`GenCodec.materializeRecursively`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec$.html#materializeRecursively[T]:com.avsystem.commons.serialization.GenCodec[T]):

```scala
case class Address(city: String, zipcode: String)
case class Person(name: String, address: Address)
object Person {
  implicit val codec: GenCodec[Person] = GenCodec.materializeRecursively[Person]
}
```

`materializeRecursively` will generate a codec for `Address`. However, this codec will be visible only by the `Person` codec.
That means you can now serialize `Person` objects, but you still can't serialize `Address` objects by themselves. Also,
remember that `materializeRecursively` descends into dependencies only when it actually needs to do it, i.e. first it
tries to use any already declared [`GenCodec`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/GenCodec.html).

### Types supported by automatic materialization

`materialize` and `materializeRecursively` macros work for:
* case classes, provided that all field types are serializable
* case class like types, i.e. classes or traits whose companion object contains a pair of matching `apply`/`unapply`
  methods defined like in case class companion, provided that all field types are serializable
* singleton types, e.g. types of `object`s or `this.type`
* sealed traits or sealed abstract classes, provided that `GenCodec` can also be materialized for all non-abstract subtypes (typically case classes). If the nested serialization format is used (i.e. `@flatten` annotation is **not** used) then `GenCodec`s for subtypes may also be declared explicitly and will be reused by sealed trait's codec.

### Recursive types, generic types and GADTs (generalized algebraic data types)

A recursively defined case class:

```scala
case class SimpleTree(children: List[SimpleTree])
object SimpleTree extends HasGenCodec[SimpleTree]
```

A generic (and recursive) data type:

```scala
sealed trait Tree[T]
case class Leaf[T](value: T) extends Tree[T]
case class Branch[T](left: Tree[T], right: Tree[T]) extends Tree[T]
object Tree extends HasPolyGenCodec[Tree]
```

A generalized algebraic data type (also recursive):

```scala
sealed trait Expr[T]
case class StringLiteral(value: String) extends Expr[String]
case class IntLiteral(value: Int) extends Expr[Int]
case object NullLiteral extends Expr[Null]
case class Plus[T](lhs: Expr[T], rhs: Expr[T]) extends Expr[T]
object Expr extends HasGadtCodec[Expr]
```

Note that for generic types we must use `HasPolyGenCodec` or `HasGadtCodec` instead of `HasGenCodec`. It should also be
relatively easy to create custom versions of these base companion classes for whatever combination of type kind, bounds
and implicit dependencies you need to use.

### Customizing annotations

All annotations are governed by [annotation processing](Annotations.md) rules.

* [`@name`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/name.html)
* [`@whenAbsent`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/whenAbsent.html)
* [`@transparent`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/transparent.html)
* [`@transientDefault`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/transientDefault.html)
* [`@flatten`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/flatten.html)
* [`@defaultCase`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/defaultCase.html)
* [`@outOfOrder`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/outOfOrder.html)
* [`@generated`](http://avsystem.github.io/scala-commons/api/com/avsystem/commons/serialization/generated.html)

### Safely introducing changes to serialized classes (retaining backwards compatibility)

When changing definitions of serialized data types, you must ensure that your changes don't break compatibility. `GenCodec` guarantees that some changes are safe
while other are not.

Let's assume a case class with an autogenerated `GenCodec` instance:

```scala
case class Data(int: Int, str: String)
object Data extends HasGenCodec[Data]
```

A summary of safe and unsafe changes to this class definition:

```scala
// OK - renaming a case class is fine because its name is not serialized (but NOT OK if the case class is a part of a sealed hierarchy - see later)
case class RenamedData(num: Int, str: String)

// OK - reordering fields is fine because the codec can read fields in any order
case class Data(str: String, num: Int)

// OK - removing a field is fine because the codec ignores nonexistent fields
case class Data(num: Int)

// NOT OK - adding a field is not fine because the field may be missing in previously serialized data
case class Data(num: Int, str: String, flag: Boolean)

// OK - adding a field with default value is fine because the default value will be used when the field is missing in previously serialized data
case class Data(num: Int, str: String, flag: Boolean = false)
case class Data(num: Int, str: String, @whenAbsent(false) flag: Boolean)

// NOT OK - changing the name of a field is not fine because previously serialized data will contain the old name
case class Data(number: Int, str: String)

// OK - changing the name of a field is ok if the previous name is retained in `@name` annotation
case class Data(@name("num") number: Int, str: String)

// NOT OK - changing the type of a field to a type with different representation is not fine
case class Data(num: String, str: String)

// OK - changing the type of a field to a type with compatible representation is fine (all `Int` values will properly deserialize as `Long`)
case class Data(num: Long, str: String)

// OK - changing the type of a field to a type that is a transparent wrapper of the previous type is fine
case class Amount(value: Int) extends AnyVal
object Amount extends IntWrapperCompanion[Amount]

case class Data(num: Amount, str: String)

// OK - lifting a case class into a sealed hierarchy is fine but only for flat format and when using `@defaultCase` annotation
@flatten sealed trait Info
@defaultCase case class Data(num: Int, str: String) extends Info
case object Empty extends Info
object Info extends HasGenCodec[Info]
```

Now let's assume a sealed hierarchy with an autogenerated `GenCodec` instance:

```scala
sealed trait Info
case class Data(num: Int, str: Int) extends Info
case object Empty extends Info
object Info extends HasGenCodec[Info]
```

Case classes in a sealed hierarchy may be changed in the same way as if they were standalone classes (except for renaming).
An additional summary of safe and unsafe changes to this trait definition:

```scala
// NOT OK - renaming a case class/object in hierarchy is not fine because the class name is used as a discriminator
case class InfoData(num: Int, str: Int) extends Info

// OK - renaming a case class/object in hierarchy is fine when the old name is preserved in `@name` annotation
@name("Data") case class InfoData(num: Int, str: Int) extends Info

// OK - adding more case classes/objects to the hierarchy is fine
...
case class MoreInfo(num: Double, flag: Boolean) extends Info

// NOT OK - changing the sealed hierarchy format between nested and flat is not fine
@flatten sealed trait Info
...
```

Remember that the rules listed above only apply to automatically materialized codecs.
If you implement your codecs manually then you need to ensure compatibility manually as well.

## Performance

There are JMH [benchmarks](https://github.com/AVSystem/scala-commons/blob/master/commons-benchmark/jvm/src/main/scala/com/avsystem/commons/ser/JsonSerializationBenchmark.scala)
implemented for JSON serialization, comparing `GenCodec` with [Circe](https://circe.github.io/circe/) and [uPickle](https://github.com/lihaoyi/upickle).

Example results (higher score is better):

```
[info] Benchmark                                      Mode  Cnt        Score        Error  Units
[info] JsonReadingBenchmark.readCCCirce              thrpt   10   649967.336 ±  12828.283  ops/s
[info] JsonReadingBenchmark.readCCGenCodec           thrpt   10  1050431.352 ±  16007.544  ops/s
[info] JsonReadingBenchmark.readCCUpickle            thrpt   10   698061.199 ±  13618.354  ops/s
[info] JsonReadingBenchmark.readFlatSHGenCodec       thrpt   10   457469.129 ±   7083.221  ops/s
[info] JsonReadingBenchmark.readFoosCirce            thrpt   10     3016.276 ±     39.010  ops/s
[info] JsonReadingBenchmark.readFoosGenCodec         thrpt   10     3098.885 ±     39.858  ops/s
[info] JsonReadingBenchmark.readFoosUpickle          thrpt   10     3083.453 ±     29.865  ops/s
[info] JsonReadingBenchmark.readPrimitivesCirce      thrpt   10  1438760.846 ±  26739.354  ops/s
[info] JsonReadingBenchmark.readPrimitivesGenCodec   thrpt   10  1852565.866 ±  16059.918  ops/s
[info] JsonReadingBenchmark.readPrimitivesUpickle    thrpt   10  1889290.939 ±  24997.719  ops/s
[info] JsonReadingBenchmark.readSHCirce              thrpt   10   278322.679 ±   3304.612  ops/s
[info] JsonReadingBenchmark.readSHGenCodec           thrpt   10   533291.220 ±   4675.452  ops/s
[info] JsonReadingBenchmark.readSHUpickle            thrpt   10   298463.522 ±   3644.145  ops/s
[info] JsonWritingBenchmark.writeCCCirce             thrpt   10   688167.968 ±   8117.737  ops/s
[info] JsonWritingBenchmark.writeCCGenCodec          thrpt   10  1529662.369 ±  26909.344  ops/s
[info] JsonWritingBenchmark.writeCCUpickle           thrpt   10   985345.145 ±  43860.322  ops/s
[info] JsonWritingBenchmark.writeFlatSHGenCodec      thrpt   10   810840.889 ±  52376.100  ops/s
[info] JsonWritingBenchmark.writeFoosCirce           thrpt   10     2896.501 ±    327.638  ops/s
[info] JsonWritingBenchmark.writeFoosGenCodec        thrpt   10     4520.711 ±    182.557  ops/s
[info] JsonWritingBenchmark.writeFoosUpickle         thrpt   10     3096.716 ±    199.797  ops/s
[info] JsonWritingBenchmark.writePrimitivesCirce     thrpt   10  1263376.319 ±  49751.056  ops/s
[info] JsonWritingBenchmark.writePrimitivesGenCodec  thrpt   10  2904553.884 ± 207071.275  ops/s
[info] JsonWritingBenchmark.writePrimitivesUpickle   thrpt   10  2142443.367 ±  85212.161  ops/s
[info] JsonWritingBenchmark.writeSHCirce             thrpt   10   190421.595 ±   4475.344  ops/s
[info] JsonWritingBenchmark.writeSHGenCodec          thrpt   10   912956.930 ±  17628.358  ops/s
[info] JsonWritingBenchmark.writeSHUpickle           thrpt   10   239291.635 ±   8626.623  ops/s
```


text:
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:paddingBottom="@dimen/activity_vertical_margin"
    android:paddingLeft="@dimen/activity_horizontal_margin"
    android:paddingRight="@dimen/activity_horizontal_margin"
    android:paddingTop="@dimen/activity_vertical_margin"
    tools:context=".MainActivity"
    android:background="#444">

    <ImageView
        android:id="@+id/passengerImageView"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignParentLeft="true"
        android:layout_alignParentTop="true"
        android:src="@drawable/app_logo" />

    <TextView
        android:id="@+id/text1"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignBottom="@+id/passengerImageView"
        android:layout_toRightOf="@+id/passengerImageView"
        android:text="TransitWand"
        android:textColor="#aaa"
        android:textSize="30sp" />

    <ImageButton
        android:id="@+id/WandButton"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_below="@+id/passengerImageView"
        android:layout_marginRight="55dp"
        android:layout_marginTop="75dp"
        android:layout_toLeftOf="@+id/ReviewButton"
        android:background="@null"
        android:src="@drawable/wand_button" />

    <ImageButton
        android:id="@+id/UploadButton"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignTop="@+id/ReviewButton"
        android:layout_marginLeft="55dp"
        android:layout_toRightOf="@+id/ReviewButton"
        android:background="@null"
        android:src="@drawable/upload_button" />

    <TextView
        android:id="@+id/descriptionText"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignLeft="@+id/WandButton"
        android:layout_below="@+id/WandButton"
        android:text="Capture"
        android:textColor="#aaa" />

    <TextView
        android:id="@+id/totalPasssengerCount"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignBaseline="@+id/descriptionText"
        android:layout_alignBottom="@+id/descriptionText"
        android:layout_centerHorizontal="true"
        android:text="Review"
        android:textColor="#aaa" />

    <TextView
        android:id="@+id/TextView02"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignBaseline="@+id/totalPasssengerCount"
        android:layout_alignBottom="@+id/totalPasssengerCount"
        android:layout_alignLeft="@+id/UploadButton"
        android:text="Upload"
        android:textColor="#aaa" />

    <TextView
        android:id="@+id/UnitIdText"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignParentBottom="true"
        android:layout_alignParentRight="true"
        android:text=""
        android:textColor="#aaa"
        android:textSize="30sp" />

    <ImageButton
        android:id="@+id/ReviewButton"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignTop="@+id/WandButton"
        android:layout_centerHorizontal="true"
        android:background="@null"
        android:src="@drawable/review_button" />

    <TextView
        android:id="@+id/UserNameText"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_above="@+id/UnitIdText"
        android:layout_alignRight="@+id/UnitIdText"
        android:text=""
        android:textColor="#aaa" />

</RelativeLayout>


text:
'use strict';

const {
  __SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED,
} = require('ReactDOM-fb');

module.exports =
  __SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED.getVendorPrefixedEventName;


text:
// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

using Google.Cloud.ClientTesting;
using Google.Cloud.Diagnostics.Common;
using Google.Cloud.Diagnostics.Common.IntegrationTests;
using Google.Protobuf.WellKnownTypes;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.TestHost;
using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using Xunit;

namespace Google.Cloud.Diagnostics.AspNetCore.Snippets
{
    [SnippetOutputCollector]
    public class DiagnosticsSnippetsTests : IDisposable
    {
        private static readonly ErrorEventEntryPolling s_errorPolling = new ErrorEventEntryPolling();
        private static readonly TraceEntryPolling s_tracePolling = new TraceEntryPolling();

        private const string Service = EntryData.Service;
        private const string Version = EntryData.Version;
        private static readonly string ProjectId = TestEnvironment.GetTestProjectId();

        private readonly string _testId;

        private readonly TestServer _server;
        private readonly HttpClient _client;

        private readonly DateTime _startTime;

        public DiagnosticsSnippetsTests()
        {
            // Sample: UseGoogleDiagnostics
            var webHostBuilder = new WebHostBuilder()
                // Replace ProjectId with your Google Cloud Project ID.
                // Replace Service with a name or identifier for the service.
                // Replace Version with a version for the service.
                .UseGoogleDiagnostics(ProjectId, Service, Version)
                .UseStartup<Startup>();
            // End sample

            _server = new TestServer(webHostBuilder);
            _client = _server.CreateClient();

            _testId = IdGenerator.FromDateTime();
            _startTime = DateTime.UtcNow;
        }

        /// <summary>
        /// When activating diagnostics the three aspects of it should work, Trace, Loggin and Error Logging.
        /// This tests that Error Logging works.
        /// </summary>
        [Fact]
        public async Task Logs_UnhandledException()
        {
            await Assert.ThrowsAsync<Exception>(() => _client.GetAsync($"/ErrorLoggingSamples/{nameof(ErrorLoggingSamplesController.ThrowsException)}/{_testId}"));

            var errorEvent = s_errorPolling.GetEvents(_startTime, _testId, 1).Single();

            ErrorEventEntryVerifiers.VerifyFullErrorEventLogged(errorEvent, _testId, nameof(ErrorLoggingSamplesController.ThrowsException), (int)HttpStatusCode.OK);
        }

        /// <summary>
        /// When activating diagnostics the three aspects of it should work, Trace, Loggin and Error Logging.
        /// This tests that Trace works.
        /// </summary>
        [Fact]
        public async Task Traces_Block()
        {
            var uri = $"/TraceSamples/{nameof(TraceSamplesController.TraceHelloWorld)}/{_testId}";
            var response = await _client.GetAsync(uri);

            var trace = s_tracePolling.GetTrace(uri, Timestamp.FromDateTime(_startTime));

            TraceEntryVerifiers.AssertParentChildSpan(trace, uri, _testId);
            TraceEntryVerifiers.AssertSpanLabelsContains(
                trace.Spans.First(s => s.Name == uri), TraceEntryData.HttpGetSuccessLabels);
            Assert.False(response.Headers.Contains(TraceHeaderContext.TraceHeader));
        }

        /// <summary>
        /// When activating diagnostics the three aspects of it should work, Trace, Loggin and Error Logging.
        /// This tests that Logging works.
        /// </summary>
        [Fact]
        public async Task Logs_Information()
        {
            await _client.GetAsync($"/LoggingSamples/LogInformation/{_testId}");

            LoggingSnippetsTests.PollAndVerifyLog(_startTime, _testId);
        }

        public void Dispose()
        {
            _client.Dispose();
            _server.Dispose();
        }
    }
}


text:
package io.indexr.segment.rc;

import java.util.Collection;
import java.util.Collections;

public interface LogicalOperator extends RCOperator {
    @Override
    default Collection<Attr> attr() {
        return Collections.emptySet();
    }

    @Override
    default public boolean isAccurate() {
        boolean ok = true;
        for (RCOperator op : children()) {
            ok &= op.isAccurate();
        }
        return ok;
    }
}


text:
from functools import reduce
from operator import mul

def factorial(n):
    return reduce(mul, range(1, n + 1), 1)

f5 = factorial(5)  # f5 = 120
print(f5)
print([factorial(k) for k in range(10)])


text:
using namespace Tempest;

MemReader::MemReader(const char *vec, size_t sz):vec(reinterpret_cast<const uint8_t*>(vec)), sz(sz){
  }

MemReader::MemReader(const uint8_t *vec, size_t sz):vec(vec), sz(sz) {
  }

MemReader::MemReader(const std::vector<char> &vec):MemReader(vec.data(),vec.size()){
  }

MemReader::MemReader(const std::vector<uint8_t> &vec):MemReader(vec.data(),vec.size()){
  }

size_t MemReader::read(void *dest, size_t count) {
  size_t c = std::min(count, sz-pos);

  std::memcpy(dest, vec+pos, c);
  pos+=c;

  return c;
  }

size_t MemReader::size() const {
  return sz;
  }

uint8_t MemReader::peek() {
  if(pos==sz)
    return 0;
  return vec[pos];
  }

size_t MemReader::seek(size_t advance) {
  size_t c = std::min(advance, sz-pos);
  pos += c;
  return c;
  }

size_t MemReader::unget(size_t advance) {
  if(advance>pos){
    size_t ret = advance-(advance-pos);
    pos = 0;
    return ret;
    }
  pos-=advance;
  return advance;
  }


text:
package org.bouncycastle.oer.its.ieee1609dot2.basetypes;

import org.bouncycastle.asn1.ASN1Object;
import org.bouncycastle.asn1.ASN1OctetString;
import org.bouncycastle.asn1.ASN1Primitive;
import org.bouncycastle.asn1.ASN1Sequence;
import org.bouncycastle.asn1.DEROctetString;
import org.bouncycastle.oer.its.ItsUtils;
import org.bouncycastle.util.Arrays;

/**
 * EcdsaP384Signature ::= SEQUENCE {
 * rSig  EccP384CurvePoint,
 * sSig  OCTET STRING (SIZE (48))
 * }
 */
public class EcdsaP384Signature
    extends ASN1Object
{
    private final EccP384CurvePoint rSig;
    private final ASN1OctetString sSig;

    public EcdsaP384Signature(EccP384CurvePoint rSig, ASN1OctetString sSig)
    {
        this.rSig = rSig;
        this.sSig = sSig;
    }

    private EcdsaP384Signature(ASN1Sequence sequence)
    {
        if (sequence.size() != 2)
        {
            throw new IllegalArgumentException("expected sequence size of 2");
        }
        rSig = EccP384CurvePoint.getInstance(sequence.getObjectAt(0));
        sSig = ASN1OctetString.getInstance(sequence.getObjectAt(1));
    }

    public static EcdsaP384Signature getInstance(Object object)
    {

        if (object instanceof EcdsaP384Signature)
        {
            return (EcdsaP384Signature)object;
        }

        if (object != null)
        {
            return new EcdsaP384Signature(ASN1Sequence.getInstance(object));
        }

        return null;
    }

    public EccP384CurvePoint getRSig()
    {
        return rSig;
    }

    public ASN1OctetString getSSig()
    {
        return sSig;
    }

    public ASN1Primitive toASN1Primitive()
    {
        return ItsUtils.toSequence(rSig, sSig);
    }

    public static Builder builder()
    {
        return new Builder();
    }

    public static class Builder
    {

        private EccP384CurvePoint rSig;
        private ASN1OctetString sSig;

        public Builder setRSig(EccP384CurvePoint rSig)
        {
            this.rSig = rSig;
            return this;
        }

        public Builder setSSig(ASN1OctetString sSig)
        {
            this.sSig = sSig;
            return this;
        }

        public Builder setSSig(byte[] sSig)
        {
            return setSSig(new DEROctetString(Arrays.clone(sSig)));
        }

        public EcdsaP384Signature createEcdsaP384Signature()
        {
            return new EcdsaP384Signature(rSig, sSig);
        }
    }
}


text:
mkdir build
cd build

cmake -DCMAKE_TOOLCHAIN_FILE=/usr/emsdk_portable/emscripten/tag-1.36.1/cmake/Modules/Platform/Emscripten.cmake ..
make

find .


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using DotNetNuke.Common.Utilities;

namespace DotNetNuke.Common.Lists
{
	[Serializable]
	public class CachedCountryList : Dictionary<string, CachedCountryList.Country>
	{

		#region " Structures "

		[Serializable]
		public struct Country
		{
			public int Id;
			public string Name;
			public string Code;
			public string FullName;
			public string NormalizedFullName;
		}

		#endregion

		#region " Constructors "
		public CachedCountryList(string locale)
			: base()
		{

			foreach (ListEntryInfo li in (new ListController()).GetListEntryInfoItems("Country"))
			{
				string text = li.Text;
				Country c = new Country
				{
					Id = li.EntryID,
					Code = li.Value,
					FullName = string.Format("{0} ({1})", text, li.Value),
					Name = text
				};
				c.NormalizedFullName = c.FullName.NormalizeString();
				Add(li.Value, c);
			}

		}
		#endregion

		#region " Static Methods "
		public static CachedCountryList GetCountryList(string locale)
		{

			CachedCountryList res = null;
			try
			{
				res = (CachedCountryList)DotNetNuke.Common.Utilities.DataCache.GetCache(CacheKey(locale));
			}
			catch (Exception ex)
			{
			}
			if (res == null)
			{
				res = new CachedCountryList(locale);
				DotNetNuke.Common.Utilities.DataCache.SetCache(CacheKey(locale), res);
			}
			return res;

		}

		public static string CacheKey(string locale)
		{
			return string.Format("CountryList:{0}", locale);
		}
		#endregion

	}
}


text:
<!DOCTYPE html>

	<meta charset="UTF-8" />
	<title>AjLisp Test Suite</title>
	<link rel="stylesheet" href="qunit.css" type="text/css" media="screen">

ajlisp Test Suite

test markup

(define mapfirst (fn lst)
  (if (nilp lst)
    nil
    (cons
      (fn (first lst))
      (mapfirst fn (rest lst))
    )
  )
)

(define append (x y) (if (nilp x) y (cons (first x) (append (rest x) y))))

(define mapcond (fn lst)
	(if (nilp lst)
		nil
		(if (fn (first lst))
			(cons
				(first lst)
				(mapcond fn (rest lst))
			)
			(mapcond fn (rest lst))
		)
	)
)

(define definem (mlambda x (list 'define (first x) (cons 'mlambda (rest x)))))

(definem cond lst (if (nilp lst) nil (list 'if (first (first lst)) (cons 'do (rest (first lst))) (cons 'cond (rest lst)))))

(define atomp (x) (cond ((nilp x) true) ((listp x) false) (true true)))

(definem and lst (if (nilp lst) true (list 'cond (list (first lst) (cons 'and (rest lst))) '(true false))))

(definem backquote (lst) (cond
                ((nilp lst) nil)
                ((atomp lst) (list 'quote lst))
                ((equalp (first lst) 'unquote) (first (rest lst)))
                ((and (listp (first lst)) (equalp (first (first lst)) 'unquote-slice)) (list 'append (first (rest (first lst))) (list 'backquote (rest lst))))
                (true (list 'cons (list 'backquote (first lst)) (list 'backquote (rest lst))))
                ))


text:
using System;
using Microsoft.VisualStudio.Shell.Interop;

namespace Microsoft.VisualStudio.LanguageServices.CSharp.ProjectSystemShim
{
    // The native project system requires that project sites implement IVsEditorFactoryNotify, and
    // the project system stores an internal list of the interface pointers to them. The old editor
    // factory would then call each implementation from it's own IVsEditorFactoryNotify methods.
    // Since we now supply our own editor factory that doesn't do this, these methods will never be
    // called. Still, we must implement the interface or we'll never load at all.
    internal partial class CSharpProjectShim : IVsEditorFactoryNotify
    {
        public int NotifyDependentItemSaved(IVsHierarchy hier, uint itemidParent, string documentParentMoniker, uint itemidDpendent, string documentDependentMoniker)
        {
            throw new NotSupportedException();
        }

        public int NotifyItemAdded(uint grfEFN, IVsHierarchy hier, uint itemid, string documentId)
        {
            throw new NotSupportedException();
        }

        public int NotifyItemRenamed(IVsHierarchy hier, uint itemid, string documentOldMoniker, string documentNewMoniker)
        {
            throw new NotSupportedException();
        }
    }
}


text:
using Lucene.Net.Expressions.JS;
using Lucene.Net.Search;
using Lucene.Net.Util;
using NUnit.Framework;
using System;
using Assert = Lucene.Net.TestFramework.Assert;

namespace Lucene.Net.Expressions
{

    /// <summary>Tests validation of bindings</summary>
    public class TestExpressionValidation : LuceneTestCase
    {
        [Test]
        public virtual void TestValidExternals()
        {
            SimpleBindings bindings = new SimpleBindings();
            bindings.Add(new SortField("valid0", SortFieldType.INT32));
            bindings.Add(new SortField("valid1", SortFieldType.INT32));
            bindings.Add(new SortField("valid2", SortFieldType.INT32));
            bindings.Add(new SortField("_score", SortFieldType.SCORE));
            bindings.Add("valide0", JavascriptCompiler.Compile("valid0 - valid1 + valid2 + _score"
                ));
            bindings.Validate();
            bindings.Add("valide1", JavascriptCompiler.Compile("valide0 + valid0"));
            bindings.Validate();
            bindings.Add("valide2", JavascriptCompiler.Compile("valide0 * valide1"));
            bindings.Validate();
        }

        [Test]
        public virtual void TestInvalidExternal()
        {
            SimpleBindings bindings = new SimpleBindings();
            bindings.Add(new SortField("valid", SortFieldType.INT32));
            bindings.Add("invalid", JavascriptCompiler.Compile("badreference"));
            try
            {
                bindings.Validate();
                Assert.Fail("didn't get expected exception");
            }
            catch (ArgumentException expected)
            {
                Assert.IsTrue(expected.Message.Contains("Invalid reference"));
            }
        }

        [Test]
        public virtual void TestInvalidExternal2()
        {
            SimpleBindings bindings = new SimpleBindings();
            bindings.Add(new SortField("valid", SortFieldType.INT32));
            bindings.Add("invalid", JavascriptCompiler.Compile("valid + badreference"));
            try
            {
                bindings.Validate();
                Assert.Fail("didn't get expected exception");
            }
            catch (ArgumentException expected)
            {
                Assert.IsTrue(expected.Message.Contains("Invalid reference"));
            }
        }

        [Test,Ignore("StackOverflowException can't be caught in .NET")]
        public virtual void TestSelfRecursion()
        {
            SimpleBindings bindings = new SimpleBindings();
            bindings.Add("cycle0", JavascriptCompiler.Compile("cycle0"));
            try
            {
                bindings.Validate();
                Assert.Fail("didn't get expected exception");
            }
            catch (ArgumentException expected)
            {
                Assert.IsTrue(expected.Message.Contains("Cycle detected"));
            }
        }

        [Test, Ignore("StackOverflowException can't be caught in .NET")]
        public virtual void TestCoRecursion()
        {
            SimpleBindings bindings = new SimpleBindings();
            bindings.Add("cycle0", JavascriptCompiler.Compile("cycle1"));
            bindings.Add("cycle1", JavascriptCompiler.Compile("cycle0"));
            try
            {
                bindings.Validate();
                Assert.Fail("didn't get expected exception");
            }
            catch (ArgumentException expected)
            {
                Assert.IsTrue(expected.Message.Contains("Cycle detected"));
            }
        }

        [Test, Ignore("StackOverflowException can't be caught in .NET")]
        public virtual void TestCoRecursion2()
        {
            SimpleBindings bindings = new SimpleBindings();
            bindings.Add("cycle0", JavascriptCompiler.Compile("cycle1"));
            bindings.Add("cycle1", JavascriptCompiler.Compile("cycle2"));
            bindings.Add("cycle2", JavascriptCompiler.Compile("cycle0"));
            try
            {
                bindings.Validate();
                Assert.Fail("didn't get expected exception");
            }
            catch (ArgumentException expected)
            {
                Assert.IsTrue(expected.Message.Contains("Cycle detected"));
            }
        }

        [Test, Ignore("StackOverflowException can't be caught in .NET")]
        public virtual void TestCoRecursion3()
        {
            SimpleBindings bindings = new SimpleBindings();
            bindings.Add("cycle0", JavascriptCompiler.Compile("100"));
            bindings.Add("cycle1", JavascriptCompiler.Compile("cycle0 + cycle2"));
            bindings.Add("cycle2", JavascriptCompiler.Compile("cycle0 + cycle1"));
            try
            {
                bindings.Validate();
                Assert.Fail("didn't get expected exception");
            }
            catch (ArgumentException expected)
            {
                Assert.IsTrue(expected.Message.Contains("Cycle detected"));
            }
        }

        [Test, Ignore("StackOverflowException can't be caught in .NET")]
        public virtual void TestCoRecursion4()
        {
            SimpleBindings bindings = new SimpleBindings();
            bindings.Add("cycle0", JavascriptCompiler.Compile("100"));
            bindings.Add("cycle1", JavascriptCompiler.Compile("100"));
            bindings.Add("cycle2", JavascriptCompiler.Compile("cycle1 + cycle0 + cycle3"));
            bindings.Add("cycle3", JavascriptCompiler.Compile("cycle0 + cycle1 + cycle2"));
            try
            {
                bindings.Validate();
                Assert.Fail("didn't get expected exception");
            }
            catch (ArgumentException expected)
            {
                Assert.IsTrue(expected.Message.Contains("Cycle detected"));
            }
        }
    }
}


text:
=================================
 Welcome to Bosun Policy Manager
=================================

Introduction
============

Bosun is the reference implementation (GEri) of FIWARE Policy Manager GE, and
its component Cloto provides a REST API to create rules associated to servers,
subscribe to Context Broker to get information about resources consumption of
those servers, and launch actions described in rules when conditions are met.

Policy Manager provides the basic management of cloud resources based on rules,
as well as management of the corresponding resources within the FIWARE Cloud
instance like actions based on physical monitoring or infrastructure, security
monitoring of resources and services or whatever that could be defined by facts,
actions and rules. Policy Manager is a easy rule engine designed to be used in
the OpenStack ecosystem and, of course, inside the FIWARE Cloud.

IMPORTANT NOTE: This GE reference implementation product is only of interest
to potential FIWARE instance providers and therefore has been used to build
the basic FIWARE platform core infrastructure of FIWARE Lab. If you are an
application developer, you don't need to create a complete FIWARE instance
locally in order to start building applications based on FIWARE. You may rely
on instances of FIWARE GEris linked to the Data/Media Context Management, the
IoT Services Enablement and the Advanced Web-based User Interface chapters, or
some GEris of the Applications/Services Ecosystem and Delivery Framework chapter
(WireCloud) as well as the Security chapter (Access Control). Those instances
are either global instances or instances you can create on FIWARE Lab, but also
instances you may create by downloading, installing and configuring the
corresponding software in your own premises.

Bosun Policy Manager REST API source code can be found here__.

__ `FIWARE Cloto GitHub Repository`_

Documentation
=============

GitHub's README__ provides a good documentation summary, and the following
cover more advanced topics:

__ `FIWARE Cloto GitHub README`_

.. title:: Home

.. toctree::
   :maxdepth: 1

   user_guide
   admin_guide
   architecture
   open_spec

.. REFERENCES

.. _FIWARE Cloto GitHub Repository: https://github.com/telefonicaid/fiware-cloto.git
.. _FIWARE Cloto GitHub README: https://github.com/telefonicaid/fiware-cloto/blob/master/README.rst


text:
TmsFaqBundle API: [PATCH] Question yep or nope
==============================================

Update partially Question (field yep or nope)

## General
|             | Values
|-------------|-------
| **Method**  | GET
| **Path**    | /questions/{id}/yepnope/{value}.{_format}
| **Formats** | json, xml
| **Secured** | false

## HTTP Request parameters
No request parameters

## Parameter path
| Name   | Optional | Default | Requirements | Description
|--------|----------|---------|--------------|------------
| id     | false    |         | \d+          | Question id
| value  | false    |         | yep or nope  | CountYep or countNope value

## HTTP Response codes
| Code | Description
|------|------------
| 200  | Ok
| 404  | Not found
| 500  | Server error