text:
#
# spgen_test.py
#
# Copyright (c) 2013 Luis Garcia.
# This source file is subject to terms of the MIT License. (See accompanying file LICENSE)
#

import unittest
from spgen_parser import *
from spgen_processor import *

class TestParser(unittest.TestCase):
	def test_eof_detection_1(self):
		result = Parser().try_eof(SourceIterator(''))
		self.assertEqual(result, True)

	def test_eof_detection_2(self):
		result = Parser().try_eof(SourceIterator('    \n\r   \t  '))
		self.assertEqual(result, True)

	def test_eof_detection_3(self):
		result = Parser().try_eof(SourceIterator('   as'))
		self.assertEqual(result, False)

	def test_comment_detection_1(self):
		result = Parser().try_comment(SourceIterator('    // adasd\n'))
		self.assertEqual(result, True)

	def test_comment_detection_2(self):
		result = Parser().try_comment(SourceIterator('    //adasd'))
		self.assertEqual(result, True)

	def test_comment_detection_3(self):
		result = Parser().try_comment(SourceIterator('//'))
		self.assertEqual(result, True)

	def test_comment_detection_4(self):
		result = Parser().try_comment(SourceIterator('/'))
		self.assertEqual(result, False)

	def test_comment_detection_5(self):
		result = Parser().try_comment(SourceIterator(' asdasd // asw \n'))
		self.assertEqual(result, False)

	def test_expect_token_1(self):
		result = Parser().expect_token(SourceIterator(' = '), '=', alphanumeric_token = False)
		self.assertEqual(result, True)

	def test_expect_token_2(self):
		result = Parser().expect_token(SourceIterator('asd'), 'asd', alphanumeric_token = True)
		self.assertEqual(result, True)

	def test_expect_token_3(self):
		result = Parser().expect_token(SourceIterator('   asd  '), 'asd', alphanumeric_token = True)
		self.assertEqual(result, True)

	def test_expect_token_4(self):
		result = Parser().expect_token(SourceIterator('asdr'), 'asd', alphanumeric_token = True)
		self.assertEqual(result, False)

	def test_expect_token_5(self):
		result = Parser().expect_token(SourceIterator('   asrd'), 'asd', alphanumeric_token = True)
		self.assertEqual(result, False)

	def test_expect_token_6(self):
		result = Parser().expect_token(SourceIterator('|=  '), '=', alphanumeric_token = False)
		self.assertEqual(result, False)

	def test_expect_token_7(self):
		result = Parser().expect_token(SourceIterator('asdr'), 'asd', alphanumeric_token = False)
		self.assertEqual(result, True)

	def test_expect_identifier_1(self):
		result = Parser().expect_identifier(SourceIterator(' Aasd32ds asd'))
		self.assertEqual(result, 'Aasd32ds')

	def test_expect_identifier_2(self):
		result = Parser().expect_identifier(SourceIterator(' 32'))
		self.assertEqual(result, None)

	def test_expect_identifier_3(self):
		result = Parser().expect_identifier(SourceIterator('!=asd'))
		self.assertEqual(result, None)

	def test_expect_identifier_4(self):
		result = Parser().expect_identifier(SourceIterator(' '))
		self.assertEqual(result, None)

	def test_expect_identifier_5(self):
		result = Parser().expect_identifier(SourceIterator(' A'))
		self.assertEqual(result, 'A')

	def test_expect_string_1(self):
		result = Parser().expect_string(SourceIterator(' Aasd32ds asd'))
		self.assertEqual(result, None)

	def test_expect_string_2(self):
		result = Parser().expect_string(SourceIterator('*asd \'fail\''))
		self.assertEqual(result, None)

	def test_expect_string_3(self):
		result = Parser().expect_string(SourceIterator(' \'asd\''))
		self.assertEqual(result, 'asd')

	def test_expect_string_4(self):
		result = Parser().expect_string(SourceIterator(''))
		self.assertEqual(result, None)

	def test_property_detection_1(self):
		result, property_info = Parser().try_property(SourceIterator('property abc = \'value\';'))
		self.assertEqual(result, True)
		self.assertEqual(property_info, PropertyInfo('abc', 'value'))

	def test_property_detection_2(self):
		result, property_info = Parser().try_property(SourceIterator('property abcds23 = \'\';'))
		self.assertEqual(result, True)
		self.assertEqual(property_info, PropertyInfo('abcds23', ''))

	def test_property_detection_3(self):
		result, property_info = Parser().try_property(SourceIterator('prop abcds23 = \'\';'))
		self.assertEqual(result, False)

	def test_property_detection_4(self):
		result, property_info = Parser().try_property(SourceIterator('property abcds23 = ;'))
		self.assertEqual(result, False)

	def test_property_detection_5(self):
		result, property_info = Parser().try_property(SourceIterator('property abcds23 = \'asd\''))
		self.assertEqual(result, False)

	def test_property_detection_6(self):
		result, property_info = Parser().try_property(SourceIterator('property abcds23 = asd;'))
		self.assertEqual(result, False)

	def test_property_detection_7(self):
		result, property_info = Parser().try_property(SourceIterator('property = asd;'))
		self.assertEqual(result, False)

	def test_property_detection_8(self):
		result, property_info = Parser().try_property(SourceIterator('abcds23 = asd;'))
		self.assertEqual(result, False)

	def test_property_detection_9(self):
		result, property_info = Parser().try_property(SourceIterator('=;'))
		self.assertEqual(result, False)

	def test_property_detection_10(self):
		result, property_info = Parser().try_property(SourceIterator(''))
		self.assertEqual(result, False)

	def test_token_detection_1(self):
		result, token_info = Parser().try_token(SourceIterator('token var : \'var\';'))
		self.assertEqual(result, True)
		self.assertEqual(token_info, TokenInfo('var', GrammarConstant('var')))

	def test_token_detection_2(self):
		result, token_info = Parser().try_token(SourceIterator('token rule : anotherRule zeroOrMany* ;'))
		self.assertEqual(result, True)
		self.assertEqual(token_info, TokenInfo('rule',
			GrammarExpressionList([
				GrammarReference('anotherRule'),
				GrammarZeroOrMany(GrammarReference('zeroOrMany'))
			])))

	def test_token_detection_3(self):
		result, token_info = Parser().try_token(SourceIterator('token var : a (;'))
		self.assertEqual(result, False)

	def test_token_detection_4(self):
		result, token_info = Parser().try_token(SourceIterator('token var : *;'))
		self.assertEqual(result, False)

	def test_token_detection_5(self):
		result, token_info = Parser().try_token(SourceIterator('token 4 : a (;'))
		self.assertEqual(result, False)

	def test_token_detection_6(self):
		result, token_info = Parser().try_token(SourceIterator('token : a (;'))
		self.assertEqual(result, False)

	def test_token_detection_7(self):
		result, token_info = Parser().try_token(SourceIterator('tvar : a (;'))
		self.assertEqual(result, False)

	def test_token_detection_8(self):
		result, token_info = Parser().try_token(SourceIterator('token var : ;'))
		self.assertEqual(result, False)

	def test_token_detection_9(self):
		result, token_info = Parser().try_token(SourceIterator('token var ;'))
		self.assertEqual(result, False)

	def test_token_detection_10(self):
		result, token_info = Parser().try_token(SourceIterator(';'))
		self.assertEqual(result, False)

	def test_token_detection_11(self):
		result, token_info = Parser().try_token(SourceIterator(''))
		self.assertEqual(result, False)

	def test_token_detection_12(self):
		result, token_info = Parser().try_token(SourceIterator('token á家ل : \'a\' ;'))
		self.assertEqual(result, True)

	def test_token_detection_13(self):
		result, token_info = Parser().try_token(SourceIterator('token t : a | b ;'))
		self.assertEqual(result, True)

	def test_token_detection_14(self):
		result, token_info = Parser().try_token(SourceIterator('token t : a | b | c ;'))
		self.assertEqual(result, True)

	def test_token_detection_15(self):
		result, token_info = Parser().try_token(SourceIterator('token t : a | (b* | c ;'))
		self.assertEqual(result, False)

	def test_token_detection_16(self):
		result, token_info = Parser().try_token(SourceIterator('token t : a | b | ;'))
		self.assertEqual(result, False)

	def test_token_detection_17(self):
		result, token_info = Parser().try_token(SourceIterator('token t : ()* | b ;'))
		self.assertEqual(result, False)

	def test_fragment_detection_1(self):
		result, fragment_info = Parser().try_fragment(SourceIterator('fragment var : \'var\';'))
		self.assertEqual(result, True)
		self.assertEqual(fragment_info, FragmentInfo('var', GrammarConstant('var')))

	def test_fragment_detection_2(self):
		result, fragment_info = Parser().try_fragment(SourceIterator('fragment rule : anotherRule zeroOrMany* ;'))
		self.assertEqual(result, True)
		self.assertEqual(fragment_info, FragmentInfo('rule',
			GrammarExpressionList([
				GrammarReference('anotherRule'),
				GrammarZeroOrMany(GrammarReference('zeroOrMany'))
			])))

	def test_fragment_detection_3(self):
		result, fragment_info = Parser().try_fragment(SourceIterator('fragment var : a (;'))
		self.assertEqual(result, False)

	def test_fragment_detection_4(self):
		result, fragment_info = Parser().try_fragment(SourceIterator('fragment var : *;'))
		self.assertEqual(result, False)

	def test_fragment_detection_5(self):
		result, fragment_info = Parser().try_fragment(SourceIterator('fragment 4 : a (;'))
		self.assertEqual(result, False)

	def test_fragment_detection_6(self):
		result, fragment_info = Parser().try_fragment(SourceIterator('fragment : a (;'))
		self.assertEqual(result, False)

	def test_fragment_detection_7(self):
		result, fragment_info = Parser().try_fragment(SourceIterator('tvar : a (;'))
		self.assertEqual(result, False)

	def test_fragment_detection_8(self):
		result, fragment_info = Parser().try_fragment(SourceIterator('fragment var : ;'))
		self.assertEqual(result, False)

	def test_fragment_detection_9(self):
		result, fragment_info = Parser().try_fragment(SourceIterator('fragment var ;'))
		self.assertEqual(result, False)

	def test_fragment_detection_10(self):
		result, fragment_info = Parser().try_fragment(SourceIterator(';'))
		self.assertEqual(result, False)

	def test_fragment_detection_11(self):
		result, fragment_info = Parser().try_fragment(SourceIterator(''))
		self.assertEqual(result, False)

	def test_free_context_1(self):
		text = """ // A comment
		           property propName     = 'propValue';
		           token tokenName       : 'constant'* andReference;
		           fragment fragmentName : 'anotherConstant' ; """

		context = Context()
		Parser().free_context(SourceIterator(text), context)

		expected = Context()
		expected.properties['propName'] = 'propValue'
		expected.rules['tokenName'] = RuleInfo(
			'tokenName', RuleTypes.TOKEN,
				GrammarExpressionList([
					GrammarZeroOrMany(GrammarConstant('constant')),
					GrammarReference('andReference')]))
		expected.rules['fragmentName'] = RuleInfo(
			'fragmentName', RuleTypes.FRAGMENT, GrammarConstant('anotherConstant'))

		self.assertEqual(context, expected)

class TestTokenRuleGrammarParser(unittest.TestCase):
	def test_expect_token_grammar_constant_1(self):
		result = Parser().expect_token_grammar(SourceIterator('\'var\''))
		self.assertEqual(result, GrammarConstant('var'))

	def test_expect_token_grammar_reference_1(self):
		result = Parser().expect_token_grammar(SourceIterator('tokenId'))
		self.assertEqual(result, GrammarReference('tokenId'))

	def test_expect_token_grammar_list_1(self):
		result = Parser().expect_token_grammar(SourceIterator('\'var\' name'))
		expected = GrammarExpressionList([GrammarConstant('var'), GrammarReference('name')])
		self.assertEqual(result, expected)

	def test_expect_token_grammar_list_2(self):
		result = Parser().expect_token_grammar(SourceIterator('\'first\' \'second\''))
		expected = GrammarExpressionList([GrammarConstant('first'), GrammarConstant('second')])
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_1(self):
		result = Parser().expect_token_grammar(SourceIterator('\'a\'*'))
		expected = GrammarZeroOrMany(GrammarConstant('a'))
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_2(self):
		result = Parser().expect_token_grammar(SourceIterator('\'a\'+'))
		expected = GrammarOneOrMany(GrammarConstant('a'))
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_3(self):
		result = Parser().expect_token_grammar(SourceIterator('\'a\'?'))
		expected = GrammarZeroOrOne(GrammarConstant('a'))
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_4(self):
		result = Parser().expect_token_grammar(SourceIterator('\'a\'?bcd*'))
		expected = GrammarExpressionList([GrammarZeroOrOne(GrammarConstant('a')), GrammarZeroOrMany(GrammarReference('bcd'))])
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_5(self):
		result = Parser().expect_token_grammar(SourceIterator('\'a\'? b c d*'))
		expected = GrammarExpressionList([
			GrammarZeroOrOne(GrammarConstant('a')),
			GrammarReference('b'),
			GrammarReference('c'),
			GrammarZeroOrMany(GrammarReference('d'))
			])
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_6(self):
		result = Parser().expect_token_grammar(SourceIterator('\'a\'? (b) c (d)*'))
		expected = GrammarExpressionList([
			GrammarZeroOrOne(GrammarConstant('a')),
			GrammarReference('b'),
			GrammarReference('c'),
			GrammarZeroOrMany(GrammarReference('d'))
			])
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_7(self):
		result = Parser().expect_token_grammar(SourceIterator('\'a\'? (b c)+ d*'))
		expected = GrammarExpressionList([
			GrammarZeroOrOne(GrammarConstant('a')),
			GrammarOneOrMany(GrammarExpressionList([GrammarReference('b'), GrammarReference('c')])),
			GrammarZeroOrMany(GrammarReference('d'))
			])
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_8(self):
		result = Parser().expect_token_grammar(SourceIterator('(a*)*'))
		expected = GrammarZeroOrMany(GrammarZeroOrMany(GrammarReference('a')))
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_9(self):
		result = Parser().expect_token_grammar(SourceIterator('(a'))
		self.assertEqual(result, None)

	def test_expect_token_grammar_expression_10(self):
		result = Parser().expect_token_grammar(SourceIterator('(a*'))
		self.assertEqual(result, None)

	def test_expect_token_grammar_expression_11(self):
		result = Parser().expect_token_grammar(SourceIterator('a*\''))
		self.assertEqual(result, GrammarZeroOrMany(GrammarReference('a')))

	def test_expect_token_grammar_expression_12(self):
		result = Parser().expect_token_grammar(SourceIterator(''))
		self.assertEqual(result, None)

	def test_expect_token_grammar_expression_13(self):
		result = Parser().expect_token_grammar(SourceIterator('()*'))
		self.assertEqual(result, None)

	def test_expect_token_grammar_expression_14(self):
		result = Parser().expect_token_grammar(SourceIterator('a | b'))
		expected = GrammarOrExpressionList([GrammarReference('a'), GrammarReference('b')])
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_15(self):
		result = Parser().expect_token_grammar(SourceIterator('a* | b?'))
		expected = GrammarOrExpressionList([
			GrammarZeroOrMany(GrammarReference('a')),
			GrammarZeroOrOne(GrammarReference('b')) ])
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_16(self):
		result = Parser().expect_token_grammar(SourceIterator('(a* | b?) | c'))
		expected = GrammarOrExpressionList([
			GrammarOrExpressionList([
				GrammarZeroOrMany(GrammarReference('a')),
				GrammarZeroOrOne(GrammarReference('b')) ]),
			GrammarReference('c') ])
		self.assertEqual(result, expected)

	def test_expect_token_grammar_expression_16(self):
		result = Parser().expect_token_grammar(SourceIterator('a* | b? | \'c\''))
		expected = GrammarOrExpressionList([
			GrammarZeroOrMany(GrammarReference('a')),
			GrammarZeroOrOne(GrammarReference('b')),
			GrammarConstant('c') ])
		self.assertEqual(result, expected)

	def test_special_input_recognition_1(self):
		result = recognize_input_scape('abc')
		self.assertEqual(result, ['a', 'b', 'c'])

	def test_special_input_recognition_2(self):
		result = recognize_input_scape('a\\tc')
		self.assertEqual(result, ['a', '\t', 'c'])

	def test_special_input_recognition_3(self):
		result = recognize_input_scape('a\\t\\\\\\n\\rc')
		self.assertEqual(result, ['a', '\t', '\\', '\n', '\r', 'c'])

	def test_special_input_recognition_4(self):
		result = recognize_input_scape('a\\wc')
		self.assertEqual(result, ['a', SpecialInput.LETTER, 'c'])

	def test_special_input_recognition_5(self):
		result = recognize_input_scape('a\\w\\.\\d\\D\\W\\sc')
		self.assertEqual(result, [
			'a',
			SpecialInput.LETTER,
			SpecialInput.ANY,
			SpecialInput.DIGIT,
			SpecialInput.NON_DIGIT,
			SpecialInput.NON_LETTER,
			SpecialInput.WHITESPACE,
			'c'])

	def test_special_input_recognition_6(self):
		result = recognize_input_scape('\\w')
		self.assertEqual(result, [SpecialInput.LETTER])

	def test_special_input_recognition_7(self):
		result = recognize_input_scape('á家ل')
		self.assertEqual(result, ['á', '家', 'ل'])

def create_nfa_graph(moves, accepting_states):
	states = {}

	for u, v, i in moves:
		if u not in states:
			states[u] = NFAState()
			states[u].index = u
		if v not in states:
			states[v] = NFAState()
			states[v].index = v
		states[u].consume(i, states[v])

	for u, r in accepting_states:
		if u not in states:
			states[u] = NFAState()
			states[u].index = u
		states[u].rule = r

	graph = NFAGraph()
	graph.states = list(states.values())
	return graph

def create_dfa_graph(moves, accepting_states):
	states = {}

	for u, v, i in moves:
		if u not in states:
			states[u] = DFAState()
			states[u].index = u
		if v not in states:
			states[v] = DFAState()
			states[v].index = v
		states[u].consume(i, states[v])

	for u, r in accepting_states:
		if u not in states:
			states[u] = DFAState()
			states[u].index = u
		states[u].rules = r

	graph = DFAGraph()
	graph.states = list(states.values())
	return graph

def match_grammar(grammar, text):
	context = Context()
	Parser().free_context(SourceIterator(grammar), context)
	nfa_graph = NFAGraphGenerator().generate(context)
	dfa_graph = DFAGraphGenerator().generate(nfa_graph)
	transition_table = TransitionTableGenerator().generate(dfa_graph)

	return TransitionTableTraverser().traverse(transition_table, text)

class TestLexerGenerator(unittest.TestCase):
	def test_nfa_generation_1(self):
		context = Context()
		context.rules['var'] = RuleInfo('var', RuleTypes.TOKEN, GrammarConstant('var'))
		result = NFAGraphGenerator().generate(context)

		expected = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(1, 2, LexerInput.char('v')),
						(2, 3, LexerInput.char('a')),
						(3, 4, LexerInput.char('r'))],
					accepting_states = [
						(4, 'var')])

		self.assertEqual(result, expected)

	def test_nfa_generation_2(self):
		context = Context()
		context.rules['t'] = RuleInfo('t', RuleTypes.TOKEN, GrammarZeroOrOne(GrammarConstant('var')))
		result = NFAGraphGenerator().generate(context)

		expected = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(1, 2, LexerInput.char('v')),
						(2, 3, LexerInput.char('a')),
						(3, 4, LexerInput.char('r')),
						(1, 4, LexerInput.DEFAULT)],
					accepting_states = [
						(4, 't')])

		self.assertEqual(result, expected)

	def test_nfa_generation_3(self):
		context = Context()
		context.rules['t'] = RuleInfo('t', RuleTypes.TOKEN, GrammarZeroOrMany(GrammarConstant('var')))
		result = NFAGraphGenerator().generate(context)

		expected = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(1, 2, LexerInput.DEFAULT),
						(1, 3, LexerInput.char('v')),
						(3, 4, LexerInput.char('a')),
						(4, 5, LexerInput.char('r')),
						(5, 1, LexerInput.DEFAULT)],
					accepting_states = [
						(2, 't')])

		self.assertEqual(result, expected)

	def test_nfa_generation_4(self):
		context = Context()
		context.rules['t'] = RuleInfo('t', RuleTypes.TOKEN, GrammarOneOrMany(GrammarConstant('var')))
		result = NFAGraphGenerator().generate(context)

		expected = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(1, 2, LexerInput.char('v')),
						(2, 3, LexerInput.char('a')),
						(3, 4, LexerInput.char('r')),
						(4, 5, LexerInput.DEFAULT),
						(4, 6, LexerInput.char('v')),
						(6, 7, LexerInput.char('a')),
						(7, 8, LexerInput.char('r')),
						(8, 4, LexerInput.DEFAULT)],
					accepting_states = [
						(5, 't')])

		self.assertEqual(result, expected)

	def test_nfa_generation_5(self):
		context = Context()
		context.rules['t'] = RuleInfo('t', RuleTypes.TOKEN,
			GrammarExpressionList([
				GrammarZeroOrMany(GrammarConstant('var')),
				GrammarConstant('foo')]))
		result = NFAGraphGenerator().generate(context)

		expected = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(1, 2, LexerInput.DEFAULT),
						(1, 3, LexerInput.char('v')),
						(3, 4, LexerInput.char('a')),
						(4, 5, LexerInput.char('r')),
						(5, 1, LexerInput.DEFAULT),
						(2, 6, LexerInput.char('f')),
						(6, 7, LexerInput.char('o')),
						(7, 8, LexerInput.char('o'))],
					accepting_states = [
						(8, 't')])

		self.assertEqual(result, expected)

	def test_nfa_generation_6(self):
		context = Context()
		context.rules['t'] = RuleInfo('t', RuleTypes.TOKEN, GrammarConstant('foo'))
		context.rules['s'] = RuleInfo('s', RuleTypes.TOKEN, GrammarZeroOrOne(GrammarConstant('bar')))
		result = NFAGraphGenerator().generate(context)

		expected = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.char('f')),
						(1, 2, LexerInput.char('o')),
						(2, 3, LexerInput.char('o')),
						(0, 4, LexerInput.char('b')),
						(4, 5, LexerInput.char('a')),
						(5, 6, LexerInput.char('r')),
						(0, 6, LexerInput.DEFAULT)],
					accepting_states = [
						(3, 't'),
						(6, 's')])

		self.assertEqual(result, expected)

	def test_nfa_generation_7(self):
		context = Context()
		context.rules['t'] = RuleInfo('t', RuleTypes.TOKEN, GrammarConstant('foo'))
		context.rules['s'] = RuleInfo('s', RuleTypes.TOKEN, GrammarZeroOrOne(GrammarConstant('foobar')))
		result = NFAGraphGenerator().generate(context)

		expected = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.char('f')),
						(1, 2, LexerInput.char('o')),
						(2, 3, LexerInput.char('o')),
						(0, 4, LexerInput.char('f')),
						(4, 5, LexerInput.char('o')),
						(5, 6, LexerInput.char('o')),
						(6, 7, LexerInput.char('b')),
						(7, 8, LexerInput.char('a')),
						(8, 9, LexerInput.char('r')),
						(0, 9, LexerInput.DEFAULT)],
					accepting_states = [
						(3, 't'),
						(9, 's')])

		self.assertEqual(result, expected)

	def test_nfa_generation_8(self):
		context = Context()
		context.rules['t'] = RuleInfo('t', RuleTypes.TOKEN,
			GrammarOrExpressionList([
				GrammarZeroOrMany(GrammarConstant('var')),
				GrammarConstant('foo')]))
		result = NFAGraphGenerator().generate(context)

		expected = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(1, 3, LexerInput.DEFAULT),
						(1, 8, LexerInput.DEFAULT),
						(3, 4, LexerInput.DEFAULT),
						(3, 5, LexerInput.char('v')),
						(5, 6, LexerInput.char('a')),
						(6, 7, LexerInput.char('r')),
						(7, 3, LexerInput.DEFAULT),
						(4, 2, LexerInput.DEFAULT),
						(8, 9, LexerInput.char('f')),
						(9, 10, LexerInput.char('o')),
						(10, 11, LexerInput.char('o')),
						(11, 2, LexerInput.DEFAULT)],
					accepting_states = [
						(2, 't')])

		self.assertEqual(result, expected)

	def test_nfa_generation_9(self):
		context = Context()
		context.rules['t'] = RuleInfo('t', RuleTypes.TOKEN, GrammarZeroOrOne(GrammarReference('s')))
		context.rules['s'] = RuleInfo('s', RuleTypes.FRAGMENT, GrammarConstant('var'))
		result = NFAGraphGenerator().generate(context)

		expected = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(1, 2, LexerInput.char('v')),
						(2, 3, LexerInput.char('a')),
						(3, 4, LexerInput.char('r')),
						(1, 4, LexerInput.DEFAULT)],
					accepting_states = [
						(4, 't')])

	def test_dfa_generation_1(self):
		nfa_graph = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.char('v')),
						(1, 2, LexerInput.char('a')),
						(2, 3, LexerInput.char('r'))],
					accepting_states = [
						(3, 'var')])

		result = DFAGraphGenerator().generate(nfa_graph)

		expected = create_dfa_graph(
					moves = [
						(0, 1, LexerInput.char('v')),
						(1, 2, LexerInput.char('a')),
						(2, 3, LexerInput.char('r'))],
					accepting_states = [
						(3, ['var'])])

		self.assertEqual(result, expected)

	def test_dfa_generation_2(self):
		nfa_graph = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.char('v')),
						(1, 2, LexerInput.char('a')),
						(2, 3, LexerInput.char('r')),
						(0, 3, LexerInput.DEFAULT)],
					accepting_states = [
						(3, 't')])

		result = DFAGraphGenerator().generate(nfa_graph)

		expected = create_dfa_graph(
					moves = [
						(0, 1, LexerInput.char('v')),
						(1, 2, LexerInput.char('a')),
						(2, 3, LexerInput.char('r'))],
					accepting_states = [
						(0, ['t']), (3, ['t'])])

		self.assertEqual(result, expected)

	def test_dfa_generation_3(self):
		nfa_graph = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(0, 2, LexerInput.char('v')),
						(2, 3, LexerInput.char('a')),
						(3, 4, LexerInput.char('r')),
						(4, 0, LexerInput.DEFAULT)],
					accepting_states = [
						(1, 't')])

		result = DFAGraphGenerator().generate(nfa_graph)

		expected = create_dfa_graph(
					moves = [
						(0, 1, LexerInput.char('v')),
						(1, 2, LexerInput.char('a')),
						(2, 3, LexerInput.char('r')),
						(3, 1, LexerInput.char('v'))],
					accepting_states = [
						(0, ['t']), (3, ['t'])])

		self.assertEqual(result, expected)

	def test_dfa_generation_4(self):
		nfa_graph = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.char('v')),
						(1, 2, LexerInput.char('a')),
						(2, 3, LexerInput.char('r')),
						(3, 4, LexerInput.DEFAULT),
						(3, 5, LexerInput.char('v')),
						(5, 6, LexerInput.char('a')),
						(6, 7, LexerInput.char('r')),
						(7, 3, LexerInput.DEFAULT)],
					accepting_states = [
						(4, 't')])

		result = DFAGraphGenerator().generate(nfa_graph)

		expected = create_dfa_graph(
					moves = [
						(0, 1, LexerInput.char('v')),
						(1, 2, LexerInput.char('a')),
						(2, 3, LexerInput.char('r')),
						(3, 4, LexerInput.char('v')),
						(4, 5, LexerInput.char('a')),
						(5, 6, LexerInput.char('r')),
						(6, 4, LexerInput.char('v'))],
					accepting_states = [
						(3, ['t']), (6, ['t'])])

		self.assertEqual(result, expected)

	def test_dfa_generation_5(self):
		nfa_graph = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(0, 2, LexerInput.char('v')),
						(2, 3, LexerInput.char('a')),
						(3, 4, LexerInput.char('r')),
						(4, 0, LexerInput.DEFAULT),
						(1, 5, LexerInput.char('f')),
						(5, 6, LexerInput.char('o')),
						(6, 7, LexerInput.char('o'))],
					accepting_states = [
						(7, 't')])

		result = DFAGraphGenerator().generate(nfa_graph)

		expected = create_dfa_graph(
					moves = [
						(0, 1, LexerInput.char('f')),
						(0, 2, LexerInput.char('v')),
						(1, 3, LexerInput.char('o')),
						(2, 4, LexerInput.char('a')),
						(3, 5, LexerInput.char('o')),
						(4, 6, LexerInput.char('r')),
						(6, 1, LexerInput.char('f')),
						(6, 2, LexerInput.char('v'))],
					accepting_states = [
						(5, 't')])

		self.assertEqual(result, expected)

	def test_nfa_generation_6(self):
		nfa_graph = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.char('f')),
						(1, 2, LexerInput.char('o')),
						(2, 3, LexerInput.char('o')),
						(0, 4, LexerInput.char('b')),
						(4, 5, LexerInput.char('a')),
						(5, 6, LexerInput.char('r')),
						(0, 6, LexerInput.DEFAULT)],
					accepting_states = [
						(3, 't'),
						(6, 's')])

		result = DFAGraphGenerator().generate(nfa_graph)

		expected = create_dfa_graph(
					moves = [
						(0, 1, LexerInput.char('b')),
						(0, 2, LexerInput.char('f')),
						(1, 3, LexerInput.char('a')),
						(2, 4, LexerInput.char('o')),
						(3, 5, LexerInput.char('r')),
						(4, 6, LexerInput.char('o'))],
					accepting_states = [
						(0, ['s']),
						(5, ['s']),
						(6, ['t'])])

		self.assertEqual(result, expected)

	def test_nfa_generation_7(self):
		nfa_graph = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.char('f')),
						(1, 2, LexerInput.char('o')),
						(2, 3, LexerInput.char('o')),
						(0, 4, LexerInput.char('f')),
						(4, 5, LexerInput.char('o')),
						(5, 6, LexerInput.char('o')),
						(6, 7, LexerInput.char('b')),
						(7, 8, LexerInput.char('a')),
						(8, 9, LexerInput.char('r')),
						(0, 9, LexerInput.DEFAULT)],
					accepting_states = [
						(3, 't'),
						(9, 's')])

		result = DFAGraphGenerator().generate(nfa_graph)

		expected = create_dfa_graph(
					moves = [
						(0, 1, LexerInput.char('f')),
						(1, 2, LexerInput.char('o')),
						(2, 3, LexerInput.char('o')),
						(3, 4, LexerInput.char('b')),
						(4, 5, LexerInput.char('a')),
						(5, 6, LexerInput.char('r'))],
					accepting_states = [
						(0, ['s']),
						(3, ['t']),
						(6, ['s'])])

		self.assertEqual(result, expected)

	def test_nfa_generation_8(self):
		nfa_graph = create_nfa_graph(
					moves = [
						(0, 1, LexerInput.DEFAULT),
						(1, 3, LexerInput.DEFAULT),
						(1, 8, LexerInput.DEFAULT),
						(3, 4, LexerInput.DEFAULT),
						(3, 5, LexerInput.char('v')),
						(5, 6, LexerInput.char('a')),
						(6, 7, LexerInput.char('r')),
						(7, 3, LexerInput.DEFAULT),
						(4, 2, LexerInput.DEFAULT),
						(8, 9, LexerInput.char('f')),
						(9, 10, LexerInput.char('o')),
						(10, 11, LexerInput.char('o')),
						(11, 2, LexerInput.DEFAULT)],
					accepting_states = [
						(2, 't')])

		result = DFAGraphGenerator().generate(nfa_graph)

		expected = create_dfa_graph(
					moves = [
						(0, 1, LexerInput.char('f')),
						(0, 2, LexerInput.char('v')),
						(1, 3, LexerInput.char('o')),
						(2, 4, LexerInput.char('a')),
						(3, 5, LexerInput.char('o')),
						(4, 6, LexerInput.char('r')),
						(6, 2, LexerInput.char('v'))],
					accepting_states = [
						(0, ['t']),
						(5, ['t']),
						(6, ['t'])])

		self.assertEqual(result, expected)

	def test_input_matching_1(self):
		result = LexerInput.match(LexerInput.ANY, LexerInput.char('a'))
		self.assertEqual(result, True)

	def test_input_matching_2(self):
		result = LexerInput.match(LexerInput.ANY, LexerInput.WHITESPACE)
		self.assertEqual(result, True)

	def test_input_matching_3(self):
		result = LexerInput.match(LexerInput.DIGIT, LexerInput.ANY)
		self.assertEqual(result, True)

	def test_input_matching_4(self):
		result = LexerInput.match(LexerInput.ANY, LexerInput.NON_DIGIT)
		self.assertEqual(result, True)

	def test_input_matching_5(self):
		result = LexerInput.match(LexerInput.LETTER, LexerInput.ANY)
		self.assertEqual(result, True)

	def test_input_matching_6(self):
		result = LexerInput.match(LexerInput.LETTER, LexerInput.char('a'))
		self.assertEqual(result, True)

	def test_input_matching_7(self):
		result = LexerInput.match(LexerInput.LETTER, LexerInput.WHITESPACE)
		self.assertEqual(result, False)

	def test_input_matching_8(self):
		result = LexerInput.match(LexerInput.WHITESPACE, LexerInput.LETTER)
		self.assertEqual(result, False)

	def test_input_matching_9(self):
		result = LexerInput.match(LexerInput.char('A'), LexerInput.LETTER)
		self.assertEqual(result, True)

	def test_input_matching_10(self):
		result = LexerInput.match(LexerInput.NON_DIGIT, LexerInput.LETTER)
		self.assertEqual(result, True)

	def test_input_matching_11(self):
		result = LexerInput.match(LexerInput.NON_DIGIT, LexerInput.NON_LETTER)
		self.assertEqual(result, True)

	def test_input_matching_12(self):
		result = LexerInput.match(LexerInput.LETTER, LexerInput.DIGIT)
		self.assertEqual(result, False)

	def test_input_matching_13(self):
		result = LexerInput.match(LexerInput.char('a'), LexerInput.DIGIT)
		self.assertEqual(result, False)

	def test_input_matching_14(self):
		result = LexerInput.match(LexerInput.char('8'), LexerInput.DIGIT)
		self.assertEqual(result, True)

	def test_input_matching_15(self):
		result = LexerInput.match(LexerInput.NON_DIGIT, LexerInput.char('a'))
		self.assertEqual(result, True)

	def test_input_matching_16(self):
		result = LexerInput.match(LexerInput.NON_DIGIT, LexerInput.char('8'))
		self.assertEqual(result, False)

	def test_input_matching_17(self):
		result = LexerInput.match(LexerInput.char(' '), LexerInput.WHITESPACE)
		self.assertEqual(result, True)

	def test_input_matching_18(self):
		self.assertRaises(NotImplementedError, LexerInput.match, LexerInput.DEFAULT, LexerInput.DEFAULT)

	def test_input_matching_19(self):
		self.assertRaises(NotImplementedError, LexerInput.match, LexerInput.DEFAULT, LexerInput.WHITESPACE)

	def test_input_matching_20(self):
		self.assertRaises(NotImplementedError, LexerInput.match, LexerInput.char('A'), LexerInput.DEFAULT)

	def test_input_matching_21(self):
		result = LexerInput.match(LexerInput.DIGIT, LexerInput.NON_LETTER)
		self.assertEqual(result, True)

	def test_table_traverser_1(self):
		grammar = """ token s : 'foo';
		              token t : 'foobar'; """
		result = match_grammar(grammar, 'foo')
		self.assertEqual(result, [(0, 3, 's')])

	def test_table_traverser_2(self):
		grammar = """ token s : 'foofoo';
		              token t : 'foo'; """
		result = match_grammar(grammar, 'foofoo')
		self.assertEqual(result, [(0, 6, 's')])

	def test_table_traverser_3(self):
		grammar = """ token s : 'foo'*;
		              token t : 'far'; """
		result = match_grammar(grammar, 'foofoofar')
		self.assertEqual(result, [(0, 6, 's'), (6, 3, 't')])

	def test_table_traverser_4(self):
		grammar = """ token s : 'foo';
		              token t : 'bar'; """
		result = match_grammar(grammar, 'foobar')
		self.assertEqual(result, [(0, 3, 's'), (3, 3, 't')])

	def test_table_traverser_5(self):
		grammar = """ token s : 'f'?'oo';
		              token t : 'bar'; """
		result = match_grammar(grammar, 'foo')
		self.assertEqual(result, [(0, 3, 's')])

	def test_table_traverser_6(self):
		grammar = """ token s : 'f'?'oo';
		              token t : 'bar'; """
		result = match_grammar(grammar, 'bar')
		self.assertEqual(result, [(0, 3, 't')])

	def test_table_traverser_7(self):
		grammar = """ token a : '!=';
		              token b : '!';
		              token c : '='; """
		result = match_grammar(grammar, '!=')
		self.assertEqual(result, [(0, 2, 'a')])

	def test_table_traverser_8(self):
		grammar = """ token a : '\\d'*;
		              token b : '\\w'*;
		              token c : '\\s'*; """
		result = match_grammar(grammar, 'h3llo world 42')
		self.assertEqual(result, [(0, 1, 'b'), (1, 1, 'a'), (2, 3, 'b'), (5, 1, 'c'), (6, 5, 'b'), (11, 1, 'c'), (12, 2, 'a')])

	def test_table_traverser_9(self):
		grammar = """ token a : '\\w'('\\w'*)('\\d'*); """
		result = match_grammar(grammar, 'anIdentifier42')
		self.assertEqual(result, [(0, 14, 'a')])

	def test_table_traverser_10(self):
		grammar = """ token a : '\\w'*;
		              token b : 'var'; """
		result = match_grammar(grammar, 'var')
		self.assertEqual(result, [(0, 3, 'a'), (0, 3, 'b')])

	def test_table_traverser_11(self):
		grammar = """ token a : 'var';
		              token b : '\\w'*; """
		result = match_grammar(grammar, 'var')
		self.assertEqual(result, [(0, 3, 'a'), (0, 3, 'b')])

	def test_table_traverser_12(self):
		grammar = """ token a : '\\w'*; """
		result = match_grammar(grammar, 'habíaunárabellamadoلyunchinollamado家')
		self.assertEqual(result, [(0, 36, 'a')])

	def test_table_traverser_13(self):
		grammar = """ token a : '\\w'*; """
		self.assertRaises(LexerError, match_grammar, grammar, 'there\'s no space')

	def test_table_traverser_14(self):
		grammar = """ token a : '\\w'*; """
		self.assertRaises(LexerError, match_grammar, grammar, '42')

	def test_table_traverser_15(self):
		grammar = """ token a : '\\d'*; """
		self.assertRaises(LexerError, match_grammar, grammar, 'asdadasd')

	def test_table_traverser_16(self):
		grammar = """ token a : '\\s'*; """
		self.assertRaises(LexerError, match_grammar, grammar, ' 	\nd    \r   ')

	def test_table_traverser_17(self):
		grammar = """ token a : 'abc'; """
		self.assertRaises(LexerError, match_grammar, grammar, 'abcde')

	def test_table_traverser_18(self):
		grammar = """ token a : 'abc' | 'cba'; """
		result = match_grammar(grammar, 'abccba')
		self.assertEqual(result, [(0, 3, 'a'), (3, 3, 'a')])

	def test_table_traverser_19(self):
		grammar = """ token a : ('abc'* | 'cba') 'd'; """
		result = match_grammar(grammar, 'abcabcabcd')
		self.assertEqual(result, [(0, 10, 'a')])

	def test_table_traverser_20(self):
		grammar = """ token a : ('abc' | 'cba')*; """
		result = match_grammar(grammar, 'abccbaabc')
		self.assertEqual(result, [(0, 9, 'a')])

	def test_table_traverser_21(self):
		grammar = """ token b : 'abccba' ; """
		result = match_grammar(grammar, 'abccbaabccba')
		self.assertEqual(result, [(0, 6, 'b'), (6, 6, 'b')])

	def test_table_traverser_22(self):
		grammar = """ token a : ('abc' | 'cba')* ; """
		result = match_grammar(grammar, 'abccbaabccba')
		self.assertEqual(result, [(0, 12, 'a')])

	def test_table_traverser_23(self):
		grammar = """ token a : ('abc' | 'cba')* ;
		              token b : 'abccba' ; """
		result = match_grammar(grammar, 'abccbaabccba')
		self.assertEqual(result, [(0, 12, 'a')])

	def test_table_traverser_24(self):
		grammar = """ token a : ('abc' | 'cba')* ;
		              token b : 'abccba' ; """
		result = match_grammar(grammar, 'abccba')
		self.assertEqual(result, [(0, 6, 'a'), (0, 6, 'b')])

	def test_table_traverser_25(self):
		grammar = """ token a      : (abc | 'cba')* ;
		              token b      : 'abccba' ;
		              fragment abc : 'abc' ; """
		result = match_grammar(grammar, 'abccba')
		self.assertEqual(result, [(0, 6, 'a'), (0, 6, 'b')])

if __name__ == '__main__':
	unittest.main()


text:
/* @flow */

import statsMock from "../statsMock";
import SummaryColumn from "../../lib/columns/SummaryColumn";
import SearchMatch from "../../lib/SearchMatch";

describe("columns/SummaryColumn", function() {
  let column;

  beforeEach(function() {
    column = new SummaryColumn();
  });

  describe(".sortField", function() {
    it("should return given sort field value", function() {
      expect(column.sortField).toEqual("name");
    });
  });

  describe(".cellContent", function() {
    let note, path;

    beforeEach(function() {
      path = "/notes/markdown.md";
      note = {
        id: "",
        name: "markdown",
        ext: ".md",
        content: "# testing markdown\nshould be **Zzz** by now tho",
        stats: statsMock()
      };
    });

    describe("when there is no query applied", function() {
      let cellContent: any;

      beforeEach(function() {
        cellContent = column.cellContent({ note, path });
      });

      it("should return a kind of AST from which a DOM can be created", function() {
        expect(cellContent).toEqual(jasmine.any(Array));
        expect(cellContent[0]).toEqual("markdown", "title");
        expect(cellContent[1]).toEqual({
          attrs: { className: "text-subtle" },
          content: ".md"
        });
        expect(cellContent[2]).toEqual(
          jasmine.any(String),
          "should be a separator between title and note content preview"
        );
        expect(cellContent[3]).toEqual(
          jasmine.any(Object),
          "note content preview"
        );
        expect(cellContent[3].attrs).toEqual(jasmine.any(Object));
        expect(cellContent[3].content).toEqual(jasmine.any(String));
      });
    });

    describe("when search match in the middle of string", function() {
      it("should provide what parts of title and note content preview that should be highlighted", function() {
        const searchMatch: any = new SearchMatch(
          // as generated by sifter.js
          // eslint-disable-next-line no-misleading-character-class
          /[kƘƙꝀꝁḰḱǨǩḲḳḴḵκϰ₭][dĎďḊḋḐḑḌḍḒḓḎḏĐđD̦d̦ƉɖƊɗƋƌᵭᶁᶑȡᴅDdð]/i
        );
        const cellContent: any = column.cellContent({
          note,
          path,
          searchMatch
        });
        expect(cellContent).toEqual(jasmine.any(Array));
        expect(cellContent[0]).toEqual(
          jasmine.any(Array),
          "title w/ highlight"
        );
        expect(cellContent[0]).toEqual([
          "mar",
          { attrs: { className: "text-highlight" }, content: "kd" },
          "own"
        ]);
        expect(cellContent[1]).toEqual({
          attrs: { className: "text-subtle" },
          content: ".md"
        });
        expect(cellContent[2]).toEqual(" - ");
        expect(cellContent[3].content).toEqual([
          "# testing mar",
          { attrs: { className: "text-highlight" }, content: "kd" },
          "own\nshould be **Zzz** by now tho"
        ]);
      });

      it("should provide suffix part when 1st match is deep into the content", () => {
        note.content =
          Array(5000).join("x") +
          [
            "Need to some kind of comparison matrix to explain pros/cons of each solution.",
            "In addition to the random string above part of this longer text should also be included in the preview",
            "In addition to the random string above part of this longer text should also be included in the preview",
            "In addition to the random string above part of this longer text should also be included in the preview",
            "In addition to the random string above part of this longer text should also be included in the preview"
          ].join(" \n");

        const searchMatch: any = new SearchMatch(
          // eslint-disable-next-line no-misleading-character-class
          /expl/i
        );
        const cellContent: any = column.cellContent({
          note,
          path,
          searchMatch
        });
        expect(cellContent).toEqual(jasmine.any(Array));
        expect(cellContent[0]).toEqual("markdown");
        expect(cellContent[1]).toEqual({
          attrs: { className: "text-subtle" },
          content: ".md"
        });
        expect(cellContent[2]).toEqual(" - ");
        expect(cellContent[3].content).toEqual([
          "…omparison matrix to ",
          { attrs: { className: "text-highlight" }, content: "expl" },
          [
            "ain pros/cons of each solution.",
            "In addition to the random string above part of this longer text should also be included in the preview",
            "In addition to the random string above part of this longer text should also be included in the preview",
            "In addition to the random string above part of this longer text should also be included in the preview",
            "In addition to the random stri"
          ].join(" \n")
        ]);
      });
    });

    describe("when search match in the beginning of string", function() {
      it("should provide what parts of title and note content preview that should be highlighted", function() {
        const cellContent: any = column.cellContent({
          note: note,
          path: path,
          searchMatch: new SearchMatch(/m[aḀḁĂăÂâǍǎȺⱥȦȧẠạÄäÀàÁáĀāÃãÅåąĄÃąĄ]/i)
        });
        expect(cellContent[0]).toEqual([
          "",
          { attrs: { className: "text-highlight" }, content: "ma" },
          "rkdown"
        ]);
        expect(cellContent[1]).toEqual({
          attrs: { className: "text-subtle" },
          content: ".md"
        });
      });
    });

    describe("when search match at end of string", function() {
      it("should provide what parts of title and note content preview that should be highlighted", function() {
        const cellContent: any = column.cellContent({
          note: note,
          path: path,
          searchMatch: new SearchMatch(
            // eslint-disable-next-line no-misleading-character-class
            /[wẂẃẀẁŴŵẄẅẆẇẈẉ][nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲȠƞᵰᶇɳȵɴNnŊŋ]/i
          )
        });
        expect(cellContent[0]).toEqual([
          "markdo",
          { attrs: { className: "text-highlight" }, content: "wn" },
          ""
        ]);
        expect(cellContent[1]).toEqual({
          attrs: { className: "text-subtle" },
          content: ".md"
        });
      });
    });

    describe("when there is no match", function() {
      it("should provide what parts of title and note content preview that should be highlighted", function() {
        const cellContent: any = column.cellContent({
          note: note,
          path: path,
          searchMatch: new SearchMatch(/Zzz/i)
        });
        expect(cellContent[0]).toEqual("markdown");
        expect(cellContent[1]).toEqual(
          { attrs: { className: "text-subtle" }, content: ".md" },
          "match of end should only contain rest+end"
        );
        expect(cellContent[3].content).toEqual([
          "…arkdown\nshould be **",
          { attrs: { className: "text-highlight" }, content: "Zzz" },
          "** by now tho"
        ]);
      });
    });
  });
});

redpajama_set_name: RedPajamaGithub


text:
// Copyright (c) 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import * as metrics from './metrics.js';
import {ChromeHelper} from './mojo/chrome_helper.js';
import {Mode} from './type.js';

/**
 * Thrown when fails to parse intent url.
 */
export class ParseError extends Error {
  /**
   * @param {!URL} url Intent url.
   */
  constructor(url) {
    super(`Failed to parse intent url ${url}`);
    /**
     * @const {!URL} url
     * @private
     */
    this.url_ = url;

    this.name = this.constructor.name;
  }
}

/**
 * Intent from ARC++.
 */
export class Intent {
  /**
   * @param {!URL} url
   * @param {number} intentId
   * @param {Mode} mode
   * @param {boolean} shouldHandleResult
   * @param {boolean} shouldDownScale
   * @param {boolean} isSecure
   * @private
   */
  constructor(
      url, intentId, mode, shouldHandleResult, shouldDownScale, isSecure) {
    /**
     * @const {!URL}
     */
    this.url = url;

    /**
     * @const {number}
     */
    this.intentId = intentId;

    /**
     * Capture mode of intent.
     * @const {Mode}
     */
    this.mode = mode;

    /**
     * Whether the intent should return with the captured result.
     * @const {boolean}
     */
    this.shouldHandleResult = shouldHandleResult;

    /**
     * Whether the captured image should be down-scaled.
     * @const {boolean}
     */
    this.shouldDownScale = shouldDownScale;

    /**
     * If the intent is launched when the device is under secure mode.
     * @const {boolean}
     */
    this.isSecure = isSecure;

    /**
     * Flag for avoiding intent being resolved by foreground and background
     * twice.
     * @type {boolean}
     * @private
     */
    this.done_ = false;
  }

  /**
   * @return {!ChromeHelper}
   * @private
   */
  get chromeHelper_() {
    return ChromeHelper.getInstance();
  }

  /**
   * Whether intent has been finished or canceled.
   * @return {boolean}
   */
  get done() {
    return this.done_;
  }

  /**
   * Notifies ARC++ to finish the intent.
   * @return {!Promise}
   */
  async finish() {
    if (this.done) {
      return;
    }
    this.done_ = true;
    await this.chromeHelper_.finish(this.intentId);
    metrics.log(metrics.Type.INTENT, this, metrics.IntentResultType.CONFIRMED);
  }

  /**
   * Notifies ARC++ to cancel the intent.
   * @return {!Promise}
   */
  async cancel() {
    if (this.done) {
      return;
    }
    this.done_ = true;
    await this.chromeHelper_.cancel(this.intentId);
    metrics.log(metrics.Type.INTENT, this, metrics.IntentResultType.CANCELED);
  }

  /**
   * Notifies ARC++ to append data to the intent result.
   * @param {!Uint8Array} data The data to be appended to intent result.
   * @return {!Promise}
   */
  async appendData(data) {
    if (this.done) {
      return;
    }
    await this.chromeHelper_.appendData(this.intentId, data);
  }

  /**
   * Notifies ARC++ to clear appended intent result data.
   * @return {!Promise}
   */
  async clearData() {
    if (this.done) {
      return;
    }
    await this.chromeHelper_.clearData(this.intentId);
  }

  /**
   * @param {!URL} url Url passed along with app launch event.
   * @return {!Intent} Created intent object. Returns null if input is not a
   *     valid intent url.
   * @throws {ParseError}
   */
  static create(url) {
    const params = url.searchParams;
    const getBool = (key) => params.get(key) === '1';
    let param = params.get('intentId');
    if (param === null) {
      throw new ParseError(url);
    }
    const intentId = parseInt(param, 10);

    param = params.get('mode');
    if (param === null || !Object.values(Mode).includes(param)) {
      throw new ParseError(url);
    }
    const mode = /** @type {Mode} */ (param);

    return new Intent(
        url, intentId, mode, getBool('shouldHandleResult'),
        getBool('shouldDownScale'), getBool('isSecure'));
  }
}


text:
package com.a1works.commons;

import org.junit.Test;
import static org.junit.Assert.*;

/**
 * Created by Ahmed Magdy Ezzeldin <a.magdy@a1works.com> on 15/11/15.
 */
public class MayBeTest {

    @Test
    public void testMayBeWithNull(){
        MayBe<String> mayBe = new MayBe<>();
        assertTrue("MayBe object is expected to be empty when it is not", mayBe.isEmpty());
        assertFalse("MayBe.iterator() should return an iterator with 0 elements", mayBe.iterator().hasNext());
        assertNull("MayBe object should return null on calling get()", mayBe.get());
    }

    @Test
    public void testMayBeWithObject(){
        String str = "Some text";
        MayBe<String> mayBe = new MayBe<>(str);
        assertFalse("MayBe.0isEmpty() is expected to return false.", mayBe.isEmpty());
        assertTrue("MayBe.iterator() should return an iterator with 1 element", mayBe.iterator().hasNext());
        assertEquals("MayBe.get() should NOT return the provided string", str, mayBe.get());
    }
}


text:
import { Attribute, Component, ElementRef, forwardRef, HostBinding, Input, OnInit, ViewChild } from '@angular/core';
import { ControlValueAccessor, NG_VALUE_ACCESSOR } from '@angular/forms';
import { noop } from 'rxjs';

@Component({
	selector: 'ansyn-input',
	templateUrl: './ansyn-input.component.html',
	styleUrls: ['./ansyn-input.component.less'],
	providers: [
		{
			provide: NG_VALUE_ACCESSOR,
			useExisting: forwardRef(() => AnsynInputComponent),
			multi: true
		}
	]
})
export class AnsynInputComponent implements ControlValueAccessor, OnInit {
	@Input() maxLength = 20;
	@Input() label: string;
	@Input() required = true;
	@Input() autocomplete: 'off' | 'on' = 'off';
	@Input()
	@HostBinding('class.white') white: boolean;
	disabled: boolean;

	private innerValue: any = '';
	private onTouchedCallback: () => void = noop;
	private onChangeCallback: (_: any) => void = noop;

	get value(): any {
		return this.innerValue;
	};

	set value(v: any) {
		if (v !== this.innerValue) {
			this.innerValue = v;
			this.onChangeCallback(v);
		}
	}

	@ViewChild('input') input: ElementRef;

	constructor(@Attribute('select') public selectAttr: string) {
	}

	onBlur() {
		this.onTouchedCallback();
	}

	ngOnInit(): void {
		if (this.selectAttr !== null) {
			setTimeout(() => {
				this.input.nativeElement.select();
			}, 200)
		}
	}

	writeValue(value: any) {
		if (value !== this.innerValue) {
			this.innerValue = value;
		}
	}

	registerOnChange(fn: any) {
		this.onChangeCallback = fn;
	}

	registerOnTouched(fn: any) {
		this.onTouchedCallback = fn;
	}

	setDisabledState(isDisabled: boolean): void {
		this.disabled = isDisabled;
	}

}


text:
<!DOCTYPE html>

  <meta charset="UTF-8">
  <title>Code Coverage for C:\wamp\www\cursos\sargentozf2\zf2napratica\module\Admin/src/Admin</title>
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <link href="../../css/bootstrap.min.css" rel="stylesheet">
  <link href="../../css/style.css" rel="stylesheet">

C:\wamp\www\cursos\sargentozf2\zf2napratica\module\Admin

src

Admin

(Dashboard)

Code Coverage

Lines

Functions and Methods

Classes and Traits

      Total

           100.00% covered (success)

100.00%

124 / 124

           100.00% covered (success)

100.00%

8 / 8

           100.00% covered (success)

100.00%

4 / 4

       Controller

           100.00% covered (success)

100.00%

16 / 16

           100.00% covered (success)

100.00%

3 / 3

           100.00% covered (success)

100.00%

1 / 1

       Form

           100.00% covered (success)

100.00%

25 / 25

           100.00% covered (success)

100.00%

1 / 1

           100.00% covered (success)

100.00%

1 / 1

       Model

           100.00% covered (success)

100.00%

60 / 60

           100.00% covered (success)

100.00%

1 / 1

           100.00% covered (success)

100.00%

1 / 1

       Service

           100.00% covered (success)

100.00%

23 / 23

           100.00% covered (success)

100.00%

3 / 3

           100.00% covered (success)

100.00%

1 / 1

Legend

     Low: 0% to 35%
     Medium: 35% to 70%
     High: 70% to 100%

     Generated by PHP_CodeCoverage 2.1.5 using PHP 5.5.12 and PHPUnit 4.7.3 at Wed Jul 1 3:09:14 CEST 2015.


text:
#ifndef BOOST_ASIO_DETAIL_FD_SET_ADAPTER_HPP
#define BOOST_ASIO_DETAIL_FD_SET_ADAPTER_HPP

#if defined(_MSC_VER) && (_MSC_VER >= 1200)
# pragma once
#endif // defined(_MSC_VER) && (_MSC_VER >= 1200)

#include <boost/asio/detail/config.hpp>
#include <boost/asio/detail/posix_fd_set_adapter.hpp>
#include <boost/asio/detail/win_fd_set_adapter.hpp>

namespace boost {
namespace asio {
namespace detail {

#if defined(BOOST_WINDOWS) || defined(__CYGWIN__)
typedef win_fd_set_adapter fd_set_adapter;
#else
typedef posix_fd_set_adapter fd_set_adapter;
#endif

} // namespace detail
} // namespace asio
} // namespace boost

#endif // BOOST_ASIO_DETAIL_FD_SET_ADAPTER_HPP


text:
namespace PhysX
{
	/// <summary>
	///
	/// </summary>
	public ref class VehicleAckermannGeometryData
	{
	internal:
		static VehicleAckermannGeometryData^ ToManaged(PxVehicleAckermannGeometryData data);
		static PxVehicleAckermannGeometryData ToUnmanaged(VehicleAckermannGeometryData^ data);

	public:
		/// <summary>
		/// Accuracy of Ackermann steer calculation. Range: (0,1).
		/// </summary>
		property float Accuracy;

		/// <summary>
		/// Distance between centre-point of the two front wheels, specified in m. Range: (0,inf).
		/// </summary>
		property float FrontWidth;

		/// <summary>
		/// Distance between center-point of the two rear wheels, specified in m. Range: (0,inf).
		/// </summary>
		property float RearWidth;

		/// <summary>
		/// Distance between centre of front axle and centre of rear axle, specified in m. Range: (0,inf).
		/// </summary>
		property float AxleSeparation;
	};
};


text:
const char app_version[] = "1.0\n";
const char default_name[] = "Heizung\n";

char temp_data[20];

#define SET_HEATING(v) GPIO_OUTPUT_SET(GPIO_ID_PIN(3), v)

void simpleudp_init_commands() {
        //PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0RXD_U, FUNC_GPIO3);
        easygpio_pinMode(3, EASYGPIO_NOPULL, EASYGPIO_OUTPUT);

	uint32 rtc_reg_val = 0;
	system_rtc_mem_read(64,&rtc_reg_val,sizeof(uint32));

	strcpy(temp_data, "Temperatur: ");

	add_action("heating", "Heizung", ActTypeToggle, rtc_reg_val>1?1:0, 0, 1);
	add_action("temperature", temp_data, ActTypeStateless, 0, 0, 1000);

	char* t = temp_data + 11;
	init_ds18b20_timer(t,2);
}

void ICACHE_FLASH_ATTR simpleudp_command(struct ActionEntry* entry, int value)
{
    if (strcmp(entry->id, "heating")==0) {
        entry->value = value;
	SET_HEATING(value);
	uint32 rtc_reg_val = value;
	system_rtc_mem_write(64,&rtc_reg_val,sizeof(uint32));
    } else if (strcmp(entry->id, "temperature")==0) {
        print_device_type();
        update_temp(temp_data + 11);
        printf("%s\n", temp_data);
    }
}


text:
import Ember from "ember";

const {
  Component,
  computed
} = Ember;

export default Component.extend({

  departments: computed(function() {
    let model = this.get('model');

    let columnDefs = [
      { headerName: "Office", field: "office", pinned: true, suppressMenu: true, width: 100 },
      { headerName: "Sales Rep", field: "sales_rep", pinned: true, suppressMenu: true },
      { headerName: "Year",
          children: [
            { headerName: "Total", field: "total", width: 150, columnGroupShow: 'closed'},
            { headerName: "2010", field: "year_2010", width: 150, columnGroupShow: 'open'},
            { headerName: "2011", field: "year_2011", width: 150, columnGroupShow: 'open'},
            { headerName: "2012", field: "year_2012", width: 150, columnGroupShow: 'open'},
            { headerName: "2016", field: "year_2016", width: 150, columnGroupShow: 'open'},
            { headerName: "2014", field: "year_2014", width: 150, columnGroupShow: 'open'},
            { headerName: "2015", field: "year_2015", width: 150, columnGroupShow: 'open'},
            { headerName: "2016", field: "year_2016", width: 150, columnGroupShow: 'open'}
          ]}

    ];

    let gridOptions = {
      columnDefs: columnDefs,
      rowData: model,
      rowHeight: 40,
      headerHeight: 25,
      suppressCellSelection: true, //remove option to click on cell
      suppressMovableColumns: true //remove option to move columns
    };

    return gridOptions;
  })

});


text:
package me.vilsol.nmswrapper.wraps.unparsed;

import me.vilsol.nmswrapper.NMSWrapper;
import me.vilsol.nmswrapper.reflections.ReflectiveClass;
import me.vilsol.nmswrapper.reflections.ReflectiveMethod;
import me.vilsol.nmswrapper.wraps.NMSWrap;

import javax.crypto.Cipher;
import java.security.Key;
import java.security.KeyPair;

@ReflectiveClass(name = "MinecraftEncryption")
public class NMSMinecraftEncryption extends NMSWrap {

    public NMSMinecraftEncryption(Object nmsObject){
        super(nmsObject);
    }

    /**
     * TODO Find correct name
     * @see net.minecraft.server.v1_9_R1.MinecraftEncryption#a(int, java.security.Key)
     */
    @ReflectiveMethod(name = "a", types = {int.class, Key.class})
    public Cipher a(int i, Key key){
        return (Cipher) NMSWrapper.getInstance().exec(nmsObject, i, key);
    }

    /**
     * TODO Find correct name
     * @see net.minecraft.server.v1_9_R1.MinecraftEncryption#b()
     */
    @ReflectiveMethod(name = "b", types = {})
    public KeyPair b(){
        return (KeyPair) NMSWrapper.getInstance().exec(nmsObject);
    }

}


text:
package nl.knaw.huygens.timbuctoo.v5.graphql.collectionfilter;

import java.util.List;

public interface FilterResult {
  List<String> getUriList();

  String getNextToken();

  int getTotal();

  List<Facet> getFacets();

  String getPrevToken();
}


text:
package de.tudarmstadt.ukp.clarin.webanno.brat.message;

/**
 * Response to the {@code getDocumentTimestamp} command.
 */
public class GetDocumentTimestampResponse
    extends AjaxResponse
{
    public static final String COMMAND = "getDocumentTimestamp";

    private long mtime;

    public GetDocumentTimestampResponse()
    {
        super(COMMAND);
    }

    /**
     * Get last modification time.
     *
     * @return last modification time.
     */
    public long getMtime()
    {
        return mtime;
    }

    /**
     * Set last modification time.
     *
     * @param aMtime
     *            last modfication time.
     */
    public void setMtime(long aMtime)
    {
        mtime = aMtime;
    }

    public static boolean is(String aCommand)
    {
        return COMMAND.equals(aCommand);
    }
}


text:
namespace net {

struct WriteResult;

// Chrome specific packet writer which uses a DatagramClientSocket for writing
// data.
class NET_EXPORT_PRIVATE QuicDefaultPacketWriter : public QuicPacketWriter {
 public:
  QuicDefaultPacketWriter();
  explicit QuicDefaultPacketWriter(DatagramClientSocket* socket);
  virtual ~QuicDefaultPacketWriter();

  // QuicPacketWriter
  virtual WriteResult WritePacket(
      const char* buffer, size_t buf_len,
      const net::IPAddressNumber& self_address,
      const net::IPEndPoint& peer_address) OVERRIDE;
  virtual bool IsWriteBlockedDataBuffered() const OVERRIDE;
  virtual bool IsWriteBlocked() const OVERRIDE;
  virtual void SetWritable() OVERRIDE;

  void OnWriteComplete(int rv);
  void SetConnection(QuicConnection* connection) {
    connection_ = connection;
  }

 private:
  base::WeakPtrFactory<QuicDefaultPacketWriter> weak_factory_;
  DatagramClientSocket* socket_;
  QuicConnection* connection_;
  bool write_blocked_;
};

}  // namespace net

#endif  // NET_QUIC_QUIC_DEFAULT_PACKET_WRITER_H_


text:
#include "training/graph_group_sync.h"

namespace marian {

SyncGraphGroup::SyncGraphGroup(Ptr<Options> config, Ptr<IMPIWrapper> mpi)
    : GraphGroup(config), ExponentialSmoothing(config),
      delay_{options_->get<double>("optimizer-delay")}, mpi_(mpi) { // @TODO: rename delay_ to something else; delay means delayed updated, not accumulation

  devices_ = Config::getDevices(options_, mpi_->myMPIRank(), mpi_->numMPIProcesses());
  for(auto device : devices_) {
    auto graph = New<ExpressionGraph>();
    graph->setDevice(device);
    graph->setCheckpointing(options_->get<bool>("gradient-checkpointing"));
    graph->reserveWorkspaceMB(options_->get<size_t>("workspace"));

    graphs_.push_back(graph);
    shardOpt_.push_back(Optimizer(options_));
    builders_.push_back(models::createCriterionFunctionFromOptions(options_, models::usage::training));
  }

  // Note: We may well end up with only one MPI process or only one graph per worker.
  // This part of the code will not special-case any of this here.
  // Rather, it is assumed that the communicator knows to reduce unnecessary transfers to no-ops.
  comm_ = createCommunicator(graphs_, /*noNccl=*/options_->get<bool>("no-nccl", false), /*mpi=*/mpi_);

  auto formattedDeviceType = utils::utf8ToUpper(devices_.front().typeAsString()) + "s";
  if (mpi_->numMPIProcesses() > 1)
    LOG(info, "[training] Using {} {}, distributed over {} MPI processes", mpi_->numMPIProcesses() * devices_.size(), formattedDeviceType, mpi_->numMPIProcesses());
  else
    LOG(info, "[training] Using {} {}", devices_.size(), formattedDeviceType);
}

void SyncGraphGroup::setScheduler(Ptr<Scheduler> scheduler) /*override*/ {
  validate();
  scheduler_ = scheduler;
  // optimizer has to be registered last to see changes of learning rate
  // @TODO: ^^Fix this comment. Either it refers to the scheduler, or it should be moved. Which one?
  scheduler_->registerTrainingObserver(scheduler_);

  for(auto opt : shardOpt_)
    scheduler_->registerTrainingObserver(opt);
}

void SyncGraphGroup::initialize(const Ptr<data::Batch>& exampleBatch) {
  // Initialize graphs with random weights in one forward step
  // Also allocate and clear the gradients
  comm_->foreach([&](size_t i, size_t /*begin*/, size_t /*end*/) {
    builders_[i]->build(graphs_[i], exampleBatch);
    graphs_[i]->forward();
    graphs_[i]->params()->allocateBackward();
    graphs_[i]->params()->set_zero_adjoint();
  });

  // Copy weights from 0-th graph to all other graphs
  // to have equal weights across devices
  comm_->foreach([&](size_t i, size_t /*begin*/, size_t /*end*/) {
    if (i > 0)
      graphs_[i]->params()->vals()->copyFrom(graphs_[0]->params()->vals());
  });
}

void SyncGraphGroup::initializeAvg() {
  Ptr<ExpressionGraph> graphAvg; // CPU-side temp
  std::string name = options_->get<std::string>("model");
  std::string suffix = name.substr(name.size() - 4);
  ABORT_IF(suffix != ".npz" && suffix != ".bin", "Unknown model suffix {}", suffix);

  if(filesystem::exists(name + ".orig" + suffix)) {
    // Load the averaged parameters into a temporary graph
    graphAvg = New<ExpressionGraph>();
    graphAvg->setDevice({0, DeviceType::cpu});

    // load model through builder to activate model specific loading functions.
    // This is important if a model is overloading Model::load(...) and e.g.
    // mapping matrix names as in Amun.h
    auto builder = models::createCriterionFunctionFromOptions(options_, models::usage::training);
    builder->load(graphAvg, name, false);
    graphAvg->forward(); // initialize parameters if needed
  }

  auto init = [&](size_t localDeviceIndex, size_t begin, size_t end) {
    size_t size = end-begin;

    // get the device-specific allocator
    auto paramsAllocator = New<TensorAllocator>(graphs_[localDeviceIndex]->getBackend());
    paramsAllocs_[localDeviceIndex] = paramsAllocator;

    paramsAllocator->reserveExact(size * sizeof(float));

    Tensor paramAvg;
    paramsAllocator->allocate(paramAvg, {1, (int)size});
    paramsAvg_[localDeviceIndex] = paramAvg;

    if(graphAvg)
      paramAvg->copyFrom(graphAvg  ->params()->vals()->subtensor(begin, size));
    else
      paramAvg->copyFrom(graphs_[0]->params()->vals()->subtensor(begin, size));

    // note: for multi-node, graphAvg and graphs_[0] contain a complete copy, from which
    // each MPI process copies only part into its respective shard(s)
  };

  paramsAllocs_.resize(graphs_.size()); // allocators
  paramsAvg_.resize(graphs_.size());    // averaged parameters (shards; distributed over MPI processes if applicable)
  comm_->foreach(init, /*parallel=*/false); // @TODO: is sequential operation necessary here? (is the allocation stuff sufficiently reentrant or thread-separated?)
}

Ptr<data::BatchStats> SyncGraphGroup::collectStats(const std::vector<Ptr<Vocab>>& vocabs) {
  // This function determines the granularity in which the reader provides data.
  // If no mini-batch-fit, then user provides a constant number. It reads that much. We won't get into this function.
  // If mini-batch-fit, then we get here and set miniBatchFitMultiplier_. Then...
  // If dynamic MB scaling, then we want fine-grained minibatches of the size of one GPU.
  // If not, we prefer a single large batch that can be split into equal-size parts over GPUs,
  // so that we have perfect load balancing and read precisely as much as we need (no waste).
  double multiplier = devices_.size() * mpi_->numMPIProcesses() * delay_;
  bool isDynamic = scheduler_->isDynamicMBSizeScaling();
  double readerMultiplier = isDynamic ? 1. : multiplier; // multiplier applied already by reader
  updateMultiplier_ = isDynamic ? multiplier : 1.;       // multiplier applied later in update()
  return GraphGroup::collectStats(graphs_[0], builders_[0], vocabs, readerMultiplier);
}

// helper for MB scaling: quantize the ratio with a given error margin
static double roundUpRatio(double ratio) {
  if (ratio == 0)
    return ratio;
  // find largest power of two that fits into ratio
  double p = 1;
  while (p*2 < ratio)
    p *= 2;
  // round up to nearest multiple of a largest power of 2 where relative error is within margin
  // 25% error margin seems acceptable:
  //  - using a 25% larger MB size should not break convergence
  //  - @TODO: not using the first 25% of the next block is OK since those are dominated by data exchange
  double maxError = 0.25;
  while (p >= 1) {
    double proposedRatio = ceil(ratio / p) * p;
    double error = (proposedRatio - ratio) / ratio;
    if (fabs(error) <= maxError)
      return proposedRatio;
    p /= 2;
  }
  return ratio;
}

// helper routine that handles accumulation and load-balancing of sub-batches to fill all devices
// It adds 'newBatch' to 'pendingBatches_', and if sufficient batches have been queued, then
// returns 'pendingBatches_' in 'subBatches' and resets it. If not, it returns false.
bool SyncGraphGroup::tryGetSubBatches(Ptr<data::Batch> newBatch,
    std::vector<Ptr<data::Batch>>& subBatches, size_t& numReadBatches) {
  // The reader delivers in chunks of these sizes, according to case:
  //  - no dynamic MB-size scaling:
  //     - reader batch size = update batch size, with...
  //     - mini-batch-fit:
  //        - update batch size = what fits into all GPUs, times decay_ to allow experimenting with fractional sizes
  //     - no mini-batch-fit:
  //        - update batch size = user-specified size (user guarantees that it fits if distributed over delay_ GPUs)
  //  - dynamic MB-size scaling:
  //     - update batch size = aggregate reader batch size * (dynamic progress-based ratio * reference adjustment), with...
  //     - mini-batch-fit:
  //        - aggregate reader batch size = equal to what fits into one GPU * warpSize * delay_
  //     - no mini-batch-fit:
  //        - aggregate reader batch size = user-specified size (user guarantees that it fits if distributed over delay_ GPUs)
  //     - reference adjustment =
  //        - reference batch size specified: (reference batch size / typical aggregate reader batch size)
  //        - no ref size specified: 1

  size_t warpSize = devices_.size() * mpi_->numMPIProcesses(); // warp := set of batches processed concurrently across GPus and workers

  // if not dynamic then return the big batch, but first split it over GPUs as it may be too large
  if (!scheduler_->isDynamicMBSizeScaling()) {
    // If mini-batch-fit, then the read batch is (devices_.size() * mpi_->numMPIProcesses() * delay_)
    // times what fits one GPU. If not mini-batch-fit, it is whatever the user has specified, which
    // is the user's responsibility to guarantee that it fits into 'delay_' warps.
    // Distribute evenly over all GPUs we have, using multiple warps if needed.
    size_t numWarps = (size_t)ceil(delay_);
    subBatches = newBatch->split(numWarps * warpSize);
    numReadBatches = 1;
    return true;
  }
  LOG_ONCE(info, "[training] Dynamic mini-batch scaling enabled");

  // if dynamic and mini-batch-fit, then we get batches in the size of what fits into one GPU
  pendingBatches_.push_back(newBatch);

  // what ratio (how many batches in reader's batch size) do we want, based on current training progress schedule?
  double ratio = scheduler_->getDynamicMBSizeMultiplier();

  // relative to what base? (what does ratio == 1 mean)
  ratio *= updateMultiplier_; // if mini-batch-fit, this is = warpSize * delay_, otherwise 1

  // If a reference is given, then at progress == mbWarmup.n (ratio=1), we would like to have refBatchLabels instead of whichever
  // the actual batch size is. Since we cannot know the future actual batch sizes that will be delivered
  // by the reader, we approximate them with (typicalTrgBatchWords * updateMultiplier), and scale ratio accordingly.
  auto refBatchLabels = options_->get<size_t>("mini-batch-words");
  if (refBatchLabels != 0) {
    LOG_ONCE(info, "[scheduler] Scaling to {} reference labels, using actual-batch-word estimate of {}", refBatchLabels, typicalTrgBatchWords_);
    ABORT_IF(typicalTrgBatchWords_ == 0, "Dynamic scaling with words target requires MB size to be known in words"); // happens if MB size is specified in sentences
    ratio *= (double)refBatchLabels / (double)(typicalTrgBatchWords_ * updateMultiplier_);
  }

  // round up to full batches if within a certain error margin  --@BUGBUG: Not invariant w.r.t. GPU size, as ratio is relative to what fits into 1 GPU
  ratio = roundUpRatio(ratio);

  if (pendingBatches_.size() < ratio)
    return false; // not enough data yet

  // now we have enough to fill at least 'ratio' batches
  // @BUGBUG: We do not handle the case that fixed MB size * ratio exceeds GPU memory (we'd need to split that).

  numReadBatches = pendingBatches_.size(); // remember original batch-counter increment from reader (which is not always the same as subBatches.size() in the end)

  // in fact, we got too much, so make up for it by shortening all batches to accurately reflect desired ratio
  // e.g. ratio = 3.3 for 4 batches -> Reduce each by 3.3/4
  // Alternatively, we could just shorten the last 'warp', but that would not be invariant to warp size.
  for (auto& batch : pendingBatches_) {
    auto reducedBatchSize = (size_t)ceil((double)batch->size() * ratio / (double)pendingBatches_.size());
    size_t minSize = 1;
    if (pendingBatches_.size() == 1) { // enforce a minimum (only needed/correct if still in first batch)
      size_t minTrgWords = 256;        // don't go below this number of target words, as it seems excessive  --@TODO: parameterize?
      minSize = 1 + (minTrgWords * batch->size() - 1) / batch->wordsTrg(); // approximately convert minTrgWords into a #sentences
    }
    reducedBatchSize = std::max(reducedBatchSize, minSize);
    if (reducedBatchSize < batch->size())
      batch = batch->split(/*numSubBatches=*/1, reducedBatchSize).front();
  }

  // load-balance: distribute the last numWarps-group's batches over GPUs
  // This is tricky since batches do not have the same length, therefore we can only split, but not merge.
  auto numWarps = (pendingBatches_.size() - 1) / warpSize + 1; // = ceil(#buffers / (#GPUs * #workers))
  auto availableDevices = numWarps * warpSize; // we will run this many GPUs: better use them all
  if (pendingBatches_.size() < availableDevices) {
    // last warp does not use all available GPUs: try to re-balance
    auto fullWarpsBatches = (numWarps - 1) * warpSize; // number of batches in all but the last warp. Those warps that are fully used.
    auto lastWarpSize = pendingBatches_.size() - fullWarpsBatches; // the last warp is possibly not fully used
    //LOG(info, "attempting to redistribute last {} batches over {} devices", lastWarpSize, warpSize);
    auto splitInto = warpSize / lastWarpSize;
    if (splitInto > 1) { // unfortunately we can only split in integer ratios
      // split each of last numWarps's batches into 'splitInto' batches
      // pop them first
      std::vector<Ptr<data::Batch>> batchesToSplit;
      while (pendingBatches_.size() > fullWarpsBatches) {
        batchesToSplit.push_back(pendingBatches_.back());
        pendingBatches_.pop_back();
      }
      // now split them and push them back
      for (auto& batchToSplit : batchesToSplit) {
        //LOG(info, "{}-way splitting batchToSplit with size {}", splitInto, batchToSplit->size());
        auto splitBatches = batchToSplit->split(splitInto);
        for (auto& splitBatch : splitBatches) {
          //LOG(info, " -> getting batchToSplit with size {}", splitBatch->size());
          pendingBatches_.push_back(splitBatch);
        }
      }
    }
    ABORT_IF(pendingBatches_.size() > availableDevices, "somehow split into too many batches??");
  }
  subBatches = std::move(pendingBatches_);

  // @TODO: sort by width, so that in case of delay > 1, each GPU gets about the same size
  return true;
}

void SyncGraphGroup::update(Ptr<data::Batch> newBatch) /*override*/ {
  validate();

  std::vector<Ptr<data::Batch>> subBatches;
  size_t numReadBatches; // actual #batches delivered by reader, for restoring from checkpoint   --@TODO: reader should checkpoint itself; should not go via the scheduler
  bool gotSubBatches = tryGetSubBatches(newBatch, subBatches, numReadBatches);

  // not enough data yet: return right away
  if (!gotSubBatches)
    return;

  update(subBatches, numReadBatches);
}

void SyncGraphGroup::update(std::vector<Ptr<data::Batch>> subBatches, size_t numReadBatches) {
  // determine num words for dynamic hyper-parameter adjustment
  // @TODO: We can return these directly from tryGetSubBatches()
  size_t batchSize = 0;
  size_t batchTrgWords = 0;
  for (const auto& batch : subBatches) {
    batchSize     += batch->size();
    batchTrgWords += batch->wordsTrg();
  }

  // Helper to access the subBatches array
  auto getSubBatch = [&](size_t warp, size_t localDeviceIndex, size_t rank) -> Ptr<data::Batch> {
    // Warp should be slowest changing dimension. If subBatches are sorted by
    // length, then grouping sentences of similar length into the same delay step can
    // reduce unnecessary time spent in padding.
    auto index = (warp * mpi_->numMPIProcesses() + rank) * devices_.size() + localDeviceIndex;
    if (index < subBatches.size())
      return subBatches[index];
    else
      return nullptr; // null if we reached beyond the end
  };

  // Helper to quantize the model
  auto quantizeModel = [&](size_t idx, size_t /*begin*/, size_t /*end*/) {
    quantizers_[idx]->quantize(graphs_[idx]);
  };

  // Upon very first execution, reset everything
  if(first_) {
    LOG(info, "[training] Batches are processed as {} process(es) x {} devices/process",
        mpi_->numMPIProcesses(), devices_.size());
    initialize(subBatches.front());
    if(mvAvg_ && paramsAvg_.empty())
      initializeAvg();

    // initialize model quantization
    if (options_->get<size_t>("quantize-bits") > 0) {
      for (int idx = 0; idx < graphs_.size(); idx++)
	quantizers_.push_back(New<ModelQuantizer>(options_));
      comm_->foreach(quantizeModel);
    }

    first_ = false;
  }

  // Compute gradients
  std::vector<StaticLoss> localDeviceLosses(devices_.size()); // [local device index] aggregate cost for each local device
  comm_->foreach([&](size_t localDeviceIndex, size_t /*begin*/, size_t /*end*/) { // parallel across devices. Aggregate for warp > 1.
    auto graph = graphs_[localDeviceIndex];
    // reset gradient  --presently done outside
    //graph->params()->allocateBackward();
    //graph->params()->set_zero_adjoint();
    // This happens in multiple steps if there are more subbatches than devices.
    for (size_t warp = 0; ; warp++) {
      // Execute single forward/backward step
      auto subBatch = getSubBatch(warp, localDeviceIndex, mpi_->myMPIRank());
      if (!subBatch)
        break;

      auto rationalLoss = builders_[localDeviceIndex]->build(graph, subBatch);
      graph->forward();

      localDeviceLosses[localDeviceIndex] += *rationalLoss;
      graph->backward(/*zero=*/false); // (gradients are reset before we get here)
    }
  });
  // At this point, each device on each MPI process has a gradient aggregated over a subset of the sub-batches.

  // Update parameter shard with gradient shard
  auto update = [&](size_t idx, size_t begin, size_t end) {
    auto curGrad = graphs_[idx]->params()->grads()->subtensor(begin, end-begin);
    auto curParam = graphs_[idx]->params()->vals()->subtensor(begin, end-begin);

    // actual model update
    auto updateTrgWords =
        /*if*/(options_->get<std::string>("cost-type") == "ce-sum") ?
          batchTrgWords // total number of labels across all GPUs and nodes
        /*else*/:
          OptimizerBase::mbSizeNotProvided;
    shardOpt_[idx]->update(curParam, curGrad, updateTrgWords);
    curGrad->set(0.f);

    if(mvAvg_)
      updateAvgParams(
          paramsAvg_[idx], curParam, scheduler_->numberOfBatches(), updateTrgWords);
  };

  // cost across all local devices (scheduler will aggregate cross-process)
  StaticLoss localLoss = std::accumulate(localDeviceLosses.begin(), localDeviceLosses.end(), StaticLoss());

  // model update
  if (std::isfinite(localLoss.loss) || mpi_->numMPIProcesses() > 1) { // guard against NaN (except with MPI, as this simple way could hang it)
    comm_->scatterReduceAndResetGrads(); // reduce gradients across all devices and MPI nodes into shards
    comm_->foreach(update);              // per-shard model-update
    comm_->allGatherParams();            // distribute param value shards back

    // Re-add the error residual from previous quantization,
    // then re-quantize the model back and update the error residual
    if (options_->get<size_t>("quantize-bits") > 0)
      comm_->foreach(quantizeModel);
  }
  else
    LOG(info, "[training] skipping {}-th update due to loss being {}", scheduler_->numberOfBatches(), localLoss.loss);

  if(scheduler_) {
    // track and log localLoss
    scheduler_->update(localLoss, numReadBatches, batchSize, batchTrgWords, mpi_);

    // save intermediate model (and optimizer state) to file
    if(scheduler_->saving())
      save();

    // process valid data set
    // This may save a model as well.
    if(scheduler_->validating()) {
      swapParamsAvg();
      if (isMainProcess())
        scheduler_->validate(graphs_);
      swapParamsAvg();
    }
  }
}

void SyncGraphGroup::load() /*override*/ {
  validate();

  // This function loads the main parameters in the graphs.
  // In case of exponential smoothing, we also need to restore paramsAvg_.
  // That is done lazily inside initializeAvg(), see there.

  if(!options_->get<bool>("no-reload")) {
    std::string name = options_->get<std::string>("model");

    if(filesystem::exists(name)) {
      if(scheduler_)
        scheduler_->load(name);

      std::string nameGraph = name;
      std::string suffix = name.substr(name.size() - 4);
      ABORT_IF(suffix != ".npz" && suffix != ".bin", "Unknown model suffix {}", suffix);

      if(mvAvg_ && filesystem::exists(name + ".orig" + suffix))
        // Load the original parameters from model.npz.orig.npz
        nameGraph += ".orig" + suffix;

      size_t i = 0;
      for(auto graph : graphs_)
        builders_[i++]->load(graph, nameGraph); // we just load it N times from disk (it'll be in disk cache after the first)

      // @TODO: probably we want to have the list of DeviceIds as an attribute
      std::vector<Ptr<Backend>> backends;
      for(auto graph : graphs_)
        backends.push_back(graph->getBackend());
      shardOpt_[0]->load(name + ".optimizer.npz", shardOpt_, backends, // keep npz suffix for optimize checkpoint
        [&](const std::vector<float>& optimizerStateVector, const OptimizerBase::ScatterStateSetFunc& setShardFn) {
          comm_->scatterState(optimizerStateVector, setShardFn);
        });
      LOG(info, "[training] Model reloaded from {}", name);
    } else if(options_->hasAndNotEmpty("pretrained-model")) {
      std::string nameInit = options_->get<std::string>("pretrained-model");
      LOG(info,
          "[training] Initializing model weights with the pre-trained model {}",
          nameInit);

      size_t i = 0;
      for(auto graph : graphs_)
        builders_[i++]->load(graph, nameInit, false);
    }
  }
}

void SyncGraphGroup::save(bool final) /*override*/ {
  // validate(); @TODO: get rid of this everywhere (SyncGraphGroup)
  barrier(); // (for better grouping of log messages)
  // do final validation
  if(final && scheduler_) {
    // bring the smoothed model in
    // Note that it is sharded. For multi-node, it is sharded over multiple machines, so this is a network access.
    // Also note that the swap must run on all MPI processes concurrently, although only one actually validates.
    swapParamsAvg();
    if (isMainProcess()) // in multi-node, only first MPI process saves the model (they are all identical)
      scheduler_->validate(graphs_, true);
    swapParamsAvg();
  }

  // @TODO: put all this in one place, in new branch this is already localized in one place and class, this is a quick hack which will be
  // done better after the next merge. Not doing this in other graph_groups as this would only make the merge harder.
  // Determine model suffix *.npz or *.bin, then use the same suffix for all following models saved.
  std::string name = options_->get<std::string>("model");
  std::string suffix = name.substr(name.size() - 4);
  ABORT_IF(suffix != ".npz" && suffix != ".bin", "Unknown model suffix {}", suffix);

  barrier(); // (for better grouping of log messages)
  // if smoothing then save original (unsmoothed) parameters as well
  if(mvAvg_ && paramsAvg_.size() > 0 && isMainProcess()) // only save from one MPI process
    // Save the original parameters in model.npz.orig.npz
    builders_[0]->save(graphs_[0], name + ".orig" + suffix, true);

  // Temporarily switch to the averaged parameters
  // Note: the smoothed model is sharded across GPUs, and across MPI processes if applicable. This brings it into MPI process[*].device[*]
  swapParamsAvg();

  // save main model file
  if (isMainProcess()) { // only save from one MPI process
    // if not overwrite then save a copy with number of updates in the model pathname
    if(!options_->get<bool>("overwrite") && !final) {
      std::string numberOfBatches
          = scheduler_ ? std::to_string(scheduler_->numberOfBatches())
                       : "unknown";
      std::string nameOverwrite = name;
      nameOverwrite.replace(name.size() - 4, 4, ".iter" + numberOfBatches + suffix); // @TODO: use insert?
      builders_[0]->save(graphs_[0], nameOverwrite);
    }
    // save main model file
    builders_[0]->save(graphs_[0], name, true);
    // save scheduler-related state
    if (scheduler_)
      scheduler_->save(name);
  }

  // Switch back to the original parameters
  swapParamsAvg();

  barrier(); // (for better grouping of log messages)

  // persist optimizer state
  shardOpt_[0]->save(name + ".optimizer.npz", shardOpt_,
    [&](const OptimizerBase::GatherStateGetFunc& getShardFn) {
      return comm_->gatherState(getShardFn);
    },
    isMainProcess());

  barrier(); // (for better grouping of log messages)
}

void SyncGraphGroup::finalize() /*override*/ {
  validate();
  Base::finalize();
}

}  // namespace marian


text:
<!DOCTYPE html>

    <title>Getting a Sorted List of Words</title>
    <link href="Content/themes/base/jquery-ui.css" rel="stylesheet" />

        Product:
        <input id="tags" />

Type "sq" or "ac"


text:
ACCEPTED

#### According to
Index Fungorum

#### Published in
null

#### Original name
Ustilago apiculata Ellis & Galloway

### Remarks
null


text:
package dk.brics.tajs.util;

import java.io.Serializable;

/**
 * Pair of objects.
 */
public class Pair<F, S> implements Serializable {

    private F first;

    private S second;

    private Pair(F fst, S snd) {
        this.first = fst;
        this.second = snd;
    }

    /**
     * Constructs a new pair.
     */
    public static <F, S> Pair<F, S> make(F fst, S snd) {
        return new Pair<>(fst, snd);
    }

    @Override
    public int hashCode() {
        final int prime = 31;
        int result = 1;
        result = prime * result + ((first == null) ? 0 : first.hashCode());
        result = prime * result + ((second == null) ? 0 : second.hashCode());
        return result;
    }

    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;
        if (obj == null)
            return false;
        if (getClass() != obj.getClass())
            return false;
        Pair<?, ?> other = (Pair<?, ?>) obj;
        if (first == null) {
            if (other.first != null)
                return false;
        } else if (!first.equals(other.first))
            return false;
        if (second == null) {
            if (other.second != null)
                return false;
        } else if (!second.equals(other.second))
            return false;
        return true;
    }

    /**
     * Returns the first object.
     */
    public F getFirst() {
        return first;
    }

    /**
     * Returns the second object.
     */
    public S getSecond() {
        return second;
    }

    @Override
    public String toString() {
        return "(" + first + ", " + second + ")";
    }
}


text:
Ruby JAM
--------

A time/buzz/topic tracker for Just A Minute games.

Setup
-----

* Install Ruby
* Install Rubygems
* bundle install
* Plug in the controllers from the playstation game, Buzz.

Configuration
-------------

To set up for a game of Just A Minute:

* Check screen resolution, if necessary add it to config/window.yml
* Put your desired subjects in to a file in topics/
* modify config/config.yml (more info in
  * window\_type - a config from window.yml, including resolution and a full screen flag.
  * topics - a file from topics/ with the topics you'll play with
* Configure panellists in config/panellists.yml
  * name1 - line one of their name, usually a first name
  * name2 - line two of their name, usually a surname
  * buzzer - the name of an audio file in audio/ which will sound as their buzzer

More detailed information in /docs/game/setup.md

Operation
---------

`ruby game.rb`

By default, this will restore from the last game data.
You probably don't want this
To start from a clean game, hit some keys, then press enter.

Keys
----

|  key       | control          |
|------------|------------------|
|   .        | start/stop theme music |
| space      | start timer      |
|   P        | Next subject     |
|   O        | Previous subject |
| 1, 2, 3, 4 | Simulate buzz (testing only) |
| Q, W, E, R | Increase score   |
| A, S, D, F | Decrease score   |
| esc        | Exit             |

Docs
----

There's some more in-depth information available in the docs/ folder.

/docs/game/cheat_sheet.md

This is information about how to play the game of Just A Minute. Distribute this
to all panellists prior to a game.

/docs/game/setup.md

This is a setup checklist to ensure everything is ready for the show.
Including detailed information about the config files.

/docs/tech/reporters.md

Explains the purpose of ruby-jam reporters. What is required to make one
and how they can be used and configured.


text:
@interface ViewController ()
<
UITableViewDelegate,
UITableViewDataSource
>

@property (strong, nonatomic) UITableView *tableView;
@end

@implementation ViewController

- (void)viewDidLoad {
    [super viewDidLoad];

}
#pragma mark - UITableViewDataSource

- (NSInteger)numberOfSectionsInTableView:(UITableView *)tableView
{
    return 0;
}
- (NSInteger)tableView:(UITableView *)tableView numberOfRowsInSection:(NSInteger)section
{
    return 10;
}

#pragma mark - lazy
- (UITableView *)tableView
{
    return GC_LAZY(_tableView, ({
        UITableView *view = [[UITableView alloc] init];
        view.backgroundColor = [UIColor whiteColor];
        view.separatorStyle = UITableViewCellSeparatorStyleNone;
        view.showsVerticalScrollIndicator = NO;
        view.dataSource = self;
        view.delegate = self;
        [self.view addSubview:view];
        view;
    }));
}

@end


text:
using Igneel.IA.Resources;

namespace Igneel.IA.ML.Trainers
{
    public interface ITrainingExample
    {
        ComputeBuffer Input { get; }

        ComputeBuffer Output { get; }
    }

}


text:
#include "PacketAssembly.h"
#include "ErasureCode.h"

#include <algorithm>

namespace DirectRemote {

bool PacketAssembly::processFrame(const unsigned char *bytes, int byteCount,
                                  float eccPacketsPerDataPacket) {
  data.clear();
  ecc.clear();

  const int msgCount = 1 + std::max(0, byteCount - 1) / MAX_MESSAGE_SIZE;

  for (int i = 0, iMsg = 0; i < std::max(1, byteCount);
       i += MAX_MESSAGE_SIZE, iMsg++) {
    int msgSize = std::min(MAX_MESSAGE_SIZE, byteCount - i);

    if (!processMessageInternal(bytes + i, msgSize, eccPacketsPerDataPacket,
                                dataPerMessage, eccPerMessage, eccRawPerMessage,
                                packetPtrsPerMessage)) {
      return false;
    }

    for (auto &p : dataPerMessage) {
      p.msgCount = static_cast<uint64_t>(msgCount);
      p.msgIndex = static_cast<uint64_t>(iMsg);

      data.push_back(p);
    }

    for (auto &p : eccPerMessage) {
      p.msgCount = static_cast<uint64_t>(msgCount);
      p.msgIndex = static_cast<uint64_t>(iMsg);

      ecc.push_back(p);
    }
  }

  return true;
}

bool PacketAssembly::processMessage(const unsigned char *bytes, int byteCount,
                                    float eccPacketsPerDataPacket) {
  dataPerMessage.clear();
  eccPerMessage.clear();
  eccRawPerMessage.clear();
  packetPtrsPerMessage.clear();

  return processMessageInternal(bytes, byteCount, eccPacketsPerDataPacket, data,
                                ecc, eccRawPerMessage, packetPtrsPerMessage);
}

bool PacketAssembly::processMessageInternal(
    const unsigned char *bytes, size_t byteCount, float eccPacketsPerDataPacket,
    std::vector<UdpChunk> &outData, std::vector<UdpChunk> &outEcc,
    std::vector<unsigned char> &eccRaw,
    std::vector<const unsigned char *> &packetPtrs) {
  outData.clear();
  outEcc.clear();
  eccRaw.clear();
  packetPtrs.clear();

  UdpChunk chunk = {};
  const size_t chunkCount = 1u +
                            (std::max(static_cast<size_t>(1u), byteCount) - 1) /
                                sizeof(chunk.data.bytes);

  if (chunkCount > 127) {
    return false;
  }

  for (size_t chunkIndex = 0, offset = 0; chunkIndex < chunkCount;
       chunkIndex++, offset += sizeof(chunk.data.bytes)) {
    chunk.chunkCount = chunkCount;
    chunk.chunkIndex = chunkIndex;
    chunk.isEccChunk = false;

    chunk.data.isConnected = 1;
    chunk.data.size =
        std::max(static_cast<int64_t>(0),
                 std::min(static_cast<int64_t>(byteCount - offset),
                          static_cast<int64_t>(sizeof(chunk.data.bytes))));

    memcpy(chunk.data.bytes, bytes + offset, chunk.data.size);

    outData.push_back(chunk);
  }

  static_assert(((CHUNK_ECC_SIZE % 8) == 0),
                "outEcc blocks need to be a multiple of 8.");

  packetPtrs.resize(chunkCount);

  outEcc.resize(std::max(
      1u, std::min(128u, static_cast<unsigned>(chunkCount *
                                               eccPacketsPerDataPacket))));
  eccRaw.resize(outEcc.size() * CHUNK_ECC_SIZE);

  for (size_t i = 0; i < outData.size(); i++) {
    packetPtrs[i] = outData[i].ecc.bytes;
  }

  cauchy_256_encode(static_cast<int>(chunkCount),
                    static_cast<int>(outEcc.size()),
                    reinterpret_cast<const unsigned char **>(packetPtrs.data()),
                    eccRaw.data(), CHUNK_ECC_SIZE);

  for (size_t i = 0; i < outEcc.size(); i++) {
    auto &eccChunk = outEcc[i];
    memcpy(eccChunk.ecc.bytes, &eccRaw[i * CHUNK_ECC_SIZE], CHUNK_ECC_SIZE);

    eccChunk.chunkCount = outEcc.size();
    eccChunk.chunkIndex = i;
    eccChunk.isEccChunk = true;
  }

  return true;
}
}  // namespace DirectRemote


text:
from __future__ import print_function
from builtins import range
import sys, os
sys.path.insert(1,"../../../")
import h2o
from h2o.estimators.glm import H2OGeneralizedLinearEstimator
from tests import pyunit_utils

# Michalk has caught problems with glm beta constrains.
def test_glm_beta_constraints_mk():
    df = h2o.import_file(pyunit_utils.locate("smalldata/kaggle/CreditCard/creditcard_train_cat.csv"),
                         col_types={"DEFAULT_PAYMENT_NEXT_MONTH": "enum"})
    lb_limit_bal = 0.0001
    constraints = h2o.H2OFrame({'names':["LIMIT_BAL"], 'lower_bounds': [lb_limit_bal], 'upper_bounds': [1e6]})
    # make sure we have the column names in expected order, the backend does weird things when the order is different
    constraints = constraints[["names", "lower_bounds", "upper_bounds"]]
    glm_beta = H2OGeneralizedLinearEstimator(model_id="beta_glm", beta_constraints=constraints, seed=42)
    glm_beta.train(y="DEFAULT_PAYMENT_NEXT_MONTH", training_frame=df)
    glm_coeff = glm_beta.coef()
    assert glm_coeff["LIMIT_BAL"] >= lb_limit_bal or glm_coeff["LIMIT_BAL"]==0

if __name__ == "__main__":
    pyunit_utils.standalone_test(test_glm_beta_constraints_mk)
else:
    test_glm_beta_constraints_mk()


text:
To generate all the docs, just run `make docs` from the root of the repository.
This will convert the distributed Markdown docs to HTML and generate HTML doc
for the 'std' and 'extra' libraries.

To generate HTML documentation from one source file/crate, do something like:

~~~~
rustdoc --output html-doc/ --output-format html ../src/libstd/path.rs
~~~~

(This, of course, requires a working build of the `rustdoc` tool.)

## Additional notes

To generate an HTML version of a doc from Markdown manually, you can do
something like:

~~~~
rustdoc reference.md
~~~~

(`reference.md` being the Rust Reference Manual.)

An overview of how to use the `rustdoc` command is available [in the docs][1].
Further details are available from the command line by with `rustdoc --help`.

[1]: https://github.com/rust-lang/rust/blob/master/src/doc/trpl/documentation.md