text:
export function assignGlobalReference (newInstance, referenceKey) {
  if (!newInstance.constructor || (global[referenceKey] && !global[referenceKey].constructor)) {
    throw new Error('Assigning to global reference is only supported for class instances')
  } else if (newInstance.constructor && !global[referenceKey]) {
    global[referenceKey] = newInstance
  } else if (!(
    newInstance instanceof global[referenceKey].constructor ||
    global[referenceKey] instanceof newInstance.constructor
  )) {
    throw new Error(`Not a ${global[referenceKey].constructor.name} instance`)
  }

  const oldInstance = Object.create(global[referenceKey])

  for (const prop in global[referenceKey]) {
    oldInstance[prop] = global[referenceKey][prop]
    delete global[referenceKey][prop]
  }

  for (const prop of Object.getOwnPropertySymbols(global[referenceKey])) {
    oldInstance[prop] = global[referenceKey][prop]
    delete global[referenceKey][prop]
  }

  for (const prop in newInstance) {
    global[referenceKey][prop] = newInstance[prop]
  }

  for (const prop of Object.getOwnPropertySymbols(newInstance)) {
    global[referenceKey][prop] = newInstance[prop]
  }

  return oldInstance
}

export function assignGlobalConsola (newConsola) {
  return assignGlobalReference(newConsola, 'consola')
}


text:
Deface::Override.new(:virtual_path => "spree/shared/_products",
                     :replace      => "[data-hook='products_list_item']",
                     :name         => "products",
                     :partial      => "white_label/shared/products_list_item",
                     :original     => '0b2d55cecb52786fb16a16a5250c65fbc5ef7ab6')

redpajama_set_name: RedPajamaGithub


text:
package com.peterphi.std.util;

import com.peterphi.std.types.HybridIterator;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Stream;

public class ListUtility
{
	public static <T> List<T> list(final T[] items)
	{
		return new ArrayList<T>(Arrays.asList(items));
	}

	public static <T> List<T> list(final List<T> items)
	{
		return new ArrayList<T>(items);
	}

	/**
	 * Converts an Iterable into a List
	 *
	 * @param iterable
	 * @return
	 */
	public static <T> List<T> list(Iterable<T> iterable)
	{
		List<T> list = new ArrayList<T>();

		for (T item : iterable)
		{
			list.add(item);
		}
		return list;
	}

	/**
	 * Returns true if a given collection is null or empty
	 *
	 * @param collection optional collection
	 * @return null || isEmpty
	 */
	public boolean empty(final Collection<?> collection)
	{
		return (collection == null || collection.isEmpty());
	}

	public static <T> T last(final List<T> src)
	{
		if (src == null || src.isEmpty())
			return null;
		else
			return src.get(src.size() - 1);
	}

	public static <T> T last(final Stream<T> stream)
	{
		return stream.reduce((prev, cur) -> cur).orElse(null);
	}

	/**
	 * Returns the first element of an Iterable (or null if the list is empty)
	 *
	 * @param list
	 * @return
	 */
	public static <T> T head(Iterable<T> list)
	{
		if (list == null)
			return null;

		final Iterator<T> it = list.iterator();

		if (it.hasNext())
			return it.next();
		else
			return null;
	}

	/**
	 * Returns a sublist containing all the items in the list after the first
	 *
	 * @param list
	 * @return
	 */
	public static <T> List<T> tail(List<T> list)
	{
		if (list == null || list.isEmpty())
			return Collections.emptyList();
		else
			return list.subList(1, list.size());
	}

	public static <T> HybridIterator<T> iterate(final Iterator<T> items)
	{
		return new HybridIterator<T>(items);
	}

	public static <T> HybridIterator<T> iterate(final Enumeration<T> items)
	{
		return new HybridIterator<T>(items);
	}
}


text:
import * as s from '../src/store';
import * as tds from './todosState';
import * as af from '../src/actionFactory';
import * as d from '../src/debug';

describe('actionFactory', () => {
    let debugCallback: d.debugCallbackType

    beforeEach(() => {
        resetStore();
        af.bootstrap(null);
        debugCallback = jasmine.createSpy('debugCallback');
        d.bootstrap(debugCallback);
    });

    describe('bootstrap', () => {
        it('reports initialization when debug has been enabled.', () => {
            af.bootstrap(null);

            expect(debugCallback).toHaveBeenCalledWith('Action factory has been initialized.', undefined);
        });
    });

    describe('array by cursor factory', () => {
        let renderCallback: () => void;

        beforeEach(() => {
            renderCallback = jasmine.createSpy('render');
            af.bootstrap(renderCallback);
        });

        it('changes state in array', () => {
            givenTodosStore({
                todos: [{ done: false, name: 'First todo' }, { done: false, name: 'Second todo' }]
            });

            let testAction = af.createAction<tds.ITodo, tds.ITodoParams>(
                {
                    create: (params: tds.ITodoParams) => {
                        return { key: `todos.${params.index}` };
                    }
                },
                (_state: tds.ITodo, params: tds.ITodoParams) => { return params.todo }
            );
            testAction({ index: 1, todo: { done: false, name: 'New second todo' } });

            expect(getTodos()[1].done).toBeFalsy();
            expect(getTodos()[1].name).toBe('New second todo');
        });

        it('changes nested state on existing index', () => {
            givenTodosStore({
                todos: [{ done: false, name: 'First todo' }, { done: false, name: 'Second todo' }]
            });

            let testAction = af.createAction<boolean, number>({ create: (index) => { return { key: `todos.${index}.done` } } }, () => { return true; });
            testAction(1);

            expect(getTodos()[1].done).toBeTruthy();
        });

        function givenTodosStore(state: tds.ITodosState) {
            s.setState(s.rootCursor, state);
        }

        function getTodos(): tds.ITodo[] {
            return s.getState(tds.todosCursor);
        }
    });

    describe('createAction', () => {
        describe('when action threw during handling and catching have not been enabled', () => {
            let renderCallback: () => void;
            let throwingAction: af.IAction<{}>;
            beforeEach(() => {
                renderCallback = jasmine.createSpy('render');
                af.bootstrap(renderCallback, false);
                throwingAction = af.createAction(NestedCursorTestFixture, () => {
                    throw 'MyDummyException';
                });
            });

            it('throws', () => {
                expect(throwingAction).toThrow();
            });
        });

        describe('when action threw during handling and catching have been enabled', () => {
            let renderCallback: () => void;
            let throwingAction: af.IAction<{}>;
            beforeEach(() => {
                renderCallback = jasmine.createSpy('render');
                af.bootstrap(renderCallback, true);
                throwingAction = af.createAction(NestedCursorTestFixture, () => {
                    throw 'MyDummyException';
                });
            });

            it('does not throw', () => {
                expect(throwingAction).not.toThrow();
            });

            it('logs error', () => {
                throwingAction();

                expect(debugCallback).toHaveBeenCalledWith('Action factory has been initialized.', undefined);
            });
        });

        describe('when renderCallback has not been set', () => {
            it('does not throw if action has been only declared.', () => {
                let testAction = af.createAction(NestedCursorTestFixture, (state: INestedState) => state);
                expect(testAction).not.toBeUndefined();
            });

            it('throws if key does not exist', () => {
                expect(() => {
                    let testAction = af.createAction(NestedCursorTestFixture, () => { return { state: 'new nested state' }; });
                    testAction();
                }).toThrow('Render callback must be set before first usage through bootstrap(defaultState, () => { yourRenderCallback(); }).');
            });
        });

        describe('when renderCallback has been set', () => {
            let renderCallback: () => void;

            beforeEach(() => {
                renderCallback = jasmine.createSpy('render');
                af.bootstrap(renderCallback);
            });

            it('does not report current state when state has not been changed.', () => {
                givenStore(aState('nestedStateValue'));

                af.createAction(NestedCursorTestFixture, (state: INestedState) => state)();

                expect(debugCallback).not.toHaveBeenCalledWith('Global state has been changed.', undefined);
            });

            it('reports state changed when debug has been enabled.', () => {
                let newState = { state: 'newValue' };
                givenStore(aState('nestedStateValue'));

                af.createAction(NestedCursorTestFixture, () => newState)();

                expect(debugCallback).toHaveBeenCalledWith('Global state has been changed.', undefined);
            });

            it('does not call render callback when state has not been changed', () => {
                givenStore(aState('nestedStateValue'));

                let testAction = af.createAction(NestedCursorTestFixture, (state: INestedState) => state);
                testAction();

                expect(renderCallback).not.toHaveBeenCalled();
            });

            it('calls render callback when state has been changed', () => {
                givenStore(aState('nestedStateValue'));

                let testAction = af.createAction(NestedCursorTestFixture, () => { return { state: 'newValue' }; })
                testAction();

                expect(renderCallback).toHaveBeenCalled();
            });

            it('calls nested actions', () => {
                givenStore(aState('value'));

                const nestedAction2 = af.createAction(NestedCursorTestFixture, (state: INestedState) => {
                    return { state: `${state.state} -> newValueFromNestedAction2` };
                });

                const nestedAction = af.createAction(NestedCursorTestFixture, (state: INestedState) => {
                    nestedAction2();
                    return { state: `${state.state} -> newValueFromNestedAction` };
                });

                af.createAction(NestedCursorTestFixture, () => {
                    nestedAction();
                    return { state: 'newValue' };
                })();

                expect(s.getState(NestedCursorTestFixture).state)
                    .toBe('newValue -> newValueFromNestedAction -> newValueFromNestedAction2');
            });

            it('does not throw on immutability violation in no debug mode', () => {
                d.bootstrap(undefined);
                givenStore(aState('nestedStateValue'));
                const testAction = af.createAction(SomeCursorTestFixture, (state: ISomeState) => {
                    state.nested.state = state.nested.state + 'newValue';
                    return state;
                });

                expect(() => {
                    testAction();
                }).not.toThrow();
            });
        });
    });

    describe('createActions', () => {
        describe('when renderCallback has not been set', () => {
            it('throws if key does not exist', () => {
                expect(() => {
                    let testAction = af.createActions({
                        cursor: NestedCursorTestFixture,
                        handler: (state: INestedState): INestedState => state
                    });
                    testAction();
                }).toThrow('Render callback must be set before first usage through bootstrap(defaultState, () => { yourRenderCallback(); }).');
            });
        });

        describe('when renderCallback has been set', () => {
            let renderCallback: () => void;

            beforeEach(() => {
                renderCallback = jasmine.createSpy('render');
                af.bootstrap(renderCallback);
            });

            it('does not call render callback when all states have not been changed', () => {
                givenStore(aState('nestedStateValue'));

                let testAction = af.createActions(
                    {
                        cursor: NestedCursorTestFixture,
                        handler: (state: INestedState): INestedState => state
                    }, {
                        cursor: NestedCursorTestFixture,
                        handler: (state: INestedState): INestedState => state
                    });
                testAction();

                expect(renderCallback).not.toHaveBeenCalled();
            });

            it('calls render callback when one of states has been changed', () => {
                givenStore(aState('nestedStateValue'));

                let testAction = af.createActions(
                    {
                        cursor: SomeCursorTestFixture,
                        handler: (state: ISomeState): ISomeState => { return { nested: state.nested, state: 'newStateValue' } }
                    },
                    {
                        cursor: NestedCursorTestFixture,
                        handler: (): INestedState => { return { state: 'newNestedStateValue' }; }
                    });
                testAction();

                expect(renderCallback).toHaveBeenCalled();
            });

            it('uses action as simple setter when no handler defined', () => {
                givenStore(aState('nestedStateValue'));
                const testAction = af.createAction<string, string>(NestedStateCursorTestFixture);

                testAction('nestedStateValue');
                expect(renderCallback).not.toHaveBeenCalled();

                testAction('newNestedStateValue');
                expect(renderCallback).toHaveBeenCalled();
            });
        });
    });
});

function givenStore(state: IStateTestFixture) {
    s.setState(s.rootCursor, state);
}

function resetStore(withFreezing: boolean = false) {
    s.bootstrap({ some: { nested: { state: null } } }, withFreezing);
}

function aState(nestedState: string = 'aNestedState', state: string = 'aState'): IStateTestFixture {
    return { some: { nested: { state: nestedState }, state: state } };
}

interface IStateTestFixture extends s.IState {
    some: ISomeState;
}

interface ISomeState extends s.IState {
    nested: INestedState
    state: string;
}

interface INestedState extends s.IState {
    state: string;
}

var NestedCursorTestFixture: s.ICursor<INestedState> = {
    key: 'some.nested'
}

var NestedStateCursorTestFixture: s.ICursor<string> = {
    key: 'some.nested.state'
}

var SomeCursorTestFixture: s.ICursor<ISomeState> = {
    key: 'some'
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>org.drools</groupId>
    <artifactId>drools-wb-factmodel-editor</artifactId>
    <version>7.40.0-SNAPSHOT</version>
  </parent>

  <artifactId>drools-wb-factmodel-editor-backend</artifactId>
  <packaging>jar</packaging>

  <name>Drools Workbench - Fact Model Editor - Backend</name>
  <description>Drools Workbench - Fact Model Editor - Backend</description>

  <properties>
    <java.module.name>org.drools.wb.factmodel.editor.backend</java.module.name>
  </properties>

  <dependencies>
    <dependency>
      <groupId>org.kie.soup</groupId>
      <artifactId>kie-soup-project-datamodel-api</artifactId>
    </dependency>

    <dependency>
      <groupId>org.kie.soup</groupId>
      <artifactId>kie-soup-project-datamodel-commons</artifactId>
    </dependency>

    <dependency>
      <groupId>org.drools</groupId>
      <artifactId>drools-wb-factmodel-editor-api</artifactId>
    </dependency>

    <dependency>
      <groupId>org.kie.soup</groupId>
      <artifactId>kie-soup-commons</artifactId>
    </dependency>

    <dependency>
      <groupId>org.drools</groupId>
      <artifactId>drools-compiler</artifactId>
    </dependency>
  </dependencies>

</project>


text:
<?php

namespace app\models;

use Yii;

/**
 * This is the model class for table "sales_customer".
 *
 * @property integer $karyawan_id
 * @property integer $customer_id
 *
 * @property Karyawan $karyawan
 * @property Customer $customer
 */
class SalesCustomer extends \yii\db\ActiveRecord
{
    /**
     * @inheritdoc
     */
    public static function tableName()
    {
        return 'sales_customer';
    }

    /**
     * @inheritdoc
     */
    public function rules()
    {
        return [
            [['karyawan_id', 'customer_id'], 'required'],
            [['karyawan_id', 'customer_id'], 'integer']
        ];
    }

    /**
     * @inheritdoc
     */
    public function attributeLabels()
    {
        return [
            'karyawan_id' => 'Nama Karyawan',
            'customer_id' => 'Nama Customer',
        ];
    }

    /**
     * @return \yii\db\ActiveQuery
     */
    public function getKaryawan()
    {
        return $this->hasOne(Karyawan::className(), ['id' => 'karyawan_id']);
    }

    /**
     * @return \yii\db\ActiveQuery
     */
    public function getCustomer()
    {
        return $this->hasOne(Customer::className(), ['id' => 'customer_id']);
    }
}


text:
"""Implements a format decision state object that manages whitespace decisions.

Each token is processed one at a time, at which point its whitespace formatting
decisions are made. A graph of potential whitespace formattings is created,
where each node in the graph is a format decision state object. The heuristic
tries formatting the token with and without a newline before it to determine
which one has the least penalty. Therefore, the format decision state object for
each decision needs to be its own unique copy.

Once the heuristic determines the best formatting, it makes a non-dry run pass
through the code to commit the whitespace formatting.

  FormatDecisionState: main class exported by this module.
"""

from yapf.pytree import split_penalty
from yapf.yapflib import logical_line
from yapf.yapflib import object_state
from yapf.yapflib import style
from yapf.yapflib import subtypes

class FormatDecisionState(object):
  """The current state when indenting a logical line.

  The FormatDecisionState object is meant to be copied instead of referenced.

  Attributes:
    first_indent: The indent of the first token.
    column: The number of used columns in the current line.
    line: The logical line we're currently processing.
    next_token: The next token to be formatted.
    paren_level: The level of nesting inside (), [], and {}.
    lowest_level_on_line: The lowest paren_level on the current line.
    stack: A stack (of _ParenState) keeping track of properties applying to
      parenthesis levels.
    comp_stack: A stack (of ComprehensionState) keeping track of properties
      applying to comprehensions.
    param_list_stack: A stack (of ParameterListState) keeping track of
      properties applying to function parameter lists.
    ignore_stack_for_comparison: Ignore the stack of _ParenState for state
      comparison.
    column_limit: The column limit specified by the style.
  """

  def __init__(self, line, first_indent):
    """Initializer.

    Initializes to the state after placing the first token from 'line' at
    'first_indent'.

    Arguments:
      line: (LogicalLine) The logical line we're currently processing.
      first_indent: (int) The indent of the first token.
    """
    self.next_token = line.first
    self.column = first_indent
    self.line = line
    self.paren_level = 0
    self.lowest_level_on_line = 0
    self.ignore_stack_for_comparison = False
    self.stack = [_ParenState(first_indent, first_indent)]
    self.comp_stack = []
    self.param_list_stack = []
    self.first_indent = first_indent
    self.column_limit = style.Get('COLUMN_LIMIT')

  def Clone(self):
    """Clones a FormatDecisionState object."""
    new = FormatDecisionState(self.line, self.first_indent)
    new.next_token = self.next_token
    new.column = self.column
    new.line = self.line
    new.paren_level = self.paren_level
    new.line.depth = self.line.depth
    new.lowest_level_on_line = self.lowest_level_on_line
    new.ignore_stack_for_comparison = self.ignore_stack_for_comparison
    new.first_indent = self.first_indent
    new.stack = [state.Clone() for state in self.stack]
    new.comp_stack = [state.Clone() for state in self.comp_stack]
    new.param_list_stack = [state.Clone() for state in self.param_list_stack]
    return new

  def __eq__(self, other):
    # Note: 'first_indent' is implicit in the stack. Also, we ignore 'previous',
    # because it shouldn't have a bearing on this comparison. (I.e., it will
    # report equal if 'next_token' does.)
    return (self.next_token == other.next_token and
            self.column == other.column and
            self.paren_level == other.paren_level and
            self.line.depth == other.line.depth and
            self.lowest_level_on_line == other.lowest_level_on_line and
            (self.ignore_stack_for_comparison or
             other.ignore_stack_for_comparison or self.stack == other.stack and
             self.comp_stack == other.comp_stack and
             self.param_list_stack == other.param_list_stack))

  def __ne__(self, other):
    return not self == other

  def __hash__(self):
    return hash((self.next_token, self.column, self.paren_level,
                 self.line.depth, self.lowest_level_on_line))

  def __repr__(self):
    return ('column::%d, next_token::%s, paren_level::%d, stack::[\n\t%s' %
            (self.column, repr(self.next_token), self.paren_level,
             '\n\t'.join(repr(s) for s in self.stack) + ']'))

  def CanSplit(self, must_split):
    """Determine if we can split before the next token.

    Arguments:
      must_split: (bool) A newline was required before this token.

    Returns:
      True if the line can be split before the next token.
    """
    current = self.next_token
    previous = current.previous_token

    if current.is_pseudo:
      return False

    if (not must_split and subtypes.DICTIONARY_KEY_PART in current.subtypes and
        subtypes.DICTIONARY_KEY not in current.subtypes and
        not style.Get('ALLOW_MULTILINE_DICTIONARY_KEYS')):
      # In some situations, a dictionary may be multiline, but pylint doesn't
      # like it. So don't allow it unless forced to.
      return False

    if (not must_split and subtypes.DICTIONARY_VALUE in current.subtypes and
        not style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE')):
      return False

    if previous and previous.value == '(' and current.value == ')':
      # Don't split an empty function call list if we aren't splitting before
      # dict values.
      token = previous.previous_token
      while token:
        prev = token.previous_token
        if not prev or prev.name not in {'NAME', 'DOT'}:
          break
        token = token.previous_token
      if token and subtypes.DICTIONARY_VALUE in token.subtypes:
        if not style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE'):
          return False

    if previous and previous.value == '.' and current.value == '.':
      return False

    return current.can_break_before

  def MustSplit(self):
    """Returns True if the line must split before the next token."""
    current = self.next_token
    previous = current.previous_token

    if current.is_pseudo:
      return False

    if current.must_break_before:
      return True

    if not previous:
      return False

    if style.Get('SPLIT_ALL_COMMA_SEPARATED_VALUES') and previous.value == ',':
      return True

    if (style.Get('SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES') and
        previous.value == ','):
      # Avoid breaking in a container that fits in the current line if possible
      opening = _GetOpeningBracket(current)

      # Can't find opening bracket, behave the same way as
      # SPLIT_ALL_COMMA_SEPARATED_VALUES.
      if not opening:
        return True

      if current.is_comment:
        # Don't require splitting before a comment, since it may be related to
        # the current line.
        return False

      # Allow the fallthrough code to handle the closing bracket.
      if current != opening.matching_bracket:
        # If the container doesn't fit in the current line, must split
        return not self._ContainerFitsOnStartLine(opening)

    if (self.stack[-1].split_before_closing_bracket and
        (current.value in '}]' and style.Get('SPLIT_BEFORE_CLOSING_BRACKET') or
         current.value in '}])' and style.Get('INDENT_CLOSING_BRACKETS'))):
      # Split before the closing bracket if we can.
      if subtypes.SUBSCRIPT_BRACKET not in current.subtypes:
        return current.node_split_penalty != split_penalty.UNBREAKABLE

    if (current.value == ')' and previous.value == ',' and
        not _IsSingleElementTuple(current.matching_bracket)):
      return True

    # Prevent splitting before the first argument in compound statements
    # with the exception of function declarations.
    if (style.Get('SPLIT_BEFORE_FIRST_ARGUMENT') and
        _IsCompoundStatement(self.line.first) and
        not _IsFunctionDef(self.line.first)):
      return False

    ###########################################################################
    # List Splitting
    if (style.Get('DEDENT_CLOSING_BRACKETS') or
        style.Get('INDENT_CLOSING_BRACKETS') or
        style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')):
      bracket = current if current.ClosesScope() else previous
      if subtypes.SUBSCRIPT_BRACKET not in bracket.subtypes:
        if bracket.OpensScope():
          if style.Get('COALESCE_BRACKETS'):
            if current.OpensScope():
              # Prefer to keep all opening brackets together.
              return False

          if (not _IsLastScopeInLine(bracket) or
              logical_line.IsSurroundedByBrackets(bracket)):
            last_token = bracket.matching_bracket
          else:
            last_token = _LastTokenInLine(bracket.matching_bracket)

          if not self._FitsOnLine(bracket, last_token):
            # Split before the first element if the whole list can't fit on a
            # single line.
            self.stack[-1].split_before_closing_bracket = True
            return True

        elif (style.Get('DEDENT_CLOSING_BRACKETS') or
              style.Get('INDENT_CLOSING_BRACKETS')) and current.ClosesScope():
          # Split before and dedent the closing bracket.
          return self.stack[-1].split_before_closing_bracket

    if (style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') and
        current.is_name):
      # An expression that's surrounded by parens gets split after the opening
      # parenthesis.
      def SurroundedByParens(token):
        """Check if it's an expression surrounded by parentheses."""
        while token:
          if token.value == ',':
            return False
          if token.value == ')':
            return not token.next_token
          if token.OpensScope():
            token = token.matching_bracket.next_token
          else:
            token = token.next_token
        return False

      if (previous.value == '(' and not previous.is_pseudo and
          not logical_line.IsSurroundedByBrackets(previous)):
        pptoken = previous.previous_token
        if (pptoken and not pptoken.is_name and not pptoken.is_keyword and
            SurroundedByParens(current)):
          return True

    if (current.is_name or current.is_string) and previous.value == ',':
      # If the list has function calls in it and the full list itself cannot
      # fit on the line, then we want to split. Otherwise, we'll get something
      # like this:
      #
      #     X = [
      #         Bar(xxx='some string',
      #             yyy='another long string',
      #             zzz='a third long string'), Bar(
      #                 xxx='some string',
      #                 yyy='another long string',
      #                 zzz='a third long string')
      #     ]
      #
      # or when a string formatting syntax.
      func_call_or_string_format = False
      tok = current.next_token
      if current.is_name:
        while tok and (tok.is_name or tok.value == '.'):
          tok = tok.next_token
        func_call_or_string_format = tok and tok.value == '('
      elif current.is_string:
        while tok and tok.is_string:
          tok = tok.next_token
        func_call_or_string_format = tok and tok.value == '%'
      if func_call_or_string_format:
        open_bracket = logical_line.IsSurroundedByBrackets(current)
        if open_bracket:
          if open_bracket.value in '[{':
            if not self._FitsOnLine(open_bracket,
                                    open_bracket.matching_bracket):
              return True
          elif tok.value == '(':
            if not self._FitsOnLine(current, tok.matching_bracket):
              return True

    if (current.OpensScope() and previous.value == ',' and
        subtypes.DICTIONARY_KEY not in current.next_token.subtypes):
      # If we have a list of tuples, then we can get a similar look as above. If
      # the full list cannot fit on the line, then we want a split.
      open_bracket = logical_line.IsSurroundedByBrackets(current)
      if (open_bracket and open_bracket.value in '[{' and
          subtypes.SUBSCRIPT_BRACKET not in open_bracket.subtypes):
        if not self._FitsOnLine(current, current.matching_bracket):
          return True

    ###########################################################################
    # Dict/Set Splitting
    if (style.Get('EACH_DICT_ENTRY_ON_SEPARATE_LINE') and
        subtypes.DICTIONARY_KEY in current.subtypes and not current.is_comment):
      # Place each dictionary entry onto its own line.
      if previous.value == '{' and previous.previous_token:
        opening = _GetOpeningBracket(previous.previous_token)
        if (opening and opening.value == '(' and opening.previous_token and
            opening.previous_token.is_name):
          # This is a dictionary that's an argument to a function.
          if (self._FitsOnLine(previous, previous.matching_bracket) and
              previous.matching_bracket.next_token and
              (not opening.matching_bracket.next_token or
               opening.matching_bracket.next_token.value != '.') and
              _ScopeHasNoCommas(previous)):
            # Don't split before the key if:
            #   - The dictionary fits on a line, and
            #   - The function call isn't part of a builder-style call and
            #   - The dictionary has one entry and no trailing comma
            return False
      return True

    if (style.Get('SPLIT_BEFORE_DICT_SET_GENERATOR') and
        subtypes.DICT_SET_GENERATOR in current.subtypes):
      # Split before a dict/set generator.
      return True

    if (subtypes.DICTIONARY_VALUE in current.subtypes or
        (previous.is_pseudo and previous.value == '(' and
         not current.is_comment)):
      # Split before the dictionary value if we can't fit every dictionary
      # entry on its own line.
      if not current.OpensScope():
        opening = _GetOpeningBracket(current)
        if not self._EachDictEntryFitsOnOneLine(opening):
          return style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE')

    if previous.value == '{':
      # Split if the dict/set cannot fit on one line and ends in a comma.
      closing = previous.matching_bracket
      if (not self._FitsOnLine(previous, closing) and
          closing.previous_token.value == ','):
        self.stack[-1].split_before_closing_bracket = True
        return True

    ###########################################################################
    # Argument List Splitting
    if (style.Get('SPLIT_BEFORE_NAMED_ASSIGNS') and not current.is_comment and
        subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in current.subtypes):
      if (previous.value not in {'=', ':', '*', '**'} and
          current.value not in ':=,)' and not _IsFunctionDefinition(previous)):
        # If we're going to split the lines because of named arguments, then we
        # want to split after the opening bracket as well. But not when this is
        # part of a function definition.
        if previous.value == '(':
          # Make sure we don't split after the opening bracket if the
          # continuation indent is greater than the opening bracket:
          #
          #  a(
          #      b=1,
          #      c=2)
          if (self._FitsOnLine(previous, previous.matching_bracket) and
              logical_line.IsSurroundedByBrackets(previous)):
            # An argument to a function is a function call with named
            # assigns.
            return False

          # Don't split if not required
          if (not style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') and
              not style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')):
            return False

          column = self.column - self.stack[-1].last_space
          return column > style.Get('CONTINUATION_INDENT_WIDTH')

        opening = _GetOpeningBracket(current)
        if opening:
          return not self._ContainerFitsOnStartLine(opening)

    if (current.value not in '{)' and previous.value == '(' and
        self._ArgumentListHasDictionaryEntry(current)):
      return True

    if style.Get('SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED'):
      # Split before arguments in a function call or definition if the
      # arguments are terminated by a comma.
      opening = _GetOpeningBracket(current)
      if opening and opening.previous_token and opening.previous_token.is_name:
        if previous.value in '(,':
          if opening.matching_bracket.previous_token.value == ',':
            return True

    if ((current.is_name or current.value in {'*', '**'}) and
        previous.value == ','):
      # If we have a function call within an argument list and it won't fit on
      # the remaining line, but it will fit on a line by itself, then go ahead
      # and split before the call.
      opening = _GetOpeningBracket(current)
      if (opening and opening.value == '(' and opening.previous_token and
          (opening.previous_token.is_name or
           opening.previous_token.value in {'*', '**'})):
        is_func_call = False
        opening = current
        while opening:
          if opening.value == '(':
            is_func_call = True
            break
          if (not (opening.is_name or opening.value in {'*', '**'}) and
              opening.value != '.'):
            break
          opening = opening.next_token

        if is_func_call:
          if (not self._FitsOnLine(current, opening.matching_bracket) or
              (opening.matching_bracket.next_token and
               opening.matching_bracket.next_token.value != ',' and
               not opening.matching_bracket.next_token.ClosesScope())):
            return True

    pprevious = previous.previous_token

    # A function call with a dictionary as its first argument may result in
    # unreadable formatting if the dictionary spans multiple lines. The
    # dictionary itself is formatted just fine, but the remaining arguments are
    # indented too far:
    #
    #     function_call({
    #         KEY_1: 'value one',
    #         KEY_2: 'value two',
    #     },
    #                   default=False)
    if (current.value == '{' and previous.value == '(' and pprevious and
        pprevious.is_name):
      dict_end = current.matching_bracket
      next_token = dict_end.next_token
      if next_token.value == ',' and not self._FitsOnLine(current, dict_end):
        return True

    if (current.is_name and pprevious and pprevious.is_name and
        previous.value == '('):

      if (not self._FitsOnLine(previous, previous.matching_bracket) and
          _IsFunctionCallWithArguments(current)):
        # There is a function call, with more than 1 argument, where the first
        # argument is itself a function call with arguments that does not fit
        # into the line.  In this specific case, if we split after the first
        # argument's opening '(', then the formatting will look bad for the
        # rest of the arguments. E.g.:
        #
        #     outer_function_call(inner_function_call(
        #         inner_arg1, inner_arg2),
        #                         outer_arg1, outer_arg2)
        #
        # Instead, enforce a split before that argument to keep things looking
        # good.
        if (style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') or
            style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')):
          return True

        opening = _GetOpeningBracket(current)
        if (opening and opening.value == '(' and opening.previous_token and
            (opening.previous_token.is_name or
             opening.previous_token.value in {'*', '**'})):
          is_func_call = False
          opening = current
          while opening:
            if opening.value == '(':
              is_func_call = True
              break
            if (not (opening.is_name or opening.value in {'*', '**'}) and
                opening.value != '.'):
              break
            opening = opening.next_token

          if is_func_call:
            if (not self._FitsOnLine(current, opening.matching_bracket) or
                (opening.matching_bracket.next_token and
                 opening.matching_bracket.next_token.value != ',' and
                 not opening.matching_bracket.next_token.ClosesScope())):
              return True

    if (previous.OpensScope() and not current.OpensScope() and
        not current.is_comment and
        subtypes.SUBSCRIPT_BRACKET not in previous.subtypes):
      if pprevious and not pprevious.is_keyword and not pprevious.is_name:
        # We want to split if there's a comment in the container.
        token = current
        while token != previous.matching_bracket:
          if token.is_comment:
            return True
          token = token.next_token
      if previous.value == '(':
        pptoken = previous.previous_token
        if not pptoken or not pptoken.is_name:
          # Split after the opening of a tuple if it doesn't fit on the current
          # line and it's not a function call.
          if self._FitsOnLine(previous, previous.matching_bracket):
            return False
        elif not self._FitsOnLine(previous, previous.matching_bracket):
          if len(previous.container_elements) == 1:
            return False

          elements = previous.container_elements + [previous.matching_bracket]
          i = 1
          while i < len(elements):
            if (not elements[i - 1].OpensScope() and
                not self._FitsOnLine(elements[i - 1], elements[i])):
              return True
            i += 1

          if (self.column_limit - self.column) / float(self.column_limit) < 0.3:
            # Try not to squish all of the arguments off to the right.
            return True
      else:
        # Split after the opening of a container if it doesn't fit on the
        # current line.
        if not self._FitsOnLine(previous, previous.matching_bracket):
          return True

    ###########################################################################
    # Original Formatting Splitting
    # These checks rely upon the original formatting. This is in order to
    # attempt to keep hand-written code in the same condition as it was before.
    # However, this may cause the formatter to fail to be idempotent.
    if (style.Get('SPLIT_BEFORE_BITWISE_OPERATOR') and current.value in '&|' and
        previous.lineno < current.lineno):
      # Retain the split before a bitwise operator.
      return True

    if (current.is_comment and
        previous.lineno < current.lineno - current.value.count('\n')):
      # If a comment comes in the middle of a logical line (like an if
      # conditional with comments interspersed), then we want to split if the
      # original comments were on a separate line.
      return True

    return False

  def AddTokenToState(self, newline, dry_run, must_split=False):
    """Add a token to the format decision state.

    Allow the heuristic to try out adding the token with and without a newline.
    Later on, the algorithm will determine which one has the lowest penalty.

    Arguments:
      newline: (bool) Add the token on a new line if True.
      dry_run: (bool) Don't commit whitespace changes to the FormatToken if
        True.
      must_split: (bool) A newline was required before this token.

    Returns:
      The penalty of splitting after the current token.
    """
    self._PushParameterListState(newline)

    penalty = 0
    if newline:
      penalty = self._AddTokenOnNewline(dry_run, must_split)
    else:
      self._AddTokenOnCurrentLine(dry_run)

    penalty += self._CalculateComprehensionState(newline)
    penalty += self._CalculateParameterListState(newline)

    return self.MoveStateToNextToken() + penalty

  def _AddTokenOnCurrentLine(self, dry_run):
    """Puts the token on the current line.

    Appends the next token to the state and updates information necessary for
    indentation.

    Arguments:
      dry_run: (bool) Commit whitespace changes to the FormatToken if True.
    """
    current = self.next_token
    previous = current.previous_token

    spaces = current.spaces_required_before
    if isinstance(spaces, list):
      # Don't set the value here, as we need to look at the lines near
      # this one to determine the actual horizontal alignment value.
      spaces = 0

    if not dry_run:
      current.AddWhitespacePrefix(newlines_before=0, spaces=spaces)

    if previous.OpensScope():
      if not current.is_comment:
        # Align closing scopes that are on a newline with the opening scope:
        #
        #     foo = [a,
        #            b,
        #           ]
        self.stack[-1].closing_scope_indent = self.column - 1
        if style.Get('ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT'):
          self.stack[-1].closing_scope_indent += 1
        self.stack[-1].indent = self.column + spaces
      else:
        self.stack[-1].closing_scope_indent = (
            self.stack[-1].indent - style.Get('CONTINUATION_INDENT_WIDTH'))

    self.column += spaces

  def _AddTokenOnNewline(self, dry_run, must_split):
    """Adds a line break and necessary indentation.

    Appends the next token to the state and updates information necessary for
    indentation.

    Arguments:
      dry_run: (bool) Don't commit whitespace changes to the FormatToken if
        True.
      must_split: (bool) A newline was required before this token.

    Returns:
      The split penalty for splitting after the current state.
    """
    current = self.next_token
    previous = current.previous_token

    self.column = self._GetNewlineColumn()

    if not dry_run:
      indent_level = self.line.depth
      spaces = self.column
      if spaces:
        spaces -= indent_level * style.Get('INDENT_WIDTH')
      current.AddWhitespacePrefix(
          newlines_before=1, spaces=spaces, indent_level=indent_level)

    if not current.is_comment:
      self.stack[-1].last_space = self.column
    self.lowest_level_on_line = self.paren_level

    if (previous.OpensScope() or
        (previous.is_comment and previous.previous_token is not None and
         previous.previous_token.OpensScope())):
      dedent = (style.Get('CONTINUATION_INDENT_WIDTH'),
                0)[style.Get('INDENT_CLOSING_BRACKETS')]
      self.stack[-1].closing_scope_indent = (
          max(0, self.stack[-1].indent - dedent))
      self.stack[-1].split_before_closing_bracket = True

    # Calculate the split penalty.
    penalty = current.split_penalty

    if must_split:
      # Don't penalize for a must split.
      return penalty

    if previous.is_pseudo and previous.value == '(':
      # Small penalty for splitting after a pseudo paren.
      penalty += 50

    # Add a penalty for each increasing newline we add, but don't penalize for
    # splitting before an if-expression or list comprehension.
    if current.value not in {'if', 'for'}:
      last = self.stack[-1]
      last.num_line_splits += 1
      penalty += (
          style.Get('SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT') *
          last.num_line_splits)

    if current.OpensScope() and previous.OpensScope():
      # Prefer to keep opening brackets coalesced (unless it's at the beginning
      # of a function call).
      pprev = previous.previous_token
      if not pprev or not pprev.is_name:
        penalty += 10

    return penalty + 10

  def MoveStateToNextToken(self):
    """Calculate format decision state information and move onto the next token.

    Before moving onto the next token, we first calculate the format decision
    state given the current token and its formatting decisions. Then the format
    decision state is set up so that the next token can be added.

    Returns:
      The penalty for the number of characters over the column limit.
    """
    current = self.next_token
    if not current.OpensScope() and not current.ClosesScope():
      self.lowest_level_on_line = min(self.lowest_level_on_line,
                                      self.paren_level)

    # If we encounter an opening bracket, we add a level to our stack to prepare
    # for the subsequent tokens.
    if current.OpensScope():
      last = self.stack[-1]
      new_indent = style.Get('CONTINUATION_INDENT_WIDTH') + last.last_space

      self.stack.append(_ParenState(new_indent, self.stack[-1].last_space))
      self.paren_level += 1

    # If we encounter a closing bracket, we can remove a level from our
    # parenthesis stack.
    if len(self.stack) > 1 and current.ClosesScope():
      if subtypes.DICTIONARY_KEY_PART in current.subtypes:
        self.stack[-2].last_space = self.stack[-2].indent
      else:
        self.stack[-2].last_space = self.stack[-1].last_space
      self.stack.pop()
      self.paren_level -= 1

    is_multiline_string = current.is_string and '\n' in current.value
    if is_multiline_string:
      # This is a multiline string. Only look at the first line.
      self.column += len(current.value.split('\n')[0])
    elif not current.is_pseudo:
      self.column += len(current.value)

    self.next_token = self.next_token.next_token

    # Calculate the penalty for overflowing the column limit.
    penalty = 0
    if (not current.is_pylint_comment and not current.is_pytype_comment and
        not current.is_copybara_comment and self.column > self.column_limit):
      excess_characters = self.column - self.column_limit
      penalty += style.Get('SPLIT_PENALTY_EXCESS_CHARACTER') * excess_characters

    if is_multiline_string:
      # If this is a multiline string, the column is actually the
      # end of the last line in the string.
      self.column = len(current.value.split('\n')[-1])

    return penalty

  def _CalculateComprehensionState(self, newline):
    """Makes required changes to comprehension state.

    Args:
      newline: Whether the current token is to be added on a newline.

    Returns:
      The penalty for the token-newline combination given the current
      comprehension state.
    """
    current = self.next_token
    previous = current.previous_token
    top_of_stack = self.comp_stack[-1] if self.comp_stack else None
    penalty = 0

    if top_of_stack is not None:
      # Check if the token terminates the current comprehension.
      if current == top_of_stack.closing_bracket:
        last = self.comp_stack.pop()
        # Lightly penalize comprehensions that are split across multiple lines.
        if last.has_interior_split:
          penalty += style.Get('SPLIT_PENALTY_COMPREHENSION')

        return penalty

      if newline:
        top_of_stack.has_interior_split = True

    if (subtypes.COMP_EXPR in current.subtypes and
        subtypes.COMP_EXPR not in previous.subtypes):
      self.comp_stack.append(object_state.ComprehensionState(current))
      return penalty

    if current.value == 'for' and subtypes.COMP_FOR in current.subtypes:
      if top_of_stack.for_token is not None:
        # Treat nested comprehensions like normal comp_if expressions.
        # Example:
        #     my_comp = [
        #         a.qux + b.qux
        #         for a in foo
        #   -->   for b in bar   <--
        #         if a.zut + b.zut
        #     ]
        if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and
            top_of_stack.has_split_at_for != newline and
            (top_of_stack.has_split_at_for or
             not top_of_stack.HasTrivialExpr())):
          penalty += split_penalty.UNBREAKABLE
      else:
        top_of_stack.for_token = current
        top_of_stack.has_split_at_for = newline

        # Try to keep trivial expressions on the same line as the comp_for.
        if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and newline and
            top_of_stack.HasTrivialExpr()):
          penalty += split_penalty.CONNECTED

    if (subtypes.COMP_IF in current.subtypes and
        subtypes.COMP_IF not in previous.subtypes):
      # Penalize breaking at comp_if when it doesn't match the newline structure
      # in the rest of the comprehension.
      if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and
          top_of_stack.has_split_at_for != newline and
          (top_of_stack.has_split_at_for or not top_of_stack.HasTrivialExpr())):
        penalty += split_penalty.UNBREAKABLE

    return penalty

  def _PushParameterListState(self, newline):
    """Push a new parameter list state for a function definition.

    Args:
      newline: Whether the current token is to be added on a newline.
    """
    current = self.next_token
    previous = current.previous_token

    if _IsFunctionDefinition(previous):
      first_param_column = previous.total_length + self.stack[-2].indent
      self.param_list_stack.append(
          object_state.ParameterListState(previous, newline,
                                          first_param_column))

  def _CalculateParameterListState(self, newline):
    """Makes required changes to parameter list state.

    Args:
      newline: Whether the current token is to be added on a newline.

    Returns:
      The penalty for the token-newline combination given the current
      parameter state.
    """
    current = self.next_token
    previous = current.previous_token
    penalty = 0

    if _IsFunctionDefinition(previous):
      first_param_column = previous.total_length + self.stack[-2].indent
      if not newline:
        param_list = self.param_list_stack[-1]
        if param_list.parameters and param_list.has_typed_return:
          last_param = param_list.parameters[-1].first_token
          last_token = _LastTokenInLine(previous.matching_bracket)
          total_length = last_token.total_length
          total_length -= last_param.total_length - len(last_param.value)
          if total_length + self.column > self.column_limit:
            # If we need to split before the trailing code of a function
            # definition with return types, then also split before the opening
            # parameter so that the trailing bit isn't indented on a line by
            # itself:
            #
            #   def rrrrrrrrrrrrrrrrrrrrrr(ccccccccccccccccccccccc: Tuple[Text]
            #                              ) -> List[Tuple[Text, Text]]:
            #       pass
            penalty += split_penalty.VERY_STRONGLY_CONNECTED
        return penalty

      if first_param_column <= self.column:
        # Make sure we don't split after the opening bracket if the
        # continuation indent is greater than the opening bracket:
        #
        #   a(
        #       b=1,
        #       c=2)
        penalty += split_penalty.VERY_STRONGLY_CONNECTED
      return penalty

    if not self.param_list_stack:
      return penalty

    param_list = self.param_list_stack[-1]
    if current == self.param_list_stack[-1].closing_bracket:
      self.param_list_stack.pop()  # We're done with this state.
      if newline and param_list.has_typed_return:
        if param_list.split_before_closing_bracket:
          penalty -= split_penalty.STRONGLY_CONNECTED
        elif param_list.LastParamFitsOnLine(self.column):
          penalty += split_penalty.STRONGLY_CONNECTED

      if (not newline and param_list.has_typed_return and
          param_list.has_split_before_first_param):
        # Prefer splitting before the closing bracket if there's a return type
        # and we've already split before the first parameter.
        penalty += split_penalty.STRONGLY_CONNECTED

      return penalty

    if not param_list.parameters:
      return penalty

    if newline:
      if self._FitsOnLine(param_list.parameters[0].first_token,
                          _LastTokenInLine(param_list.closing_bracket)):
        penalty += split_penalty.STRONGLY_CONNECTED

    if (not newline and style.Get('SPLIT_BEFORE_NAMED_ASSIGNS') and
        param_list.has_default_values and
        current != param_list.parameters[0].first_token and
        current != param_list.closing_bracket and
        subtypes.PARAMETER_START in current.subtypes):
      # If we want to split before parameters when there are named assigns,
      # then add a penalty for not splitting.
      penalty += split_penalty.STRONGLY_CONNECTED

    return penalty

  def _IndentWithContinuationAlignStyle(self, column):
    if column == 0:
      return column
    align_style = style.Get('CONTINUATION_ALIGN_STYLE')
    if align_style == 'FIXED':
      return ((self.line.depth * style.Get('INDENT_WIDTH')) +
              style.Get('CONTINUATION_INDENT_WIDTH'))
    if align_style == 'VALIGN-RIGHT':
      indent_width = style.Get('INDENT_WIDTH')
      return indent_width * int((column + indent_width - 1) / indent_width)
    return column

  def _GetNewlineColumn(self):
    """Return the new column on the newline."""
    current = self.next_token
    previous = current.previous_token
    top_of_stack = self.stack[-1]

    if isinstance(current.spaces_required_before, list):
      # Don't set the value here, as we need to look at the lines near
      # this one to determine the actual horizontal alignment value.
      return 0
    elif current.spaces_required_before > 2 or self.line.disable:
      return current.spaces_required_before

    cont_aligned_indent = self._IndentWithContinuationAlignStyle(
        top_of_stack.indent)

    if current.OpensScope():
      return cont_aligned_indent if self.paren_level else self.first_indent

    if current.ClosesScope():
      if (previous.OpensScope() or
          (previous.is_comment and previous.previous_token is not None and
           previous.previous_token.OpensScope())):
        return max(0,
                   top_of_stack.indent - style.Get('CONTINUATION_INDENT_WIDTH'))
      return top_of_stack.closing_scope_indent

    if (previous and previous.is_string and current.is_string and
        subtypes.DICTIONARY_VALUE in current.subtypes):
      return previous.column

    if style.Get('INDENT_DICTIONARY_VALUE'):
      if previous and (previous.value == ':' or previous.is_pseudo):
        if subtypes.DICTIONARY_VALUE in current.subtypes:
          return top_of_stack.indent

    if (not self.param_list_stack and _IsCompoundStatement(self.line.first) and
        (not (style.Get('DEDENT_CLOSING_BRACKETS') or
              style.Get('INDENT_CLOSING_BRACKETS')) or
         style.Get('SPLIT_BEFORE_FIRST_ARGUMENT'))):
      token_indent = (
          len(self.line.first.whitespace_prefix.split('\n')[-1]) +
          style.Get('INDENT_WIDTH'))
      if token_indent == top_of_stack.indent:
        return token_indent + style.Get('CONTINUATION_INDENT_WIDTH')

    if (self.param_list_stack and
        not self.param_list_stack[-1].SplitBeforeClosingBracket(
            top_of_stack.indent) and top_of_stack.indent
        == ((self.line.depth + 1) * style.Get('INDENT_WIDTH'))):
      if (subtypes.PARAMETER_START in current.subtypes or
          (previous.is_comment and
           subtypes.PARAMETER_START in previous.subtypes)):
        return top_of_stack.indent + style.Get('CONTINUATION_INDENT_WIDTH')

    return cont_aligned_indent

  def _FitsOnLine(self, start, end):
    """Determines if line between start and end can fit on the current line."""
    length = end.total_length - start.total_length
    if not start.is_pseudo:
      length += len(start.value)
    return length + self.column <= self.column_limit

  def _EachDictEntryFitsOnOneLine(self, opening):
    """Determine if each dict elems can fit on one line."""

    def PreviousNonCommentToken(tok):
      tok = tok.previous_token
      while tok.is_comment:
        tok = tok.previous_token
      return tok

    def ImplicitStringConcatenation(tok):
      num_strings = 0
      if tok.is_pseudo:
        tok = tok.next_token
      while tok.is_string:
        num_strings += 1
        tok = tok.next_token
      return num_strings > 1

    def DictValueIsContainer(opening, closing):
      """Return true if the dictionary value is a container."""
      if not opening or not closing:
        return False
      colon = opening.previous_token
      while colon:
        if not colon.is_pseudo:
          break
        colon = colon.previous_token
      if not colon or colon.value != ':':
        return False
      key = colon.previous_token
      if not key:
        return False
      return subtypes.DICTIONARY_KEY_PART in key.subtypes

    closing = opening.matching_bracket
    entry_start = opening.next_token
    current = opening.next_token.next_token

    while current and current != closing:
      if subtypes.DICTIONARY_KEY in current.subtypes:
        prev = PreviousNonCommentToken(current)
        if prev.value == ',':
          prev = PreviousNonCommentToken(prev.previous_token)
        if not DictValueIsContainer(prev.matching_bracket, prev):
          length = prev.total_length - entry_start.total_length
          length += len(entry_start.value)
          if length + self.stack[-2].indent >= self.column_limit:
            return False
        entry_start = current
      if current.OpensScope():
        if ((current.value == '{' or
             (current.is_pseudo and current.next_token.value == '{') and
             subtypes.DICTIONARY_VALUE in current.subtypes) or
            ImplicitStringConcatenation(current)):
          # A dictionary entry that cannot fit on a single line shouldn't matter
          # to this calculation. If it can't fit on a single line, then the
          # opening should be on the same line as the key and the rest on
          # newlines after it. But the other entries should be on single lines
          # if possible.
          if current.matching_bracket:
            current = current.matching_bracket
          while current:
            if current == closing:
              return True
            if subtypes.DICTIONARY_KEY in current.subtypes:
              entry_start = current
              break
            current = current.next_token
        else:
          current = current.matching_bracket
      else:
        current = current.next_token

    # At this point, current is the closing bracket. Go back one to get the end
    # of the dictionary entry.
    current = PreviousNonCommentToken(current)
    length = current.total_length - entry_start.total_length
    length += len(entry_start.value)
    return length + self.stack[-2].indent <= self.column_limit

  def _ArgumentListHasDictionaryEntry(self, token):
    """Check if the function argument list has a dictionary as an arg."""
    if _IsArgumentToFunction(token):
      while token:
        if token.value == '{':
          length = token.matching_bracket.total_length - token.total_length
          return length + self.stack[-2].indent > self.column_limit
        if token.ClosesScope():
          break
        if token.OpensScope():
          token = token.matching_bracket
        token = token.next_token
    return False

  def _ContainerFitsOnStartLine(self, opening):
    """Check if the container can fit on its starting line."""
    return (opening.matching_bracket.total_length - opening.total_length +
            self.stack[-1].indent) <= self.column_limit

_COMPOUND_STMTS = frozenset(
    {'for', 'while', 'if', 'elif', 'with', 'except', 'def', 'class'})

def _IsCompoundStatement(token):
  if token.value == 'async':
    token = token.next_token
  return token.value in _COMPOUND_STMTS

def _IsFunctionDef(token):
  if token.value == 'async':
    token = token.next_token
  return token.value == 'def'

def _IsFunctionCallWithArguments(token):
  while token:
    if token.value == '(':
      token = token.next_token
      return token and token.value != ')'
    elif token.name not in {'NAME', 'DOT', 'EQUAL'}:
      break
    token = token.next_token
  return False

def _IsArgumentToFunction(token):
  bracket = logical_line.IsSurroundedByBrackets(token)
  if not bracket or bracket.value != '(':
    return False
  previous = bracket.previous_token
  return previous and previous.is_name

def _GetOpeningBracket(current):
  """Get the opening bracket containing the current token."""
  if current.matching_bracket and not current.is_pseudo:
    return current if current.OpensScope() else current.matching_bracket

  while current:
    if current.ClosesScope():
      current = current.matching_bracket
    elif current.is_pseudo:
      current = current.previous_token
    elif current.OpensScope():
      return current
    current = current.previous_token
  return None

def _LastTokenInLine(current):
  while not current.is_comment and current.next_token:
    current = current.next_token
  return current

def _IsFunctionDefinition(current):
  prev = current.previous_token
  return current.value == '(' and prev and subtypes.FUNC_DEF in prev.subtypes

def _IsLastScopeInLine(current):
  current = current.matching_bracket
  while current:
    current = current.next_token
    if current and current.OpensScope():
      return False
  return True

def _IsSingleElementTuple(token):
  """Check if it's a single-element tuple."""
  close = token.matching_bracket
  token = token.next_token
  num_commas = 0
  while token != close:
    if token.value == ',':
      num_commas += 1
    token = token.matching_bracket if token.OpensScope() else token.next_token
  return num_commas == 1

def _ScopeHasNoCommas(token):
  """Check if the scope has no commas."""
  close = token.matching_bracket
  token = token.next_token
  while token != close:
    if token.value == ',':
      return False
    token = token.matching_bracket if token.OpensScope() else token.next_token
  return True

class _ParenState(object):
  """Maintains the state of the bracket enclosures.

  A stack of _ParenState objects are kept so that we know how to indent relative
  to the brackets.

  Attributes:
    indent: The column position to which a specified parenthesis level needs to
      be indented.
    last_space: The column position of the last space on each level.
    closing_scope_indent: The column position of the closing indentation.
    split_before_closing_bracket: Whether a newline needs to be inserted before
      the closing bracket. We only want to insert a newline before the closing
      bracket if there also was a newline after the beginning left bracket.
    num_line_splits: Number of line splits this _ParenState contains already.
      Each subsequent line split gets an increasing penalty.
  """

  # TODO(morbo): This doesn't track "bin packing."

  def __init__(self, indent, last_space):
    self.indent = indent
    self.last_space = last_space
    self.closing_scope_indent = 0
    self.split_before_closing_bracket = False
    self.num_line_splits = 0

  def Clone(self):
    state = _ParenState(self.indent, self.last_space)
    state.closing_scope_indent = self.closing_scope_indent
    state.split_before_closing_bracket = self.split_before_closing_bracket
    state.num_line_splits = self.num_line_splits
    return state

  def __repr__(self):
    return '[indent::%d, last_space::%d, closing_scope_indent::%d]' % (
        self.indent, self.last_space, self.closing_scope_indent)

  def __eq__(self, other):
    return hash(self) == hash(other)

  def __ne__(self, other):
    return not self == other

  def __hash__(self, *args, **kwargs):
    return hash((self.indent, self.last_space, self.closing_scope_indent,
                 self.split_before_closing_bracket, self.num_line_splits))


text:
<!doctype html>

		<meta charset="utf-8">

		<title>Microservices Lifecycle</title>

		<meta name="author" content="Viktor Farcic">

		<meta name="apple-mobile-web-app-capable" content="yes" />
		<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />

		<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">

		<link rel="stylesheet" href="../css/reveal.css">
		<link rel="stylesheet" href="../css/theme/black.css" id="theme">
		<link rel="stylesheet" href="../css/theme/vfarcic.css">

Microservices Lifecycle Workshop

						Viktor Farcic

						@vfarcic

						TechnologyConversations.com

						CloudBees.com

Viktor Farcic

Continuous deployment

Continuous integration

									Push/clone,
									Static analysis,
									Pre-deployment testing,
									Packaging/deployment,
									Post-deployment testing

Continuous delivery

									every commit can be deployed to production

Continuous deployment

									every commit is deployed to production

Continuous deployment

CI process

Continuous deployment

CI pipeline

Continuous deployment

Continuous delivery process

Continuous deployment

Continuous deployment pipeline

Containers (Docker)

Shipping container is an object for holding or transporting something

Isolated

Immutable

Reliable

Self-sufficient

Scalable

Containers (Docker)

VMs vs Containers

Containers (Docker)

Self-sufficient container

Containers (Docker)

Container with the separate DB

Containers (Docker)

Containers with the shared DB

Monolithic applications

Single unit

Time increases complexity and size

Time decreases development, testing and deployment speed

Layers

Change is hard and with risks

Scaling = multiplication of the entire application

Monolithic applications

Early

Monolithic applications

Later

Monolithic applications

Scaling

Microservices

Applications that fit into a screen

System composed of small services

Autonomy/independence

Data exchange through APIs

Bounded context

Microservices

Self-Sufficiency

Microservices

Gartner

							Microservices are simpler, developers get more productive and systems can be scaled quickly and precisely, rather than in large monolithic globs. And I haven't even mentioned the potential for polyglot coding and data persistence.

- Gary Olliffe

Microservices

Object-Oriented Design

							The big idea is 'messaging'.
							The key in making great and growable systems is much more to design how its modules communicate rather than what their internal properties and behaviors should be.

- Alan Kay

Microservices

Single Responsibility Principle

							Gather together those things that change for the same reason, and separate those things that change for different reasons

- Robert C. Martin

Microservices

Linux = Microservices

ps aux | grep jav[a] | awk '{print $2}' | xargs kill

Microservices

One thing or one functionality

Any tools or languages

Truly loosely coupled

Teams independence

Easier testing and CD

Decentralization

Microservices

Disadvantages

Increased operational and deployment complexity

Configuration Management

Containers (Docker)

Work shifted from development to DevOps

Remote process calls

Microservices

Advantages

Scaling

Resilience / fault isolation

Innovation

Size

Decoupling

Deployment

No need for long term commitment

Microservices

Best Practices

Containers (Docker)

Reverse proxy

Minimalist approach

CM is a must

Cross functional teams

API versioning

Deployment

Big vs small

Mutable vs immutable

Deployment

Mutable Monster Server

Deployment

Immutable Server

Deployment

Immutable Server

Deployment

Immutable Server

Deployment

Immutable Server

Deployment

Immutable Micro Services

Deployment

Immutable Micro Services

Deployment

Immutable Micro Services

Deployment

Immutable Micro Services

Deployment Pipeline

Checkout the code

Run pre-deployment tests

Compile and/or package the code

Build the container

Push the container to the registry

Deploy the container to the production server

Integrate the container

Run post-deployment tests

Deployment Pipeline: Initial Stages

Creating the CD VM

git clone https://github.com/vfarcic/ms-lifecycle.git

cd ms-lifecycle

vagrant plugin install vagrant-cachier

vagrant up cd --provision

vagrant ssh cd

Deployment Pipeline: Initial Stages

Checking out the code

git clone https://github.com/vfarcic/go-demo.git

cd go-demo

Deployment Pipeline: Initial Stages

Running pre-deployment tests and compiling and/or packaging the code

cat docker-compose-test.yml

docker-compose -f docker-compose-test.yml run --rm unit

docker ps -a

ll -t

Deployment Pipeline: Initial Stages

Building Docker containers

cat Dockerfile

docker build -t vfarcic/go-demo .

docker images

Deployment Pipeline: Initial Stages

Running containers

docker-compose up -d db app

docker-compose ps

PORT=<PORT> # Put the port from the ps command

docker-compose logs

docker-compose exec app ping -c 1 db

docker-compose exec db ping -c 1 app

Deployment Pipeline: Initial Stages

Running containers

curl -i localhost:$PORT/demo/hello

curl -XPUT localhost:$PORT/demo/person?name=Viktor

curl -XPUT localhost:$PORT/demo/person?name=Sara

curl localhost:$PORT/demo/person

docker-compose down

docker ps -a

Deployment Pipeline: Initial Stages

Pushing containers to the registry

# docker push vfarcic/go-demo

docker tag vfarcic/go-demo 10.100.198.200:5000/go-demo

docker push 10.100.198.200:5000/go-demo

exit

Deployment Pipeline: Initial Stages

Checklist

Checkout the code

Run pre-deployment tests

Compile and/or package the code

Build the container

Push the container to the registry

Deploy the container to the production server

Integrate the container

Run post-deployment tests

Configuration Management

Configuration Management

Configuring the Production Environment

vagrant up prod

vagrant ssh cd

cd /vagrant/ansible

ansible-playbook prod.yml -i hosts/prod

Configuration Management

Production environment Ansible playbook

cat prod.yml

cat roles/docker/tasks/main.yml

cat roles/docker/tasks/debian.yml

cat hosts/prod

exit

Deployment Pipeline: Intermediate Stages

Checkout the code

Run pre-deployment tests

Compile and/or package the code

Build the container

Push the container to the registry

Deploy the container to the production server

Integrate the container

Run post-deployment tests

Deployment Pipeline: Intermediate Stages

Deploying containers to the production server

vagrant ssh cd

export DOCKER_HOST=tcp://prod:2375

docker ps -a

cd go-demo

docker-compose up -d db app

docker-compose ps

Deployment Pipeline: Intermediate Stages

Post-deployment verification

docker inspect godemo_app_1

PORT=$(docker inspect \
	--format='{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}' \
	godemo_app_1)

echo $PORT

curl -XPUT prod:$PORT/demo/person?name=Viktor

curl -XPUT prod:$PORT/demo/person?name=Sara

curl prod:$PORT/demo/person

Deployment Pipeline: Intermediate Stages

Stopping Production Node

exit

vagrant halt prod

Deployment Pipeline: Intermediate Stages

Checklist

Checkout the code

Run pre-deployment tests

Compile and/or package the code

Build the container

Push the container to the registry

Deploy the container to the production server

Integrate the container

Run post-deployment tests

Service Discovery

Service Discovery

Single node

Service Discovery

Multiple nodes

Service Discovery

Service Discovery Elements

Service registry

Service registration

Service discovery

Service Discovery

Tools

Service Discovery

Consul Ansible playbook

vagrant up serv-disc-01 serv-disc-02 serv-disc-03

vagrant ssh cd

cat /vagrant/ansible/hosts/serv-disc

cat /vagrant/ansible/consul.yml

cat /vagrant/ansible/roles/consul/tasks/main.yml

cat /vagrant/ansible/roles/consul/defaults/main.yml

cat /vagrant/ansible/host_vars/10.100.194.201

Service Discovery

Setting up Consul

ansible-playbook \
    /vagrant/ansible/consul.yml \
    -i /vagrant/ansible/hosts/serv-disc

curl serv-disc-01:8500/v1/catalog/nodes | jq '.'

Service Discovery

Consul

Service Discovery

Key/value store: PUT

curl -X PUT -d 'this is a test' \
    serv-disc-01:8500/v1/kv/msg1

curl -X PUT -d 'this is another test' \
    serv-disc-02:8500/v1/kv/messages/msg2

curl -X PUT -d 'this is a test with flags' \
    serv-disc-03:8500/v1/kv/messages/msg3?flags=1234

Service Discovery

Key/value store: GET/DELETE

curl serv-disc-03:8500/v1/kv/?recurse | jq '.'

curl serv-disc-02:8500/v1/kv/msg1 | jq '.'

curl serv-disc-01:8500/v1/kv/msg1?raw

curl -X DELETE serv-disc-01:8500/v1/kv/messages/msg2

curl serv-disc-03:8500/v1/kv/?recurse | jq '.'

curl -X DELETE serv-disc-02:8500/v1/kv/?recurse

curl serv-disc-03:8500/v1/kv/?recurse | jq '.'

Service Discovery

Registrator

cat /vagrant/ansible/roles/registrator/tasks/main.yml

cat /vagrant/ansible/roles/registrator/defaults/main.yml

ansible-playbook \
    /vagrant/ansible/registrator.yml \
    -i /vagrant/ansible/hosts/serv-disc

Service Discovery

Consul / Registrator

Service Discovery

Registering service

docker -H tcp://serv-disc-01:2375 run -d \
    --name nginx \
    --env SERVICE_NAME=nginx \
    --env SERVICE_ID=nginx \
    -p 1234:80 \
    nginx:alpine

curl serv-disc-01:8500/v1/catalog/service/nginx-80 | jq '.'

Service Discovery

Registering service again

docker -H tcp://serv-disc-02:2375 run -d \
    --name nginx2 \
    --env "SERVICE_ID=nginx2" \
    --env "SERVICE_NAME=nginx" \
    --env "SERVICE_TAGS=balancer,proxy,www" \
    -p 1111:80 \
    nginx:alpine

curl serv-disc-02:8500/v1/catalog/service/nginx-80 | jq '.'

Service Discovery

Consul / Registrator / Consul Template

Service Discovery

Running Consul Template

curl serv-disc-01:8500/v1/catalog/service/nginx-80 | jq '.'

cat /data/consul-template/example2.ctmpl

consul-template \
    -consul serv-disc-01:8500 \
    -template "/data/consul-template/example2.ctmpl:/tmp/example.conf" \
    -once

cat /tmp/example.conf

Service Discovery

Cleanup

exit # Exit cd node

vagrant destroy -f serv-disc-01 serv-disc-02 serv-disc-03

Proxy Service

Proxy Service

nginx Ansible Playbook

vagrant up proxy

vagrant ssh cd

cat /vagrant/ansible/nginx.yml

cat /vagrant/ansible/roles/nginx/files/services.conf

ansible-playbook /vagrant/ansible/nginx.yml \
    -i /vagrant/ansible/hosts/proxy

export DOCKER_HOST=tcp://proxy:2375

docker ps

curl proxy:8500/v1/catalog/services | jq '.'

Proxy Service

Services Without Proxy

cd go-demo

docker-compose up -d db app

curl proxy/api/v1/books

docker-compose ps

PORT=$(docker inspect \
    --format='{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}' \
    godemo_app_1)

curl proxy:$PORT/demo/hello

exit

Proxy Service

Services Without Proxy

Proxy Service

Manually Setting Proxy

vagrant ssh proxy

PORT=$(docker inspect \
    --format='{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}' \
    godemo_app_1)

echo "location /demo {
    proxy_pass http://10.100.193.200:$PORT/demo;
}" | sudo tee /data/nginx/includes/go-demo.conf

docker kill -s HUP nginx

curl http://localhost/demo/hello

docker rm -f nginx

exit

Proxy Service

Manually Setting Proxy

Proxy Service

Docker Flow: Proxy

vagrant ssh cd

export DOCKER_HOST=tcp://proxy:2375

docker run -d \
    --name docker-flow-proxy \
    -e CONSUL_ADDRESS=10.100.193.200:8500 \
    -p 80:80 -p 8081:8080 \
    vfarcic/docker-flow-proxy

curl "proxy:8081/v1/docker-flow-proxy/reconfigure?serviceName=go-demo&servicePath=/demo" \
    | jq '.'

curl -i proxy/demo/hello

Proxy Service

Docker Flow: Proxy

cd go-demo

docker-compose ps

docker-compose scale app=3

docker-compose ps

curl "proxy:8081/v1/docker-flow-proxy/reconfigure?serviceName=go-demo&servicePath=/demo" \
    | jq '.'

curl -i proxy/demo/hello # Repeat a few times

docker-compose logs app

Deployment

Docker Flow: Proxy

Proxy Service

Cleanup

exit

vagrant halt proxy

Deployment Pipeline: The Late Stages

Checkout the code

Run pre-deployment tests

Compile and/or package the code

Build the container

Push the container to the registry

Deploy the container to the production server

Integrate the container

Run post-deployment tests

Deployment Pipeline: The Late Stages

Provision Production

vagrant up prod

vagrant ssh cd

cd /vagrant/ansible

ansible-playbook prod2.yml -i hosts/prod --skip-tags "nginx"

Deployment Pipeline: The Late Stages

Run Containers

export DOCKER_HOST=tcp://prod:2375

cd ~/go-demo

docker-compose up -d db app

curl prod:8500/v1/catalog/service/go-demo \
    | jq '.'

						Open http://10.100.198.201:8500/ui.

Deployment Pipeline: The Late Stages

Reconfigure the Proxy

curl -i prod/demo/hello

docker run -d \
    --name docker-flow-proxy \
    -e CONSUL_ADDRESS=10.100.198.201:8500 \
    -p 80:80 -p 8081:8080 \
    vfarcic/docker-flow-proxy

curl "prod:8081/v1/docker-flow-proxy/reconfigure?serviceName=go-demo&servicePath=/demo" \
    | jq '.'

curl -i prod/demo/hello

Deployment Pipeline: The Late Stages

Run Integration Tests

cat docker-compose-test.yml

unset DOCKER_HOST

export HOST_IP=10.100.198.201

docker-compose \
    -f docker-compose-test.yml \
    run --rm production

# docker push vfarcic/go-demo

Deployment Pipeline: The Late Stages

Clean Up

export DOCKER_HOST=tcp://prod:2375

docker-compose down

curl "prod:8081/v1/docker-flow-proxy/remove?serviceName=go-demo" \
    | jq '.'

Deployment Pipeline: The Late Stages

Checklist

Checkout the code

Run pre-deployment tests

Compile and/or package the code

Build the container

Push the container to the registry

Deploy the container to the production server

Integrate the container

Run post-deployment tests

Deployment Pipeline: The Late Stages

Problems

Downtime

Not scalable

Not failure tolerant

Difficult to monitor

Not automated

Standard Deployment

Standard Deployment

Standard Deployment

Blue-Green Deployment

Blue-Green Deployment

Blue-Green Deployment

Blue-Green Deployment

Blue-Green Deployment

Blue-Green Deployment

cat docker-compose-bg.yml

export NEXT_COLOR=blue

VERSION=:1.0 docker-compose -f docker-compose-bg.yml \
    up -d db app-${NEXT_COLOR}

curl "prod:8081/v1/docker-flow-proxy/reconfigure?serviceName=go-demo-${NEXT_COLOR}&servicePath=/demo" \
    | jq '.'

curl -i prod/demo/hello

docker-compose -f docker-compose-bg.yml logs

curl -XPUT -d $NEXT_COLOR prod:8500/v1/kv/go-demo/color

Blue-Green Deployment

Blue-Green Deployment

COLOR=$(curl prod:8500/v1/kv/go-demo/color?raw)

NEXT_COLOR=$(if [[ "$COLOR" == "blue" ]]; then echo "green"
else echo "blue"; fi)

VERSION=:1.1 docker-compose -f docker-compose-bg.yml \
    up -d db app-${NEXT_COLOR}

docker-compose -f docker-compose-bg.yml ps

curl -i prod/demo/hello

docker-compose -f docker-compose-bg.yml logs

Blue-Green Deployment

Blue-Green Deployment

curl "prod:8081/v1/docker-flow-proxy/reconfigure?serviceName=go-demo-${NEXT_COLOR}&servicePath=/demo" \
    | jq '.'

curl "prod:8081/v1/docker-flow-proxy/remove?serviceName=go-demo-${COLOR}" \
    | jq '.'

curl -XPUT -d $NEXT_COLOR prod:8500/v1/kv/go-demo/color

curl -i prod/demo/hello # Repeat

docker-compose -f docker-compose-bg.yml logs

docker-compose -f docker-compose-bg.yml stop app-${COLOR}

docker-compose -f docker-compose-bg.yml ps

Blue-Green Deployment

Blue-Green Deployment

docker-compose -f docker-compose-bg.yml down

curl "prod:8081/v1/docker-flow-proxy/remove?serviceName=go-demo-${NEXT_COLOR}" \
    | jq '.'

Docker Flow

export FLOW_PROXY_HOST=10.100.198.201

export FLOW_PROXY_RECONF_PORT=8081

export FLOW_CONSUL_ADDRESS=http://10.100.198.201:8500

export FLOW_PROXY_DOCKER_HOST=tcp://10.100.198.201:2375

docker-flow --flow=deploy --flow=proxy --flow=stop-old

docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"

curl -i prod/demo/hello

Docker Flow

docker-flow --flow=deploy --flow=proxy --flow=stop-old

docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"

curl -i prod/demo/hello

Clean Up

exit

vagrant destroy -f prod

Clustering And Scaling Services

Clustering And Scaling Services

Tools

Kubernetes

Mesos DCOS

Swarm

Clustering And Scaling Services

Service Discovery

Clustering And Scaling Services

Master and Nodes

Clustering And Scaling Services

Deploying the First Container

Clustering And Scaling Services

The Second Container

Clustering And Scaling Services

All Nodes Full

Clustering And Scaling Services

Adding a New Node

Clustering And Scaling Services

Failed Node

Clustering And Scaling Services

Setting Up Docker Swarm

vagrant up swarm-master swarm-node-1 swarm-node-2

vagrant ssh cd

cat /vagrant/ansible/swarm.yml

cat /vagrant/ansible/roles/swarm/tasks/main.yml

ansible-playbook /vagrant/ansible/swarm.yml \
    -i /vagrant/ansible/hosts/prod

export DOCKER_HOST=tcp://10.100.192.200:2375

docker info

docker ps -a

Clustering And Scaling Services

Deploying with Docker Swarm

cd ~/go-demo

docker-compose up -d db app

docker-compose ps

curl swarm-master:8500/v1/catalog/services \
    | jq '.'

curl swarm-master:8500/v1/catalog/service/go-demo \
    | jq '.'

docker-compose down

Clustering and Scaling Services

Swarm with Docker Flow

export FLOW_PROXY_HOST=10.100.198.200

export FLOW_PROXY_RECONF_PORT=8081

export FLOW_CONSUL_ADDRESS=http://10.100.192.200:8500

export FLOW_PROXY_DOCKER_HOST=tcp://10.100.198.200:2375

docker-flow --flow=deploy --flow=proxy --flow=stop-old

curl -i localhost/demo/hello

Clustering and Scaling Services

Swarm With Docker Flow

Clustering and Scaling Services

Swarm With Docker Flow

docker-flow --flow=deploy --flow=proxy --flow=stop-old

docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"

curl -i localhost/demo/hello

Clustering and Scaling Services

Swarm With Docker Flow

Clustering and Scaling Services

Swarm With Docker Flow

docker-flow --scale="+2" --flow=scale --flow=proxy

docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"

curl -i localhost/demo/hello

docker-flow --scale="-2" --flow=scale --flow=proxy

docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"

Clustering and Scaling Services

Swarm With Docker Flow

Clustering And Scaling Services

Cleaning Up

docker rm -f $(docker ps -a | grep godemo | awk '{ print $1 }')

sudo docker rm -f docker-flow-proxy

CI Tools

GoCD

Bamboo

Travis

CircleCI

Shippable

Jenkins

CI Tools

Jenkins Setup

ansible-playbook /vagrant/ansible/jenkins-node-swarm.yml \
    -i /vagrant/ansible/hosts/prod

ansible-playbook /vagrant/ansible/jenkins.yml \
    -c local

cat /vagrant/ansible/jenkins.yml

cat /vagrant/ansible/roles/java/tasks/main.yml

cat /vagrant/ansible/roles/jenkins/tasks/main.yml

cat /vagrant/ansible/roles/jenkins/defaults/main.yml

						Open http://10.100.198.200:8080

CI Tools

Jenkins Global Library

git clone http://10.100.198.200:8080/workflowLibs.git /tmp/workflowLibs

cd /tmp/workflowLibs

git checkout -b master

mkdir vars

cp ~/go-demo/jenkins/vars/dockerFlowWorkshop.groovy \
    /tmp/workflowLibs/vars/dockerFlow.groovy

git config --global user.name "vfarcic"

git add --all && git commit -a -m "Docker Flow"

git push --set-upstream origin master

CI Tools

Deployment Flow With Jenkins

cd ~/go-demo

cat jenkins/vars/dockerFlowWorkshop.groovy

cat Jenkinsfile

curl -XPUT -d "10.100.198.200" \
    http://swarm-master:8500/v1/kv/proxy/ip

CI Tools

Deployment With Jenkins

http://10.100.198.200:8080 > New Item

Item Name: go-demo; Type: Multibranch Pipeline > OK

Add Source > Git

Project Repository: https://github.com/vfarcic/go-demo.git > Save

http://10.100.198.200:8080/job/go-demo/branch/master/

http://10.100.198.200:8080/job/go-demo/branch/master/lastBuild/console

Say Again

Immutable over mutable deployments

Small over big

Automation over manual procedures

Service discovery over predefined configuration

Dynamic over static

Specialized over do-it-all tools

							That is, while there is value in the items on the right, we value the items on the left more.

Viktor Farcic

@vfarcic

TechnologyConversations.com

Viktor Farcic

Amazon

LeanPub

Cleanup

exit

vagrant destroy -f


text:
package com.microsoft.azure.cognitiveservices.vision.computervision.models;

import java.util.List;
import com.fasterxml.jackson.annotation.JsonProperty;

/**
 * A region consists of multiple lines (e.g. a column of text in a multi-column
 * document).
 */
public class OcrRegion {
    /**
     * Bounding box of a recognized region. The four integers represent the
     * x-coordinate of the left edge, the y-coordinate of the top edge, width,
     * and height of the bounding box, in the coordinate system of the input
     * image, after it has been rotated around its center according to the
     * detected text angle (see textAngle property), with the origin at the
     * top-left corner, and the y-axis pointing down.
     */
    @JsonProperty(value = "boundingBox")
    private String boundingBox;

    /**
     * An array of recognized lines of text.
     */
    @JsonProperty(value = "lines")
    private List<OcrLine> lines;

    /**
     * Get the boundingBox value.
     *
     * @return the boundingBox value
     */
    public String boundingBox() {
        return this.boundingBox;
    }

    /**
     * Set the boundingBox value.
     *
     * @param boundingBox the boundingBox value to set
     * @return the OcrRegion object itself.
     */
    public OcrRegion withBoundingBox(String boundingBox) {
        this.boundingBox = boundingBox;
        return this;
    }

    /**
     * Get the lines value.
     *
     * @return the lines value
     */
    public List<OcrLine> lines() {
        return this.lines;
    }

    /**
     * Set the lines value.
     *
     * @param lines the lines value to set
     * @return the OcrRegion object itself.
     */
    public OcrRegion withLines(List<OcrLine> lines) {
        this.lines = lines;
        return this;
    }

}