text:
"""Tests for the collection export tool plugin."""

import argparse

import mock

from grr.lib import access_control
from grr.lib import config_lib
from grr.lib import data_store
from grr.lib import email_alerts
from grr.lib import flags
from grr.lib import rdfvalue
from grr.lib import test_lib
from grr.lib import utils
from grr.lib.hunts import results
from grr.lib.output_plugins import email_plugin
from grr.lib.rdfvalues import client as rdf_client
from grr.lib.rdfvalues import flows as rdf_flows
from grr.lib.rdfvalues import paths as rdf_paths
from grr.tools.export_plugins import collection_plugin

class CollectionExportPluginTest(test_lib.GRRBaseTest):

  def setUp(self):
    super(CollectionExportPluginTest, self).setUp()

    client_ids = self.SetupClients(1)
    self.client_id = client_ids[0]

    data_store.default_token = access_control.ACLToken(
        username="user", reason="reason")

  def testGetValuesForExportHuntResultCollection(self):
    fd = results.HuntResultCollection("aff4:/huntcoll", token=self.token)
    fd.Add(
        rdf_flows.GrrMessage(
            payload=rdf_client.StatEntry(pathspec=rdf_paths.PathSpec(
                path="testfile", pathtype="OS")),
            source=self.client_id))

    plugin = collection_plugin.CollectionExportPlugin()
    mock_args = mock.Mock()
    mock_args.path = rdfvalue.RDFURN("aff4:/huntcoll")
    mock_args.no_legacy_warning_pause = True
    self.assertEqual(len(plugin.GetValuesForExport(mock_args)), 1)

  def testExportCollectionWithEmailPlugin(self):
    # Create a collection with URNs to some files.
    fd = results.HuntResultCollection("aff4:/testcoll", token=self.token)
    fd.Add(
        rdf_flows.GrrMessage(
            payload=rdf_client.StatEntry(pathspec=rdf_paths.PathSpec(
                path="testfile", pathtype="OS")),
            source=self.client_id))

    plugin = collection_plugin.CollectionExportPlugin()
    parser = argparse.ArgumentParser()
    plugin.ConfigureArgParser(parser)

    def SendEmail(address, sender, title, message, **_):
      self.email_messages.append(
          dict(address=address, sender=sender, title=title, message=message))

    email_address = "notify@%s" % config_lib.CONFIG["Logging.domain"]
    with utils.Stubber(email_alerts.EMAIL_ALERTER, "SendEmail", SendEmail):
      self.email_messages = []

      plugin.Run(
          parser.parse_args(args=[
              "--no_legacy_warning_pause",
              "--path",
              "aff4:/testcoll",
              email_plugin.EmailOutputPlugin.name,
              "--email_address",
              email_address,
              "--emails_limit",
              "100",
          ]))

    self.assertEqual(len(self.email_messages), 1)
    for msg in self.email_messages:
      self.assertEqual(msg["address"], email_address)
      self.assertEqual("GRR got a new result in aff4:/testcoll.", msg["title"])
      self.assertTrue(
          "GRR got a new result in aff4:/testcoll" in msg["message"])
      self.assertTrue("(Host-0)" in msg["message"])

def main(argv):
  test_lib.main(argv)

if __name__ == "__main__":
  flags.StartMain(main)


text: Modernizr.addTest('srcdoc', 'srcdoc' in document.createElement('iframe'));

redpajama_set_name: RedPajamaGithub


text:
using System.Buffers;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;

namespace System.IO
{
    public partial class FileStream : Stream
    {
        // This is an internal object extending TaskCompletionSource with fields
        // for all of the relevant data necessary to complete the IO operation.
        // This is used by IOCallback and all of the async methods.
        private unsafe class FileStreamCompletionSource : TaskCompletionSource<int>
        {
            private const long NoResult = 0;
            private const long ResultSuccess = (long)1 << 32;
            private const long ResultError = (long)2 << 32;
            private const long RegisteringCancellation = (long)4 << 32;
            private const long CompletedCallback = (long)8 << 32;
            private const ulong ResultMask = ((ulong)uint.MaxValue) << 32;

            private static Action<object> s_cancelCallback;

            private readonly FileStream _stream;
            private readonly int _numBufferedBytes;
            private CancellationTokenRegistration _cancellationRegistration;
#if DEBUG
            private bool _cancellationHasBeenRegistered;
#endif
            private NativeOverlapped* _overlapped; // Overlapped class responsible for operations in progress when an appdomain unload occurs
            private long _result; // Using long since this needs to be used in Interlocked APIs

            // Using RunContinuationsAsynchronously for compat reasons (old API used Task.Factory.StartNew for continuations)
            protected FileStreamCompletionSource(FileStream stream, int numBufferedBytes, byte[] bytes)
                : base(TaskCreationOptions.RunContinuationsAsynchronously)
            {
                _numBufferedBytes = numBufferedBytes;
                _stream = stream;
                _result = NoResult;

                // Create the native overlapped. We try to use the preallocated overlapped if possible: it's possible if the byte
                // buffer is null (there's nothing to pin) or the same one that's associated with the preallocated overlapped (and
                // thus is already pinned) and if no one else is currently using the preallocated overlapped.  This is the fast-path
                // for cases where the user-provided buffer is smaller than the FileStream's buffer (such that the FileStream's
                // buffer is used) and where operations on the FileStream are not being performed concurrently.
                Debug.Assert((bytes == null || ReferenceEquals(bytes, _stream._buffer)));

                // The _preallocatedOverlapped is null if the internal buffer was never created, so we check for
                // a non-null bytes before using the stream's _preallocatedOverlapped
                _overlapped = bytes != null && _stream.CompareExchangeCurrentOverlappedOwner(this, null) == null ?
                    _stream._fileHandle.ThreadPoolBinding.AllocateNativeOverlapped(_stream._preallocatedOverlapped) :
                    _stream._fileHandle.ThreadPoolBinding.AllocateNativeOverlapped(s_ioCallback, this, bytes);
                Debug.Assert(_overlapped != null, "AllocateNativeOverlapped returned null");
            }

            internal NativeOverlapped* Overlapped
            {
                get { return _overlapped; }
            }

            public void SetCompletedSynchronously(int numBytes)
            {
                ReleaseNativeResource();
                TrySetResult(numBytes + _numBufferedBytes);
            }

            public void RegisterForCancellation(CancellationToken cancellationToken)
            {
#if DEBUG
                Debug.Assert(cancellationToken.CanBeCanceled);
                Debug.Assert(!_cancellationHasBeenRegistered, "Cannot register for cancellation twice");
                _cancellationHasBeenRegistered = true;
#endif

                // Quick check to make sure the IO hasn't completed
                if (_overlapped != null)
                {
                    var cancelCallback = s_cancelCallback;
                    if (cancelCallback == null) s_cancelCallback = cancelCallback = Cancel;

                    // Register the cancellation only if the IO hasn't completed
                    long packedResult = Interlocked.CompareExchange(ref _result, RegisteringCancellation, NoResult);
                    if (packedResult == NoResult)
                    {
                        _cancellationRegistration = cancellationToken.Register(cancelCallback, this);

                        // Switch the result, just in case IO completed while we were setting the registration
                        packedResult = Interlocked.Exchange(ref _result, NoResult);
                    }
                    else if (packedResult != CompletedCallback)
                    {
                        // Failed to set the result, IO is in the process of completing
                        // Attempt to take the packed result
                        packedResult = Interlocked.Exchange(ref _result, NoResult);
                    }

                    // If we have a callback that needs to be completed
                    if ((packedResult != NoResult) && (packedResult != CompletedCallback) && (packedResult != RegisteringCancellation))
                    {
                        CompleteCallback((ulong)packedResult);
                    }
                }
            }

            internal virtual void ReleaseNativeResource()
            {
                // Ensure that cancellation has been completed and cleaned up.
                _cancellationRegistration.Dispose();

                // Free the overlapped.
                // NOTE: The cancellation must *NOT* be running at this point, or it may observe freed memory
                // (this is why we disposed the registration above).
                if (_overlapped != null)
                {
                    _stream._fileHandle.ThreadPoolBinding.FreeNativeOverlapped(_overlapped);
                    _overlapped = null;
                }

                // Ensure we're no longer set as the current completion source (we may not have been to begin with).
                // Only one operation at a time is eligible to use the preallocated overlapped,
                _stream.CompareExchangeCurrentOverlappedOwner(null, this);
            }

            // When doing IO asynchronously (i.e. _isAsync==true), this callback is
            // called by a free thread in the threadpool when the IO operation
            // completes.
            internal static unsafe void IOCallback(uint errorCode, uint numBytes, NativeOverlapped* pOverlapped)
            {
                // Extract the completion source from the overlapped.  The state in the overlapped
                // will either be a Win32FileStream (in the case where the preallocated overlapped was used),
                // in which case the operation being completed is its _currentOverlappedOwner, or it'll
                // be directly the FileStreamCompletion that's completing (in the case where the preallocated
                // overlapped was already in use by another operation).
                object state = ThreadPoolBoundHandle.GetNativeOverlappedState(pOverlapped);
                FileStream fs = state as FileStream;
                FileStreamCompletionSource completionSource = fs != null ?
                    fs._currentOverlappedOwner :
                    (FileStreamCompletionSource)state;
                Debug.Assert(completionSource._overlapped == pOverlapped, "Overlaps don't match");

                // Handle reading from & writing to closed pipes.  While I'm not sure
                // this is entirely necessary anymore, maybe it's possible for
                // an async read on a pipe to be issued and then the pipe is closed,
                // returning this error.  This may very well be necessary.
                ulong packedResult;
                if (errorCode != 0 && errorCode != ERROR_BROKEN_PIPE && errorCode != ERROR_NO_DATA)
                {
                    packedResult = ((ulong)ResultError | errorCode);
                }
                else
                {
                    packedResult = ((ulong)ResultSuccess | numBytes);
                }

                // Stow the result so that other threads can observe it
                // And, if no other thread is registering cancellation, continue
                if (NoResult == Interlocked.Exchange(ref completionSource._result, (long)packedResult))
                {
                    // Successfully set the state, attempt to take back the callback
                    if (Interlocked.Exchange(ref completionSource._result, CompletedCallback) != NoResult)
                    {
                        // Successfully got the callback, finish the callback
                        completionSource.CompleteCallback(packedResult);
                    }
                    // else: Some other thread stole the result, so now it is responsible to finish the callback
                }
                // else: Some other thread is registering a cancellation, so it *must* finish the callback
            }

            private void CompleteCallback(ulong packedResult)
            {
                // Free up the native resource and cancellation registration
                CancellationToken cancellationToken = _cancellationRegistration.Token; // access before disposing registration
                ReleaseNativeResource();

                // Unpack the result and send it to the user
                long result = (long)(packedResult & ResultMask);
                if (result == ResultError)
                {
                    int errorCode = unchecked((int)(packedResult & uint.MaxValue));
                    if (errorCode == Interop.Errors.ERROR_OPERATION_ABORTED)
                    {
                        TrySetCanceled(cancellationToken.IsCancellationRequested ? cancellationToken : new CancellationToken(true));
                    }
                    else
                    {
                        TrySetException(Win32Marshal.GetExceptionForWin32Error(errorCode));
                    }
                }
                else
                {
                    Debug.Assert(result == ResultSuccess, "Unknown result");
                    TrySetResult((int)(packedResult & uint.MaxValue) + _numBufferedBytes);
                }
            }

            private static void Cancel(object state)
            {
                // WARNING: This may potentially be called under a lock (during cancellation registration)

                FileStreamCompletionSource completionSource = state as FileStreamCompletionSource;
                Debug.Assert(completionSource != null, "Unknown state passed to cancellation");
                Debug.Assert(completionSource._overlapped != null && !completionSource.Task.IsCompleted, "IO should not have completed yet");

                // If the handle is still valid, attempt to cancel the IO
                if (!completionSource._stream._fileHandle.IsInvalid &&
                    !Interop.Kernel32.CancelIoEx(completionSource._stream._fileHandle, completionSource._overlapped))
                {
                    int errorCode = Marshal.GetLastWin32Error();

                    // ERROR_NOT_FOUND is returned if CancelIoEx cannot find the request to cancel.
                    // This probably means that the IO operation has completed.
                    if (errorCode != Interop.Errors.ERROR_NOT_FOUND)
                    {
                        throw Win32Marshal.GetExceptionForWin32Error(errorCode);
                    }
                }
            }

            public static FileStreamCompletionSource Create(FileStream stream, int numBufferedBytesRead, ReadOnlyMemory<byte> memory)
            {
                // If the memory passed in is the stream's internal buffer, we can use the base FileStreamCompletionSource,
                // which has a PreAllocatedOverlapped with the memory already pinned.  Otherwise, we use the derived
                // MemoryFileStreamCompletionSource, which Retains the memory, which will result in less pinning in the case
                // where the underlying memory is backed by pre-pinned buffers.
                return MemoryMarshal.TryGetArray(memory, out ArraySegment<byte> buffer) && ReferenceEquals(buffer.Array, stream._buffer) ?
                    new FileStreamCompletionSource(stream, numBufferedBytesRead, buffer.Array) :
                    new MemoryFileStreamCompletionSource(stream, numBufferedBytesRead, memory);
            }
        }

        /// <summary>
        /// Extends <see cref="FileStreamCompletionSource"/> with to support disposing of a
        /// <see cref="MemoryHandle"/> when the operation has completed.  This should only be used
        /// when memory doesn't wrap a byte[].
        /// </summary>
        private sealed class MemoryFileStreamCompletionSource : FileStreamCompletionSource
        {
            private MemoryHandle _handle; // mutable struct; do not make this readonly

            internal MemoryFileStreamCompletionSource(FileStream stream, int numBufferedBytes, ReadOnlyMemory<byte> memory) :
                base(stream, numBufferedBytes, bytes: null) // this type handles the pinning, so null is passed for bytes
            {
                _handle = memory.Retain(pin: true);
            }

            internal override void ReleaseNativeResource()
            {
                _handle.Dispose();
                base.ReleaseNativeResource();
            }
        }
    }
}


text:
package swarm

import (
	"context"
	"fmt"

	"github.com/docker/cli/cli"
	"github.com/docker/cli/cli/command"
	"github.com/docker/docker/api/types/swarm"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

type joinTokenOptions struct {
	role   string
	rotate bool
	quiet  bool
}

func newJoinTokenCommand(dockerCli command.Cli) *cobra.Command {
	opts := joinTokenOptions{}

	cmd := &cobra.Command{
		Use:   "join-token [OPTIONS] (worker|manager)",
		Short: "Manage join tokens",
		Args:  cli.ExactArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			opts.role = args[0]
			return runJoinToken(dockerCli, opts)
		},
		Annotations: map[string]string{
			"version": "1.24",
			"swarm":   "manager",
		},
	}

	flags := cmd.Flags()
	flags.BoolVar(&opts.rotate, flagRotate, false, "Rotate join token")
	flags.BoolVarP(&opts.quiet, flagQuiet, "q", false, "Only display token")

	return cmd
}

func runJoinToken(dockerCli command.Cli, opts joinTokenOptions) error {
	worker := opts.role == "worker"
	manager := opts.role == "manager"

	if !worker && !manager {
		return errors.New("unknown role " + opts.role)
	}

	client := dockerCli.Client()
	ctx := context.Background()

	if opts.rotate {
		flags := swarm.UpdateFlags{
			RotateWorkerToken:  worker,
			RotateManagerToken: manager,
		}

		sw, err := client.SwarmInspect(ctx)
		if err != nil {
			return err
		}

		if err := client.SwarmUpdate(ctx, sw.Version, sw.Spec, flags); err != nil {
			return err
		}

		if !opts.quiet {
			fmt.Fprintf(dockerCli.Out(), "Successfully rotated %s join token.\n\n", opts.role)
		}
	}

	// second SwarmInspect in this function,
	// this is necessary since SwarmUpdate after first changes the join tokens
	sw, err := client.SwarmInspect(ctx)
	if err != nil {
		return err
	}

	if opts.quiet && worker {
		fmt.Fprintln(dockerCli.Out(), sw.JoinTokens.Worker)
		return nil
	}

	if opts.quiet && manager {
		fmt.Fprintln(dockerCli.Out(), sw.JoinTokens.Manager)
		return nil
	}

	info, err := client.Info(ctx)
	if err != nil {
		return err
	}

	return printJoinCommand(ctx, dockerCli, info.Swarm.NodeID, worker, manager)
}

func printJoinCommand(ctx context.Context, dockerCli command.Cli, nodeID string, worker bool, manager bool) error {
	client := dockerCli.Client()

	node, _, err := client.NodeInspectWithRaw(ctx, nodeID)
	if err != nil {
		return err
	}

	sw, err := client.SwarmInspect(ctx)
	if err != nil {
		return err
	}

	if node.ManagerStatus != nil {
		if worker {
			fmt.Fprintf(dockerCli.Out(), "To add a worker to this swarm, run the following command:\n\n    docker swarm join --token %s %s\n\n", sw.JoinTokens.Worker, node.ManagerStatus.Addr)
		}
		if manager {
			fmt.Fprintf(dockerCli.Out(), "To add a manager to this swarm, run the following command:\n\n    docker swarm join --token %s %s\n\n", sw.JoinTokens.Manager, node.ManagerStatus.Addr)
		}
	}

	return nil
}


text:
"""Tests for collect_tensor_data."""

from absl.testing import absltest
from absl.testing import parameterized
import mock
import six
import tensorflow as tf
from tf_coder.datasets import collect_tensor_data
from tf_coder.value_search import all_operations
from tf_coder.value_search import value as value_module
from tf_coder.value_search import value_search_settings as settings_module

def new_input(raw_value):
  return value_module.InputValue(raw_value, 'NEW_INPUT')

class CollectTensorDataTest(parameterized.TestCase):

  def setUp(self):
    super(CollectTensorDataTest, self).setUp()
    self.settings = settings_module.default_settings()

    operations = all_operations.get_operations()
    self.unique_with_counts_operation = all_operations.find_operation_with_name(
        'tf.unique_with_counts(x)', operation_list=operations)
    self.indexing_operation = all_operations.find_operation_with_name(
        'IndexingOperation', operation_list=operations)
    self.gather_operation = all_operations.find_operation_with_name(
        'tf.gather(params, indices)', operation_list=operations)
    self.add_operation = all_operations.find_operation_with_name(
        'tf.add(x, y)', operation_list=operations)

    # Example with many operations.
    in1 = value_module.InputValue([1, 1, 2, 5, 6, 5], 'in1')
    in2 = value_module.InputValue([0, 10, 20, 30, 40, 50, 60, 70], 'in2')
    constant_1 = value_module.ConstantValue(1)

    unique = self.unique_with_counts_operation.apply([in1], self.settings)
    indexed = self.indexing_operation.apply([unique, constant_1], self.settings)
    gathered = self.gather_operation.apply([in2, in1], self.settings)
    self.example_value_1 = self.add_operation.apply([indexed, gathered],
                                                    self.settings)

    self.assertEqual(
        self.example_value_1.reconstruct_expression(),
        'tf.add(tf.unique_with_counts(in1)[1], tf.gather(in2, in1))')
    self.assertEqual(self.example_value_1,
                     value_module.OutputValue([10, 10, 21, 52, 63, 52]))

    # Example with many variables and new inputs.
    in3 = value_module.InputValue([1], 'in3')
    in4 = value_module.InputValue([2], 'in4')

    a = self.add_operation.apply([in3, new_input([10])], self.settings)
    b = self.add_operation.apply([in4, in3], self.settings)
    c = self.add_operation.apply([new_input([20]), in3], self.settings)
    d = self.add_operation.apply([a, b], self.settings)
    self.example_value_2 = self.add_operation.apply([c, d], self.settings)

    self.assertEqual(
        self.example_value_2.reconstruct_expression(),
        'tf.add(tf.add(NEW_INPUT, in3), '
        'tf.add(tf.add(in3, NEW_INPUT), tf.add(in4, in3)))')
    self.assertEqual(self.example_value_2, value_module.OutputValue([35]))

  def test_extract_values_with_collapsed_subtrees(self):
    expected_expressions = [
        'tf.add(tf.unique_with_counts(in1)[1], tf.gather(in2, in1))',
        'tf.add(tf.unique_with_counts(in1)[1], NEW_INPUT)',
        'tf.add(NEW_INPUT[1], tf.gather(in2, in1))',
        'tf.add(NEW_INPUT[1], NEW_INPUT)',
        'tf.add(NEW_INPUT, tf.gather(in2, in1))',
        'tf.add(NEW_INPUT, NEW_INPUT)',
        'NEW_INPUT',
    ]
    results = collect_tensor_data.extract_values_with_collapsed_subtrees(
        self.example_value_1)
    # The order doesn't matter except that 'NEW_INPUT' is last.
    self.assertEqual([v.reconstruct_expression() for v in results],
                     expected_expressions)

  def test_count_num_inputs(self):
    self.assertEqual(collect_tensor_data.count_num_inputs(self.example_value_2),
                     (4, {'in3': 3, 'in4': 1, 'NEW_INPUT': 2}))

  def test_normalize_names_and_extract_values(self):
    name_counter = {'in3': 3, 'in4': 1, 'NEW_INPUT': 2}
    copied_value = self.example_value_2.copy()
    name_value_map = collect_tensor_data.normalize_names_and_extract_values(
        copied_value, name_counter)
    self.assertEqual(
        copied_value.reconstruct_expression(use_cache=False),
        # in3 -> in1, in4 -> in2, and NEW_INPUT -> in3 and in4
        'tf.add(tf.add(in3, in1), tf.add(tf.add(in1, in4), tf.add(in2, in1)))')

    self.assertEqual(name_value_map,
                     # Check the raw values.
                     {'in1': value_module.OutputValue([1]),
                      'in2': value_module.OutputValue([2]),
                      'in3': value_module.OutputValue([20]),
                      'in4': value_module.OutputValue([10])})
    for name in name_value_map:
      self.assertIsInstance(name_value_map[name], value_module.InputValue)
      self.assertEqual(name, name_value_map[name].name)

  def test_extract_operations(self):
    self.assertCountEqual(
        collect_tensor_data.extract_operations(self.example_value_1),
        [self.unique_with_counts_operation,
         self.indexing_operation,
         self.gather_operation,
         self.add_operation])
    self.assertCountEqual(
        collect_tensor_data.extract_operations(self.example_value_2),
        [self.add_operation] * 5)

  @parameterized.named_parameters(
      ('1', 1, []),
      ('2', 2, ['tf.add(tf.unique_with_counts(in1)[1], tf.gather(in2, in1))',
                'tf.add(tf.unique_with_counts(in1)[1], in2)',
                'tf.add(in1[1], in2)',
                'tf.add(in1, in2)']),
      ('3', 3, ['tf.add(tf.unique_with_counts(in1)[1], tf.gather(in2, in1))',
                'tf.add(tf.unique_with_counts(in1)[1], in2)',
                'tf.add(in3[1], tf.gather(in2, in1))',
                'tf.add(in1[1], in2)',
                'tf.add(in3, tf.gather(in2, in1))',
                'tf.add(in1, in2)']))
  def test_extract_examples_from_value(self, max_num_inputs,
                                       expected_expressions):
    actual = collect_tensor_data.extract_examples_from_value(
        self.example_value_1, max_num_inputs=max_num_inputs)

    # Check that all expressions are as expected.
    self.assertCountEqual([example.expression for example in actual],
                          expected_expressions)

    # Check all elements of one IOExample namedtuple. This example is at index 1
    # when max_num_inputs > 1.
    if max_num_inputs > 1:
      expected_index_1 = collect_tensor_data.IOExample(
          expression='tf.add(tf.unique_with_counts(in1)[1], in2)',
          input_values=[value_module.InputValue([1, 1, 2, 5, 6, 5], 'in1'),
                        value_module.InputValue([10, 10, 20, 50, 60, 50],
                                                'in2')],
          output_value=value_module.OutputValue([10, 10, 21, 52, 63, 52]),
          num_inputs=2,
          operations=[self.add_operation,
                      self.indexing_operation,
                      self.unique_with_counts_operation])
      self.assertEqual(actual[1], expected_index_1)
      # Equality of Value objects is done by comparing the wrapped values. Check
      # the names in input_values too.
      for actual_value, expected_value in zip(actual[1].input_values,
                                              expected_index_1.input_values):
        self.assertIsInstance(actual_value, value_module.InputValue)
        self.assertEqual(actual_value.name, expected_value.name)

    # Check that all extracted examples actually work by eval-ing them.
    for example in actual:
      namespace_dict = {'tf': tf}
      self.assertLen(example.input_values, example.num_inputs)
      for input_value in example.input_values:
        namespace_dict[input_value.name] = input_value.value
      eval_output = eval(example.expression, namespace_dict)  # pylint: disable=eval-used
      self.assertEqual(value_module.OutputValue(eval_output),
                       example.output_value)
      self.assertEqual(example.output_value, self.example_value_1)

  def test_extract_examples_from_value_without_inputs(self):
    constant_1 = value_module.ConstantValue(1)
    constant_2 = value_module.ConstantValue(2)
    constant_3 = value_module.ConstantValue(3)

    subtree = self.add_operation.apply([constant_1, constant_2], self.settings)
    without_inputs = self.add_operation.apply([subtree, constant_3],
                                              self.settings)
    actual = collect_tensor_data.extract_examples_from_value(without_inputs)

    self.assertCountEqual(
        [example.expression for example in actual],
        # `tf.add(tf.add(1, 2), 3)` has no inputs and is not included.
        ['tf.add(in1, 3)'])

  def test_create_tf_examples(self):
    sparse_tensor = tf.SparseTensor(
        values=[0, -15, 30],
        indices=[[12], [34], [56]],
        dense_shape=[100])
    # This example does not represent a realistic tensor transformation. It uses
    # variety in the input/output tensors to exercise the featurization.
    io_example = collect_tensor_data.IOExample(
        expression='tf.dummy_expression(in1, in2)',
        input_values=[
            value_module.InputValue([[[0.5, 2.5, 9.0],
                                      [-0.25, 0.0, 1.25]]], 'in1'),
            value_module.InputValue(sparse_tensor, 'in2'),
        ],
        output_value=value_module.OutputValue([[1.0], [0.0], [1.0], [0.0]]),
        num_inputs=2,
        operations=[self.add_operation, self.add_operation,
                    self.gather_operation])

    with mock.patch.object(collect_tensor_data, 'COUNT_BOUNDARIES',
                           new=[0, 1, 3, 50, float('inf')]):
      with mock.patch.object(
          collect_tensor_data, 'FLOAT_BOUNDARIES',
          new=[-float('inf'), -10, -1e-8, 1e-8, 10, float('inf')]):
        tf_examples = collect_tensor_data.create_tf_examples(io_example)

    operation_list = all_operations.get_operations(
        include_sparse_operations=True)
    expected_operations = [
        2 if op.name == 'tf.add(x, y)'
        else 1 if op.name == 'tf.gather(params, indices)' else 0
        for op in operation_list]

    expected_tf_example_1 = {
        # Features from featurize_value.
        'kind': [2, 2, 3, 0],
        'dtype': [8, 8, 0, 0],
        'rank': [2, 3, 1, 0],
        'shape': [4, 1, 0, 0,
                  1, 2, 3, 0,
                  100, 0, 0, 0,
                  0, 0, 0, 0],
        'shape_buckets': [2, 1, 0, 0,
                          1, 1, 2, 0,
                          3, 0, 0, 0,
                          0, 0, 0, 0],
        'floats': [1.0, 0.0, 0.5, 0.5,
                   9.0, -0.25, 13/6, 13.5/6,
                   30, -15, 5, 15,
                   0, 0, 0, 0],
        'float_buckets': [3, 2, 3, 3,
                          3, 1, 3, 3,
                          4, 0, 3, 4,
                          2, 2, 2, 2],
        'counts': [4, 4, 2, 2, 2, 0, 4, 2,
                   6, 6, 4, 1, 0, 1, 2, 6,
                   100, 3, 1, 1, 0, 1, 1, 3,
                   1, 1, 0, 1, 0, 0, 1, 1],
        'count_buckets': [2, 2, 1, 1, 1, 0, 2, 1,
                          2, 2, 2, 1, 0, 1, 1, 2,
                          3, 2, 1, 1, 0, 1, 1, 2,
                          1, 1, 0, 1, 0, 0, 1, 1],
        'fractions': [4/4, 2/4, 2/4, 2/4, 0/4, 4/4, 2/4,
                      6/6, 4/6, 1/6, 0/6, 1/6, 2/6, 6/6,
                      3/100, 1/3, 1/3, 0/3, 1/3, 1/3, 3/3,
                      1/1, 0/1, 1/1, 0/1, 0/1, 1/1, 1/1],
        'booleans': [1, 1, 0, 1, 0, 1, 1, 0,
                     1, 1, 0, 0, 0, 0, 0, 1,
                     0, 1, 0, 0, 0, 0, 0, 1,
                     1, 1, 0, 1, 0, 1, 1, 1],
        'value_string': [
            b'tf.float32:[[1.0], [0.0], [1.0], [0.0]]',
            b'tf.float32:[[[0.5, 2.5, 9.0], [-0.25, 0.0, 1.25]]]',
            (b'SparseTensor(indices=tf.Tensor(\n[[12]\n [34]\n [56]], '
             b'shape=(3, 1), dtype=int64), '
             b'values=tf.Tensor([  0 -15  30], shape=(3,), dtype=int32), '
             b'dense_shape=tf.Tensor([100], shape=(1,), dtype=int64))'),
            b'0'],

        # Features from featurize_input_and_output.
        'io_comparisons': [2, 2, 2, 0, 2, 2, 1,
                           2, 0, 0, 2, 0, 1, 1,
                           1, 1, 1, 1, 1, 1, 1],
        'io_booleans': [0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0,
                        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                        1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
        'io_counts': [1, 2, 1,
                      1, 2, 1,
                      1, 1, 1],
        'io_count_buckets': [1, 1, 1,
                             1, 1, 1,
                             1, 1, 1],
        'io_fractions': [1/6, 2/4, 1/6, 1/4,
                         1/3, 2/4, 1/3, 1/4,
                         1/1, 1/1, 1/1, 1/1],

        # Features added in create_examples.
        'num_inputs': [2],
        'operations': expected_operations,
        'expression': [b'tf.dummy_expression(in1, in2)'],
    }

    print(tf_examples)

    self.assertLen(tf_examples, 2)
    actual_tf_example_1, actual_tf_example_2 = tf_examples

    # Check the entire first example.
    for key, expected in six.iteritems(expected_tf_example_1):
      some_list = actual_tf_example_1.features.feature[key]
      if some_list.HasField('float_list'):
        actual = some_list.float_list.value
        actual = [round(f, 6) for f in actual]
        expected = [round(f, 6) for f in expected]
      elif some_list.HasField('int64_list'):
        actual = some_list.int64_list.value
      elif some_list.HasField('bytes_list'):
        actual = some_list.bytes_list.value
      else:
        self.fail('Failed to extract list from TF example.')

      # Printing side-by-side like this in the test log is more helpful than the
      # AssertionError message. Look at the Python3 log, which prints ints
      # without the L suffix.
      print('key: {}\n'
            '  expected: {}\n'
            '  got:      {}'.format(key, expected, actual))
      self.assertEqual(actual, expected)

    # Less rigorous checks for the second example, where the two inputs have
    # swapped.
    self.assertEqual(
        actual_tf_example_2.features.feature['rank'].int64_list.value,
        [2, 1, 3, 0])
    self.assertEqual(
        actual_tf_example_2.features.feature['shape'].int64_list.value,
        [4, 1, 0, 0,
         100, 0, 0, 0,
         1, 2, 3, 0,
         0, 0, 0, 0])

if __name__ == '__main__':
  absltest.main()


text:
package org.eclipse.jdt.internal.ui.preferences;

import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;

import org.eclipse.core.resources.IProject;

import org.eclipse.ui.PlatformUI;
import org.eclipse.ui.preferences.IWorkbenchPreferenceContainer;

import org.eclipse.jdt.internal.ui.IJavaHelpContextIds;
import org.eclipse.jdt.internal.ui.JavaPlugin;

/*
 * The page to configure the task tags
 */
public class TodoTaskPreferencePage extends PropertyAndPreferencePage {

	public static final String PREF_ID= "org.eclipse.jdt.ui.preferences.TodoTaskPreferencePage"; //$NON-NLS-1$
	public static final String PROP_ID= "org.eclipse.jdt.ui.propertyPages.TodoTaskPreferencePage"; //$NON-NLS-1$

	private TodoTaskConfigurationBlock fConfigurationBlock;

	public TodoTaskPreferencePage() {
		setPreferenceStore(JavaPlugin.getDefault().getPreferenceStore());
		setDescription(PreferencesMessages.TodoTaskPreferencePage_description);

		// only used when page is shown programatically
		setTitle(PreferencesMessages.TodoTaskPreferencePage_title);
	}

	/*
	 * @see org.eclipse.jface.dialogs.IDialogPage#createControl(org.eclipse.swt.widgets.Composite)
	 */
	@Override
	public void createControl(Composite parent) {
		IWorkbenchPreferenceContainer container= (IWorkbenchPreferenceContainer) getContainer();
		fConfigurationBlock= new TodoTaskConfigurationBlock(getNewStatusChangedListener(), getProject(), container);

		super.createControl(parent);

		if (isProjectPreferencePage()) {
			PlatformUI.getWorkbench().getHelpSystem().setHelp(getControl(), IJavaHelpContextIds.TODOTASK_PROPERTY_PAGE);
		} else {
			PlatformUI.getWorkbench().getHelpSystem().setHelp(getControl(), IJavaHelpContextIds.TODOTASK_PREFERENCE_PAGE);
		}
	}

	/* (non-Javadoc)
	 * @see org.eclipse.jdt.internal.ui.preferences.PropertyAndPreferencePage#createPreferenceContent(org.eclipse.swt.widgets.Composite)
	 */
	@Override
	protected Control createPreferenceContent(Composite composite) {
		return fConfigurationBlock.createContents(composite);
	}

	/* (non-Javadoc)
	 * @see org.eclipse.jdt.internal.ui.preferences.PropertyAndPreferencePage#hasProjectSpecificOptions(org.eclipse.core.resources.IProject)
	 */
	@Override
	protected boolean hasProjectSpecificOptions(IProject project) {
		return fConfigurationBlock.hasProjectSpecificOptions(project);
	}

	/* (non-Javadoc)
	 * @see org.eclipse.jdt.internal.ui.preferences.PropertyAndPreferencePage#getPreferencePageID()
	 */
	@Override
	protected String getPreferencePageID() {
		return PREF_ID;
	}

	/* (non-Javadoc)
	 * @see org.eclipse.jdt.internal.ui.preferences.PropertyAndPreferencePage#getPropertyPageID()
	 */
	@Override
	protected String getPropertyPageID() {
		return PROP_ID;
	}

	/* (non-Javadoc)
	 * @see org.eclipse.jdt.internal.ui.preferences.PropertyAndPreferencePage#enableProjectSpecificSettings(boolean)
	 */
	@Override
	protected void enableProjectSpecificSettings(boolean useProjectSpecificSettings) {
		super.enableProjectSpecificSettings(useProjectSpecificSettings);
		if (fConfigurationBlock != null) {
			fConfigurationBlock.useProjectSpecificSettings(useProjectSpecificSettings);
		}
	}

	/*
	 * @see org.eclipse.jface.preference.IPreferencePage#performDefaults()
	 */
	@Override
	protected void performDefaults() {
		super.performDefaults();
		if (fConfigurationBlock != null) {
			fConfigurationBlock.performDefaults();
		}
	}

	/*
	 * @see org.eclipse.jface.preference.IPreferencePage#performOk()
	 */
	@Override
	public boolean performOk() {
		if (fConfigurationBlock != null && !fConfigurationBlock.performOk()) {
			return false;
		}
		return super.performOk();
	}

	/*
	 * @see org.eclipse.jface.preference.IPreferencePage#performApply()
	 */
	@Override
	public void performApply() {
		if (fConfigurationBlock != null) {
			fConfigurationBlock.performApply();
		}
	}

	/* (non-Javadoc)
	 * @see org.eclipse.jface.dialogs.DialogPage#dispose()
	 */
	@Override
	public void dispose() {
		if (fConfigurationBlock != null) {
			fConfigurationBlock.dispose();
		}
		super.dispose();
	}

}


text:
package org.apache.flink.contrib.streaming.state;

import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.runtime.state.IncrementalRemoteKeyedStateHandle;
import org.apache.flink.runtime.state.KeyGroupRange;
import org.apache.flink.runtime.state.StateHandleID;
import org.apache.flink.runtime.state.StreamStateHandle;
import org.apache.flink.runtime.state.TestStreamStateHandle;
import org.apache.flink.runtime.state.memory.ByteStreamStateHandle;
import org.apache.flink.util.TestLogger;

import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
import java.util.UUID;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;

/** Test class for {@link RocksDBStateDownloader}. */
public class RocksDBStateDownloaderTest extends TestLogger {
    @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();

    /** Test that the exception arose in the thread pool will rethrow to the main thread. */
    @Test
    public void testMultiThreadRestoreThreadPoolExceptionRethrow() {
        SpecifiedException expectedException =
                new SpecifiedException("throw exception while multi thread restore.");
        StreamStateHandle stateHandle = new ThrowingStateHandle(expectedException);

        Map<StateHandleID, StreamStateHandle> stateHandles = new HashMap<>(1);
        stateHandles.put(new StateHandleID("state1"), stateHandle);

        IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
                new IncrementalRemoteKeyedStateHandle(
                        UUID.randomUUID(),
                        KeyGroupRange.EMPTY_KEY_GROUP_RANGE,
                        1,
                        stateHandles,
                        stateHandles,
                        stateHandle);

        try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(5)) {
            rocksDBStateDownloader.transferAllStateDataToDirectory(
                    incrementalKeyedStateHandle,
                    temporaryFolder.newFolder().toPath(),
                    new CloseableRegistry());
            fail();
        } catch (Exception e) {
            assertEquals(expectedException, e);
        }
    }

    /** Tests that download files with multi-thread correctly. */
    @Test
    public void testMultiThreadRestoreCorrectly() throws Exception {
        Random random = new Random();
        int contentNum = 6;
        byte[][] contents = new byte[contentNum][];
        for (int i = 0; i < contentNum; ++i) {
            contents[i] = new byte[random.nextInt(100000) + 1];
            random.nextBytes(contents[i]);
        }

        List<StreamStateHandle> handles = new ArrayList<>(contentNum);
        for (int i = 0; i < contentNum; ++i) {
            handles.add(new ByteStreamStateHandle(String.format("state%d", i), contents[i]));
        }

        Map<StateHandleID, StreamStateHandle> sharedStates = new HashMap<>(contentNum);
        Map<StateHandleID, StreamStateHandle> privateStates = new HashMap<>(contentNum);
        for (int i = 0; i < contentNum; ++i) {
            sharedStates.put(new StateHandleID(String.format("sharedState%d", i)), handles.get(i));
            privateStates.put(
                    new StateHandleID(String.format("privateState%d", i)), handles.get(i));
        }

        IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
                new IncrementalRemoteKeyedStateHandle(
                        UUID.randomUUID(),
                        KeyGroupRange.of(0, 1),
                        1,
                        sharedStates,
                        privateStates,
                        handles.get(0));

        Path dstPath = temporaryFolder.newFolder().toPath();
        try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(5)) {
            rocksDBStateDownloader.transferAllStateDataToDirectory(
                    incrementalKeyedStateHandle, dstPath, new CloseableRegistry());
        }

        for (int i = 0; i < contentNum; ++i) {
            assertStateContentEqual(
                    contents[i], dstPath.resolve(String.format("sharedState%d", i)));
        }
    }

    private void assertStateContentEqual(byte[] expected, Path path) throws IOException {
        byte[] actual = Files.readAllBytes(Paths.get(path.toUri()));
        assertArrayEquals(expected, actual);
    }

    private static class SpecifiedException extends IOException {
        SpecifiedException(String message) {
            super(message);
        }
    }

    private static class ThrowingStateHandle implements TestStreamStateHandle {
        private static final long serialVersionUID = -2102069659550694805L;

        private final IOException expectedException;

        private ThrowingStateHandle(IOException expectedException) {
            this.expectedException = expectedException;
        }

        @Override
        public FSDataInputStream openInputStream() throws IOException {
            throw expectedException;
        }

        @Override
        public Optional<byte[]> asBytesIfInMemory() {
            return Optional.empty();
        }

        @Override
        public void discardState() {}

        @Override
        public long getStateSize() {
            return 0;
        }
    }
}


text:
<?xml version="1.0" encoding="iso-8859-1"?>
<!DOCTYPE html
     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

  <title>makedirs (Sys)</title>
  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />

     # File lib/rake/contrib/sys.rb, line 135
135:   def makedirs(*dirs)
136:     dirs.each do |fn|
137:       log "Making directory #{fn}"
138:       File.makedirs(fn)
139:     end
140:   end


text:
'use strict';

Object.defineProperty(exports, "__esModule", {
  value: true
});
var InputTimeRole = {
  relatedConcepts: [{
    module: 'HTML',
    concept: {
      name: 'input',
      attributes: [{
        name: 'type',
        value: 'time'
      }]
    }
  }],
  type: 'widget'
};

exports.default = InputTimeRole;


text:
#import "NSObject.h"

@class NSDictionary, NSObject<OS_dispatch_queue>, NSObject<OS_xpc_object>;

@interface CLTilesManagerClient : NSObject
{
    NSObject<OS_xpc_object> *fXpcConnection;
    NSObject<OS_dispatch_queue> *fQueue;
    NSDictionary *fNotificationHandlers;
}

+ (id)sharedClient;
+ (void)initialize;
- (_Bool)unregisterFrom:(id)arg1;
- (_Bool)registerTo:(id)arg1 onQueue:(id)arg2 block:(id)arg3;
- (void)sendNotificationRegistrationMessage:(id)arg1 isRegister:(_Bool)arg2;
- (_Bool)precacheHint:(id)arg1 onQueue:(id)arg2 responseBlock:(id)arg3;
- (_Bool)executeAsynchronousQuery:(id)arg1 onQueue:(id)arg2 responseBlock:(id)arg3;
- (void)dealloc;
- (id)init;
- (void)onNotificationReceived:(id)arg1;
- (void)reconnect;

@end


text:
using Cairo

function common_header()
    ## header to provide surface and context
    c = CairoRGBSurface(256,256);
    cr = CairoContext(c);

    save(cr);
    set_source_rgb(cr,0.8,0.8,0.8);    # light gray
    rectangle(cr,0.0,0.0,256.0,256.0); # background
    fill(cr);
    restore(cr);

    return c,cr
end

function write_out_picture(filename::String,c::CairoSurface)
    ## mark picture with current date
    move_to(cr,0.0,12.0);
    set_source_rgb (cr, 0,0,0);
    show_text(cr,strftime(time()));
    ##
    write_to_png(c,filename);
    nothing
end

function example_copy_path(cr)

    save(cr);
    # single (large) character set with text_path+stroke
    select_font_face (cr, "Sans", Cairo.FONT_SLANT_NORMAL,
                        Cairo.FONT_WEIGHT_BOLD);
    set_font_size (cr, 100.0);
    translate(cr, 10.0, 100.0);
    text_path (cr, "J");
    stroke_preserve(cr);

    # copied and converted
    opath = Cairo.convert_cairo_path_data(Cairo.copy_path(cr));

    restore(cr);

    l = 2 # something like a line counter

    # loop over all entries and just print out command and available points
    for x in opath
        if x.element_type == Cairo.CAIRO_PATH_MOVE_TO
            s0 = "moveto"
        elseif x.element_type == Cairo.CAIRO_PATH_LINE_TO
            s0 = "lineto"
        elseif x.element_type == Cairo.CAIRO_PATH_CURVE_TO
            s0 = "curveto"
        elseif x.element_type == Cairo.CAIRO_PATH_CLOSE_PATH
            s0 = "closepath"
        end
        s1 = repr(x.points)
        move_to(cr,100.0,14.0*l)
        l += 1
        show_text(cr,s0*s1); #
    end
    nothing
end

c,cr = common_header();
save(cr);
example_copy_path(cr);
restore(cr);
write_out_picture("sample_copy_path.png",c);


text:
package com.mopub.mobileads.util;

import android.os.AsyncTask;

import java.util.concurrent.*;

import static com.mopub.mobileads.util.Reflection.MethodBuilder;
import static com.mopub.mobileads.util.VersionCode.ICE_CREAM_SANDWICH;
import static com.mopub.mobileads.util.VersionCode.currentApiLevel;

public class AsyncTasks {
    public static <P> void safeExecuteOnExecutor(AsyncTask<P, ?, ?> asyncTask, P... params) throws Exception {
        if (asyncTask == null) {
            throw new IllegalArgumentException("Unable to execute null AsyncTask.");
        }

        if (currentApiLevel().isAtLeast(ICE_CREAM_SANDWICH)) {
            Executor threadPoolExecutor = (Executor) AsyncTask.class.getField("THREAD_POOL_EXECUTOR").get(AsyncTask.class);

            new MethodBuilder(asyncTask, "executeOnExecutor")
                    .addParam(Executor.class, threadPoolExecutor)
                    .addParam(Object[].class, params)
                    .execute();
        } else {
            asyncTask.execute(params);
        }
    }
}


text:
<?php namespace Backend\Classes;

/**
 * Report Widget base class
 * Report widgets are used inside the ReportContainer.
 *
 * @package backend
 *
 */
class ReportWidgetBase extends WidgetBase
{
    use \System\Traits\PropertyContainer;

    public function __construct($controller, $properties = [])
    {
        $this->properties = $this->validateProperties($properties);

        parent::__construct($controller);
    }
}


text:
@interface FOONSDocumentsDirectoryLocatorTests : XCTestCase {
    FOONSDocumentsDirectoryLocator *testObject;
    NSFileManager *fileManager;
}
@end

@implementation FOONSDocumentsDirectoryLocatorTests

- (void)setUp {
    [super setUp];
    fileManager = mock([NSFileManager class]);
    testObject = [[FOONSDocumentsDirectoryLocator alloc] initWithFileManager:fileManager];
}

- (void)testApplicationDocumentsDirectory {
    NSURL *url1 = mock([NSURL class]);
    NSURL *url2 = mock([NSURL class]);

    [when([fileManager URLsForDirectory:NSDocumentDirectory
                              inDomains:NSUserDomainMask]) thenReturn:@[url1, url2]];

    NSURL *actual = [testObject applicationDocumentsDirectory];

    XCTAssertEqualObjects(url2, actual);
}

@end


text:
// apollo/modules/common/util/file.cc

#include "utils/apollo_file.h"

#include <dirent.h>
#include <errno.h>
#include <limits.h>
#include <fstream>

#include "utils/apollo_string_util.h"

namespace bubblefs {
namespace myapollo {
namespace common {
namespace util {
namespace {

std::string GetRosHome() {
  // Note that ROS_ROOT env points to <ROS_HOME>/share/ros.
  const std::string known_tail = "/share/ros";
  const std::string ros_root = std::getenv("ROS_ROOT");
  PANIC_ENFORCE(EndWith(ros_root, known_tail), "not /share/ros");
  return ros_root.substr(0, ros_root.length() - known_tail.length());
}

}  // namespace

bool GetContent(const std::string &file_name, std::string *content) {
  std::ifstream fin(file_name);
  if (!fin) {
    return false;
  }

  std::stringstream str_stream;
  str_stream << fin.rdbuf();
  *content = str_stream.str();
  return true;
}

std::string TranslatePath(const std::string &src_path) {
  static const std::string kRosHomePlaceHolder = "<ros>";
  static const std::string kRosHome = GetRosHome();

  std::string result(src_path);

  // Replace ROS home place holder.
  const auto pos = src_path.find(kRosHomePlaceHolder);
  if (pos != std::string::npos) {
    result.replace(pos, kRosHomePlaceHolder.length(), kRosHome);
  }

  return result;
}

std::string GetAbsolutePath(const std::string &prefix,
                            const std::string &relative_path) {
  if (relative_path.empty()) {
    return prefix;
  }
  // If prefix is empty or relative_path is already absolute.
  if (prefix.empty() || relative_path[0] == '/') {
    return relative_path;
  }

  if (prefix.back() == '/') {
    return (prefix + relative_path);
  }
  return (prefix + "/" + relative_path);
}

bool PathExists(const std::string &path) {
  struct stat info;
  return stat(path.c_str(), &info) == 0;
}

bool DirectoryExists(const std::string &directory_path) {
  struct stat info;
  if (stat(directory_path.c_str(), &info) != 0) {
    return false;
  }

  if (info.st_mode & S_IFDIR) {
    return true;
  }

  return false;
}

bool CopyFile(const std::string &from, const std::string &to) {
  std::ifstream src(from, std::ios::binary);
  if (!src) {
    PRINTF_ERROR("Source path doesn't exist: %s\n", from.c_str());
    return false;
  }

  std::ofstream dst(to, std::ios::binary);
  if (!dst) {
    PRINTF_ERROR("Target path is not writable: %s\n", to.c_str());
    return false;
  }

  dst << src.rdbuf();
  return true;
}

bool CopyDir(const std::string &from, const std::string &to) {
  DIR *directory = opendir(from.c_str());
  if (directory == nullptr) {
    PRINTF_ERROR("Cannot open directory %s\n", from.c_str());
    return false;
  }
  if (!EnsureDirectory(to)) {
    PRINTF_ERROR("Cannot create target directory %s\n", to.c_str());
    return false;
  }

  struct dirent *entry;
  bool ret = true;
  while ((entry = readdir(directory)) != nullptr) {
    // skip directory_path/. and directory_path/..
    if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) {
      continue;
    }
    const std::string sub_path_from = from + "/" + entry->d_name;
    const std::string sub_path_to = to + "/" + entry->d_name;
    if (entry->d_type == DT_DIR) {
      ret = CopyDir(sub_path_from, sub_path_to) && ret;
    } else {
      ret = CopyFile(sub_path_from, sub_path_to) && ret;
    }
  }
  closedir(directory);
  return ret;
}

bool Copy(const std::string &from, const std::string &to) {
  return DirectoryExists(from) ? CopyDir(from, to) : CopyFile(from, to);
}

bool EnsureDirectory(const std::string &directory_path) {
  std::string path = directory_path;
  for (size_t i = 1; i < directory_path.size(); ++i) {
    if (directory_path[i] == '/') {
      // Whenever a '/' is encountered, create a temporary view from
      // the start of the path to the character right before this.
      path[i] = 0;

      if (mkdir(path.c_str(), S_IRWXU) != 0) {
        if (errno != EEXIST) {
          return false;
        }
      }

      // Revert the temporary view back to the original.
      path[i] = '/';
    }
  }

  // Make the final (full) directory.
  if (mkdir(path.c_str(), S_IRWXU) != 0) {
    if (errno != EEXIST) {
      return false;
    }
  }

  return true;
}

bool RemoveAllFiles(const std::string &directory_path) {
  DIR *directory = opendir(directory_path.c_str());
  if (directory == nullptr) {
    PRINTF_ERROR("Cannot open directory %s\n", directory_path.c_str());
    return false;
  }

  struct dirent *file;
  while ((file = readdir(directory)) != nullptr) {
    // skip directory_path/. and directory_path/..
    if (!strcmp(file->d_name, ".") || !strcmp(file->d_name, "..")) {
      continue;
    }
    // build the path for each file in the folder
    std::string file_path = directory_path + "/" + file->d_name;
    if (unlink(file_path.c_str()) < 0) {
      PRINTF_ERROR("Fail to remove file %s : %s\n", file_path.c_str(), strerror(errno));
      closedir(directory);
      return false;
    }
  }
  closedir(directory);
  return true;
}

std::vector<std::string> ListSubDirectories(const std::string &directory_path) {
  std::vector<std::string> result;
  DIR *directory = opendir(directory_path.c_str());
  if (directory == nullptr) {
    PRINTF_ERROR("Cannot open directory %s\n", directory_path.c_str());
    return result;
  }

  struct dirent *entry;
  while ((entry = readdir(directory)) != nullptr) {
    // skip directory_path/. and directory_path/..
    if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) {
      continue;
    }

    if (entry->d_type == DT_DIR) {
      result.emplace_back(entry->d_name);
    }
  }
  closedir(directory);
  return result;
}

}  // namespace util
}  // namespace common
}  // namespace myapollo
}  // namespace bubblefs


text:
!
! Copyright (c) 2014, Matthew Emmett and Michael Minion.  All rights reserved.
!

! Transfer (interpolate, restrict) routines.

module transfer
  use feval
  use multigrid
  implicit none
contains

  subroutine interpolate(qF, qG, fine, crse, t)
    real(pfdp),     intent(in   ) :: t, qG(:)
    real(pfdp),     intent(  out) :: qF(:)
    type(pf_level), intent(in   ) :: fine, crse
    call interp(qF, size(qF), qG)
  end subroutine interpolate

  subroutine restrict(qF, qG, fine, crse, t)
    real(pfdp),     intent(in   ) :: t, qF(:)
    real(pfdp),     intent(  out) :: qG(:)
    type(pf_level), intent(in   ) :: fine, crse
    integer :: xrat
    xrat  = size(qF) / size(qG)
    qG = qF(::xrat)
  end subroutine restrict
end module transfer


text:
<?php
/**
 * @file
 * Override Symfony Console's TextDescriptor to customize the appearance of the
 * command list and each command's help.
 */

namespace Platformsh\Cli\Console;

use Platformsh\Cli\Command\CommandBase;
use Symfony\Component\Console\Application as ConsoleApplication;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Descriptor\ApplicationDescription;
use Symfony\Component\Console\Descriptor\TextDescriptor;
use Symfony\Component\Console\Helper\Helper;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Terminal;

class CustomTextDescriptor extends TextDescriptor
{
    protected $cliExecutableName;

    /**
     * @param string|null $cliExecutableName
     *   The name of the CLI command.
     */
    public function __construct($cliExecutableName = null)
    {
        $this->cliExecutableName = $cliExecutableName ?: basename($_SERVER['PHP_SELF']);
    }

    /**
     * @inheritdoc
     */
    protected function describeCommand(Command $command, array $options = [])
    {
        $command->getSynopsis();
        $command->mergeApplicationDefinition(false);

        $this->writeText("<comment>Command:</comment> " . $command->getName(), $options);

        $aliases = $command instanceof CommandBase ? $command->getVisibleAliases() : $command->getAliases();
        if ($aliases) {
            $this->writeText("\n");
            $this->writeText('<comment>Aliases:</comment> ' . implode(', ', $aliases), $options);
        }

        if ($description = $command->getDescription()) {
            $this->writeText("\n");
            $this->writeText("<comment>Description:</comment> $description", $options);
        }
        $this->writeText("\n\n");

        $this->writeText('<comment>Usage:</comment>', $options);
        $this->writeText("\n");
        $this->writeText(' ' . $command->getSynopsis(), $options);
        $this->writeText("\n");

        if ($definition = $command->getNativeDefinition()) {
            $this->writeText("\n");
            $this->describeInputDefinition($definition, $options);
            $this->writeText("\n");
        }

        if ($help = $command->getProcessedHelp()) {
            $this->writeText("\n");
            $this->writeText('<comment>Help:</comment>', $options);
            $this->writeText("\n");
            $this->writeText(' ' . str_replace("\n", "\n ", $help), $options);
            $this->writeText("\n");
        }

        if ($command instanceof CommandBase && ($examples = $command->getExamples())) {
            $this->writeText("\n");
            $this->writeText('<comment>Examples:</comment>', $options);
            $name = $command->getName();
            $shortName = count($aliases) === 1 ? reset($aliases) : $name;
            foreach ($examples as $arguments => $description) {
                $this->writeText("\n $description:\n   <info>{$this->cliExecutableName} $shortName $arguments</info>\n");
            }
        }
    }

    /**
     * @inheritdoc
     */
    protected function describeApplication(ConsoleApplication $application, array $options = [])
    {
        $describedNamespace = isset($options['namespace']) ? $options['namespace'] : null;
        $description = new ApplicationDescription($application, $describedNamespace, !empty($options['all']));

        if (isset($options['raw_text']) && $options['raw_text']) {
            $width = $this->getColumnWidth($description->getCommands());

            foreach ($description->getCommands() as $command) {
                $this->writeText(sprintf("%-${width}s %s", $command->getName(), $command->getDescription()), $options);
                $this->writeText("\n");
            }
        } else {
            $width = $this->getColumnWidth($description->getCommands());

            $this->writeText($application->getHelp(), $options);
            $this->writeText("\n\n");

            if ($describedNamespace) {
                $this->writeText(
                    sprintf("<comment>Available commands for the \"%s\" namespace:</comment>", $application->findNamespace($describedNamespace)),
                    $options
                );
            } else {
                $this->writeText('<comment>Available commands:</comment>', $options);
            }

            // Display commands grouped by namespace.
            foreach ($description->getNamespaces() as $namespace) {
                // Filter hidden commands in the namespace.
                /** @var Command[] $commands */
                $commands = [];
                foreach ($namespace['commands'] as $name) {
                    $command = $description->getCommand($name);

                    // Ensure the command is only shown under its canonical name.
                    if ($name === $command->getName()) {
                        $commands[$name] = $command;
                    }
                }

                // Skip the namespace if it doesn't contain any commands.
                if (!count($commands)) {
                    continue;
                }

                // Display the namespace name.
                if (!$describedNamespace && ApplicationDescription::GLOBAL_NAMESPACE !== $namespace['id']) {
                    $this->writeText("\n");
                    $this->writeText('<comment>' . $namespace['id'] . '</comment>', $options);
                }

                // Display each command.
                foreach ($commands as $name => $command) {
                    $aliases = $command instanceof CommandBase
                        ? $command->getVisibleAliases()
                        : $command->getAliases();

                    $this->writeText("\n");
                    $this->writeText(
                        sprintf(
                            "  %-${width}s %s",
                            "<info>$name</info>" . $this->formatAliases($aliases),
                            $command->getDescription()
                        ),
                        $options
                    );
                }
            }

            $this->writeText("\n");
        }
    }

    /**
     * {@inheritdoc}
     */
    protected function writeText($content, array $options = [])
    {
        $this->write(
            isset($options['raw_text']) && $options['raw_text'] ? strip_tags($content) : $content,
            isset($options['raw_output']) ? !$options['raw_output'] : true
        );
    }

    /**
     * @param array $aliases
     *
     * @return string
     */
    protected function formatAliases(array $aliases)
    {
        return $aliases ? " (" . implode(', ', $aliases) . ")" : '';
    }

    /**
     * Formats input option/argument default value.
     *
     * @param mixed $default
     *
     * @return string
     */
    protected function formatDefaultValue($default)
    {
        if (PHP_VERSION_ID < 50400) {
            return str_replace('\/', '/', json_encode($default));
        }

        return json_encode($default, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
    }

    /**
     * @param Command[] $commands
     *
     * @return int
     */
    protected function getColumnWidth(array $commands)
    {
        $width = 0;
        foreach ($commands as $command) {
            $aliasesString = $this->formatAliases($command->getAliases());
            $commandWidth = strlen($command->getName()) + strlen($aliasesString);
            $width = $commandWidth > $width ? $commandWidth : $width;
        }

        // Limit to a maximum.
        $terminalWidth = (new Terminal())->getWidth();
        if ($width / $terminalWidth > 0.4) {
            $width = floor($terminalWidth * 0.4);
        }

        // Start at a minimum.
        if ($width < 20) {
            $width = 20;
        }

        // Add the indent.
        $width += 2;

        // Accommodate tags.
        $width += strlen('<info></info>');

        return $width;
    }

    /**
     * {@inheritdoc}
     */
    protected function describeInputOption(InputOption $option, array $options = array())
    {
        if ($option->acceptValue() && null !== $option->getDefault() && (!is_array($option->getDefault()) || count($option->getDefault()))) {
            $default = sprintf('<comment> [default: %s]</comment>', $this->formatDefaultValue($option->getDefault()));
        } else {
            $default = '';
        }

        $value = '';
        if ($option->acceptValue()) {
            $value = '='.strtoupper($option->getName());

            if ($option->isValueOptional()) {
                $value = '['.$value.']';
            }
        }

        $totalWidth = isset($options['total_width']) ? $options['total_width'] : $this->calculateTotalWidthForOptions(array($option));
        $synopsis = sprintf('%s%s',
            $option->getShortcut() ? sprintf('-%s, ', $option->getShortcut()) : '    ',
            sprintf('--%s%s', $option->getName(), $value)
        );

        $spacingWidth = $totalWidth - Helper::strlen($synopsis);

        // Ensure the description is indented and word-wrapped to fit the
        // terminal width.
        $descriptionWidth = (new Terminal())->getWidth() - $totalWidth - 4;
        $description = $option->getDescription();
        $description .= $default;
        if ($option->isArray()) {
            $description .= '<comment> (multiple values allowed)</comment>';
        }
        $description = preg_replace('/\s*[\r\n]\s*/', "\n".str_repeat(' ', $totalWidth + 4), wordwrap($description, $descriptionWidth));

        $this->writeText(sprintf('  <info>%s</info>  %s%s',
            $synopsis,
            str_repeat(' ', $spacingWidth),
            $description
        ), $options);
    }

    /**
     * @param InputOption[] $options
     *
     * @return int
     */
    private function calculateTotalWidthForOptions(array $options)
    {
        $totalWidth = 0;
        foreach ($options as $option) {
            // "-" + shortcut + ", --" + name
            $nameLength = 1 + max(Helper::strlen($option->getShortcut()), 1) + 4 + Helper::strlen($option->getName());

            if ($option->acceptValue()) {
                $valueLength = 1 + Helper::strlen($option->getName()); // = + value
                $valueLength += $option->isValueOptional() ? 2 : 0; // [ + ]

                $nameLength += $valueLength;
            }
            $totalWidth = max($totalWidth, $nameLength);
        }

        return $totalWidth;
    }
}


text:
Zig Zag

            Godard single-origin coffee twee before they sold out, distillery whatever dolore narwhal ex squid proident art party. Heirloom sint church-key, direct trade lumbersexual McSweeney's Odd Future readymade. Chambray accusamus art party, put a bird on it adipisicing bitters labore meh pork belly aute.

 Helvetica cold-pressed dolor, 3 wolf moon heirloom vegan consequat eiusmod velit umami scenester retro hashtag anim Brooklyn. Delectus Thundercats vinyl, Blue Bottle organic street art you probably haven't heard of them reprehenderit. Cardigan ugh leggings YOLO elit mumblecore. Hashtag street art four dollar toast meditation plaid quis.

            www.myprojectlink.com


text:
namespace v8_inspector {

template <typename T>
std::unique_ptr<T> wrapUnique(T* ptr) {
  return std::unique_ptr<T>(ptr);
}

}  // namespace v8_inspector

#endif  // V8_INSPECTOR_PROTOCOLPLATFORM_H_


text:
<?xml version="1.0"?>
<layout version="0.1.0">
<default>
         <reference name="root">
            <action method="setTemplate"><template>page/2columns-left.phtml</template></action>
       <block type="core/template" name="full-width-slider" template="kartparadigm-home/kartparadigm-home-page-slider.phtml" />
     </reference>
      <remove name="paypal.partner.right.logo" />

 </default>
 <reference name="right">

   </reference>
    <cms_index_index translate="label">
    	<reference name="root">
            <block type="core/template" name="full-width-slider_home"  template="home/home-slider.phtml" />
        </reference>
        <remove name='full-width-slider'/>
    </cms_index_index>
  <catalog_product_view translate="label">
    <label>Catalog Product View (Any)</label>
    <!-- Mage_Catalog -->
    <reference name="root">
        <action method="setTemplate"><template>page/2columns-right.phtml</template>
        </action>
    </reference>
    	 </catalog_product_view>
 </layout>


text:
---
uid: SolidEdgeFrameworkSupport.CenterLine.Rotate(System.Double,System.Double,System.Double)
summary:
remarks:
syntax:
    parameters:
    - id: Angle
      description: Specifies, in radians, the angle of rotation.
    - id: x
      description: Specifies the x coordinate for the point of rotation.
    - id: y
      description: Specifies the y coordinate for the point of rotation.
example: [*content]
---
[!code-vb[Main](..\snippets\SolidEdgeFrameworkSupport.CenterLine.Rotate.vb]
[!code-csharp[Main](..\snippets\SolidEdgeFrameworkSupport.CenterLine.Rotate.cs]


text:
function [dataA, dataB] = processBuffer(buffer, verticalScale)
% PROCESSBUFFER takes an input buffer of uint8s and splits into two output
% buffers, dataA and dataB, that are doubles. It scales the input data
% from [0, 255] to (-Vs, Vs).


text:
var krb5;

module.exports = krb5 = {
  kinit: function(config) {
    var command;
    command = "kinit";
    if (config.keytab === true) {
      " -k";
    } else if (config.keytab && typeof config.keytab === 'string') {
      command += ` -kt ${config.keytab}`;
    } else if (config.password) {
      command = `echo ${config.password} | ${command}`;
    } else {
      throw Error("Incoherent config: expects one of keytab or password");
    }
    command += ` ${config.principal}`;
    return command = krb5.su(config, command);
  },
  su: function(config, command) {
    if (config.uid) {
      command = `su - ${config.uid} -c '${command}'`;
    }
    return command;
  }
};


text:
using LiveSplit.UI;
using LiveSplit.Web;
using System;
using System.Globalization;
using System.Xml;

namespace LiveSplit.Model
{
    public struct Attempt
    {
        public int Index { get; set; }
        public Time Time { get; set; }
        public AtomicDateTime? Started { get; set; }
        public AtomicDateTime? Ended { get; set; }
        public TimeSpan? PauseTime { get; set; }

        /// <summary>
        /// Returns the Real Time Duration of the attempt.
        /// This either returns a 1.6+ Time Stamp based duration
        /// or the duration of the run (assuming it's not resetted)
        /// if it's from before LiveSplit 1.6. If it is from before
        /// 1.6 and resetted then it will return null.
        /// </summary>
        public TimeSpan? Duration
        {
            get
            {
                if (Ended.HasValue && Started.HasValue)
                    return Ended - Started;
                else
                    return Time.RealTime;
            }
        }

        public Attempt(int index, Time time, AtomicDateTime? started, AtomicDateTime? ended, TimeSpan? pauseTime)
            : this()
        {
            Index = index;
            Time = time;
            Started = started;
            Ended = ended;
            PauseTime = pauseTime;
        }

        public XmlNode ToXml(XmlDocument document)
        {
            var attempt = document.CreateElement("Attempt");

            var time = Time.ToXml(document);
            attempt.InnerXml = time.InnerXml;

            var id = document.CreateAttribute("id");
            id.InnerText = Index.ToString();
            attempt.Attributes.Append(id);

            if (Started.HasValue)
            {
                var started = document.CreateAttribute("started");
                started.InnerText = Started.Value.Time.ToUniversalTime().ToString(CultureInfo.InvariantCulture);
                attempt.Attributes.Append(started);
                attempt.Attributes.Append(SettingsHelper.ToAttribute(document, "isStartedSynced", Started.Value.SyncedWithAtomicClock));
            }

            if (Ended.HasValue)
            {
                var ended = document.CreateAttribute("ended");
                ended.InnerText = Ended.Value.Time.ToUniversalTime().ToString(CultureInfo.InvariantCulture);
                attempt.Attributes.Append(ended);
                attempt.Attributes.Append(SettingsHelper.ToAttribute(document, "isEndedSynced", Ended.Value.SyncedWithAtomicClock));
            }

            if (PauseTime.HasValue)
            {
                var pauseTime = document.CreateElement("PauseTime");
                pauseTime.InnerText = PauseTime.ToString();
                attempt.AppendChild(pauseTime);
            }

            return attempt;
        }

        public static Attempt ParseXml(XmlElement node)
        {
            var newTime = Time.FromXml(node);
            var index = int.Parse(node.Attributes["id"].InnerText, CultureInfo.InvariantCulture);
            AtomicDateTime? started = null;
            var startedSynced = false;
            AtomicDateTime? ended = null;
            var endedSynced = false;

            if (node.HasAttribute("started"))
            {
                var startedTime = DateTime.Parse(node.Attributes["started"].InnerText, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal);
                if (node.HasAttribute("isStartedSynced"))
                    startedSynced = bool.Parse(node.Attributes["isStartedSynced"].InnerText);
                started = new AtomicDateTime(startedTime, startedSynced);
            }

            if (node.HasAttribute("ended"))
            {
                var endedTime = DateTime.Parse(node.Attributes["ended"].InnerText, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal);
                if (node.HasAttribute("isEndedSynced"))
                    endedSynced = bool.Parse(node.Attributes["isEndedSynced"].InnerText);
                ended = new AtomicDateTime(endedTime, endedSynced);
            }

            TimeSpan? pauseTime = null;
            if (node.GetElementsByTagName("PauseTime").Count > 0)
            {
                TimeSpan x;
                if (TimeSpan.TryParse(node["PauseTime"].InnerText, out x))
                    pauseTime = x;
            }

            return new Attempt(index, newTime, started, ended, pauseTime);
        }

        public DynamicJsonObject ToJson()
        {
            dynamic json = new DynamicJsonObject();
            json.id = Index;
            json.realTime = Time.RealTime;
            json.gameTime = Time.GameTime;
            json.started = Started;
            json.ended = Ended;
            json.pauseTime = PauseTime;
            return json;
        }
    }
}


text:
require 'rails_helper'

describe "Home Page" do
  it "renders" do
    visit "/"
    expect(page).to have_content "Sonoma County Fires"
    expect(page).to have_content "Lost & Found"
  end

  it "accepts reports" do
    visit "/"
    click_link "Report an Animal"
    expect(current_path).to eq "/reports/new"
  end
end


text:
ZappiFactory::ZappiFactory(){

}
ZappiFactory::~ZappiFactory(){

}

Zappi* ZappiFactory::generarZappiRandom(){
    Pizza Napolitana ;
    Napolitana.nombre = "Grande de Napolitana";
    Napolitana.duracion = 15;
    Napolitana.precio = 120;
    Pizza Muzzarella;
    Muzzarella.nombre = "Grande de Muzzarella";
    Muzzarella.duracion = 10;
    Muzzarella.precio = 100;
    Pizza Fugazetta;
    Fugazetta.nombre = "Grande de Fugazetta";
    Fugazetta.duracion = 20;
    Fugazetta.precio = 160;
    std::vector<Pizza> vectorGustos;
    vectorGustos.push_back(Napolitana);
    vectorGustos.push_back(Muzzarella);
    vectorGustos.push_back(Fugazetta);
    int posicion =(rand() % 3);
    Pizza elegida = vectorGustos[posicion];
    Zappi* zappi = new Zappi(elegida.nombre, elegida.duracion, elegida.precio);
    return zappi;
}


text:
layout: post
theme: UbuntuMono
title: Microservices - Service Discovery
excerpt: "Clients of a service use either Client-side discovery or Server-side discovery to determine the location of a service instance to which to send requests. Why use Service Discovery?"
modified: 2017-08-10T11:51:25-04:00
categories: articles
tags: [Service Discovery, Scalability, Spring Cloud, Microservices]
image:
  vendor: unsplash
  feature: /photo-1500479694472-551d1fb6258d?dpr=1.5&auto=format&fit=crop&w=1500&h=1000&q=80&cs=tinysrgb&crop=
  credit: Scott Walsh
  creditlink: https://unsplash.com/@outsighted
comments: true
share: true
references:
  - title: "Microservice Architecture - Pattern: Service registry"
    url: "http://microservices.io/patterns/service-registry.html"
  - title: "Microservice Architecture - Pattern: Client-side service discovery"
    url: "http://microservices.io/patterns/client-side-discovery.html"
  - title: "Docker: using containers to implement a microservices architecture"
    url: "https://alexandreesl.com/2016/01/08/docker-using-containers-to-implement-a-microservices-architecture/"
  - title: "Spring Cloud Netflix"
    url: "http://cloud.spring.io/spring-cloud-netflix/spring-cloud-netflix.html"
  - title: "Introduction to Spring Cloud Netflix – Eureka"
    url: "http://www.baeldung.com/spring-cloud-netflix-eureka"
  - title: "Docker networking and service discovery"
    url: "https://www.oreilly.com/learning/docker-networking-service-discovery"
  - title: "Service Discovery in a Microservices Architecture"
    url: "https://www.nginx.com/blog/service-discovery-in-a-microservices-architecture/"

---

* TOC
{:toc}

## Concept
Assume we have three services, **Hero**, **Villain** and **Police office**

```
* Villain       -> Police office "I'm here!"
* Police office -> Hero          "Move!"
* Hero          -> Villain       "Catch you!"
* Hero          -> Police office "Give him to you!"
```

## Service Discovery

Clients of a service use either **Client-side discovery** or **Server-side discovery** to determine the location of a service instance to which to send requests.

Why Use Service Discovery? Please refer to the section in [Service Discovery in a Microservices Architecture](https://www.nginx.com/blog/service-discovery-in-a-microservices-architecture/)

### Service Discovery Products

* [Eureka](https://github.com/Netflix/eureka)
* [ZooKeeper](https://zookeeper.apache.org/)
* [Consul](https://www.consul.io/)
* [etcd](https://github.com/coreos/etcd)

[Spring Cloud Netflix](https://cloud.spring.io/spring-cloud-netflix/)

[Spring Cloud Zookeeper](https://cloud.spring.io/spring-cloud-zookeeper/)

[Consul vs. Other Software](https://www.consul.io/intro/vs/zookeeper.html)

## Building

### Parent POM

首先配置 parent maven 配置文件:

```xml
<parent>
  <groupId>org.springframework.cloud</groupId>
  <artifactId>spring-cloud-starter-parent</artifactId>
  <version>Dalston.SR2</version>
  <relativePath/>
</parent>

<groupId>wang.tiven.microservices</groupId>
<artifactId>service-registration-and-discovery</artifactId>
<version>0.0.1-SNAPSHOT</version>

<packaging>pom</packaging>

<modules>
  <module>villain-service</module>
  <module>hero-service</module>
  <module>police-service</module>
  <module>eureka-service</module>
</modules>
```

* 设置本项目的 parent 为 `spring-cloud-starter-parent` 版本为本文编写时最新稳定版 `Dalston.SR2`
* 并包含几个子模块 `eureka-service` 提供本地测试用的 service registry 服务
* 其他三个子模块分别对应我们的 Hero Villain 和 Police

### Eureka Service

> [Eureka][eureka] is a REST (Representational State Transfer) based service that is primarily used in the AWS cloud for locating services for the purpose of load balancing and failover of middle-tier servers.
>
> At Netflix, Eureka is used for the following purposes apart from playing a critical part in mid-tier load balancing.
{: .Quotes}

> [Spring Cloud Netflix][spring-cloud-netflix] provides Netflix OSS integrations for Spring Boot apps through autoconfiguration and binding to the Spring Environment and other Spring programming model idioms.
{: .Quotes}

```xml
<parent>
  <groupId>wang.tiven.microservices</groupId>
  <artifactId>service-registration-and-discovery</artifactId>
  <version>0.0.1-SNAPSHOT</version>
</parent>

<name>service-registration-and-discovery:eureka-service</name>
<artifactId>eureka-service</artifactId>
<packaging>jar</packaging>
<properties>
  <start-class>wang.tiven.microservices.registry.Application</start-class>
</properties>

<dependencies>
  <dependency>
    <groupId>org.springframework.cloud</groupId>
    <artifactId>spring-cloud-starter-eureka-server</artifactId>
  </dependency>
</dependencies>
```

To run your own server use the `spring-cloud-starter-eureka-server` dependency and `@EnableEurekaServer`.

```java
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.netflix.eureka.server.EnableEurekaServer;

@SpringBootApplication
@EnableEurekaServer
public class Application {

  public static void main(String[] args) {
    SpringApplication.run(Application.class, args);
  }
}
```

Config the attribute `server port` default as `8761` in

```yaml
server:
  port: 8761

eureka:
  client:
    registerWithEureka: false
    fetchRegistry: false
  server:
    waitTimeInMsWhenSyncEmpty: 0
```

You can run the eureka server through command:
`mvn spring-boot:run`

Once the server startup, you can access the eureka application cockpit in local through url
[*http://127.0.0.1:8761/*](http://127.0.0.1:8761/){:target="\_blank"}

### Villain Service

Big villain here. The project most attributes are similar to eureka service's, so we only introduce the dependencies for this application:

```xml
<dependencies>
  <dependency>
    <groupId>org.springframework.cloud</groupId>
    <artifactId>spring-cloud-starter-eureka</artifactId>
  </dependency>
  <dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-web</artifactId>
  </dependency>
  <dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-mongodb</artifactId>
  </dependency>
</dependencies>
```

* `spring-cloud-starter-eureka` enable this spring boot application to be discovery-aware
* in order to create a RESTful api we have to include the `spring-boot-starter-web` package
* the application need to store the avatar for villain in database, so we include `spring-boot-starter-data-mongodb` package

The application configuration file:

```yaml
server:
  port: 8060

eureka:
  client:
    serviceUrl:
      defaultZone: http://localhost:8761/eureka/
```

* The web application port default as **8060**
* The server url for local eureka service is *http://localhost:8761/eureka/*

The application's Eureka instance name (the name by which it will be registered in Eureka) will be derived from the value of the `spring.application.name` property on the application.

```java
@RequestMapping(method = {RequestMethod.POST, RequestMethod.PUT})
ResponseEntity set(String villainId,
                      @RequestParam MultipartFile multipartFile,
                      UriComponentsBuilder uriBuilder) throws IOException {

    InputStream inputStream = multipartFile.getInputStream();
    this.gridFsTemplate.store(inputStream, villainId);

    URI uri = uriBuilder.path("/{villainId}").buildAndExpand(villainId).toUri();
    HttpHeaders headers = new HttpHeaders();
    headers.setLocation(uri);
    return new ResponseEntity(headers, HttpStatus.CREATED);
}
```

* The set method receive a villain id and avatar, then store it in mongodb
* The get method return a villain for who query him
* `@EnableEurekaClient` tell the application to registry in eureka service

First startup a mongodb server in local, for example using docker:
`docker run --rm --name mongoserver -d -p 27017:27017 mongo`

Startup the application and check the cockpit of eureka, you will find the registered application in section **Instances currently registered with Eureka**

`mvn spring-boot:run`

### Police Service

Add the [H2][h2database] database in this application to provide a datasource to store police offices and villains information.

> [H2](https://en.wikipedia.org/wiki/H2_(DBMS)) is a relational database management system written in Java. It can be embedded in Java applications or run in the client-server mode.
{: .Quotes}

```xml
<dependency>
  <groupId>org.springframework.boot</groupId>
  <artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
  <groupId>com.h2database</groupId>
  <artifactId>h2</artifactId>
</dependency>
```

The police service provide a Restful api to enable villain report himself to the police office:

```java
@RequestMapping(value = "/villains", method = RequestMethod.POST)
@ResponseBody Villain createVillain(@PathVariable String officeName, @RequestBody Villain villain) {

  System.out.println(officeName + ", I'm here, '" + villain.getName() + "'");

  PoliceOffice policeOffice = policeOfficeRepository.findByOfficeName(officeName);

  villain.setPoliceOffice(policeOffice);

  return this.villainRepository.save(villain);
}
```

#### Basic Service Discovery

Now we add logic in villain service to report a villain himself to police office.

Use the `RestTemplate` to call Restful api, and add the `@LoadBalanced` on it, this tells Spring Cloud that we want to take advantage of its load balancing support.

```java
@SpringBootApplication
@EnableEurekaClient
public class Application {

    ...

    @LoadBalanced
    @Bean
    RestTemplate restTemplate() {
        return new RestTemplate();
    }
}
```

Then we can auto wire the `RestTemplate` instance into controller, and call the police office restful api when creating villain. The url of police write as *http://\<service-name\>/\<api-path\>*, e.g. *http://police-service/Gotham-City/villains*

```java
...

@Autowired
private RestTemplate restTemplate;

@RequestMapping(method = {RequestMethod.POST, RequestMethod.PUT})
ResponseEntity<?> set(@PathVariable String villainId,
                      @RequestParam MultipartFile multipartFile,
                      UriComponentsBuilder uriBuilder) throws IOException {

  ...

  this.restTemplate.exchange(
          "http://police-service/Gotham-City/villains",
          HttpMethod.POST,
          new HttpEntity<Villain>(new Villain(villainId)),
          new ParameterizedTypeReference<Villain>() {
          });

  ...

  return new ResponseEntity<>(headers, HttpStatus.CREATED);
}
```

#### Test Basic Service Discovery

Now restart villain service and startup police service,

Send a post request to url *http://localhost:8060/xman*, then you will get the information *Gotham-City, I'm here, 'xman'* in the console of police server.

You can get the complete application sourcecode for this step on [Github](https://github.com/tiven-wang/try-cf/tree/service-discovery-1)

### Hero Service

In this hero service, we change the approach of calling restful api to using feign client. Add the spring cloud feign package:

```xml
<dependency>
    <groupId>org.springframework.cloud</groupId>
    <artifactId>spring-cloud-starter-feign</artifactId>
</dependency>
```

Enable the application's feign client ability by `@EnableFeignClients`

```java
@SpringBootApplication
@EnableEurekaClient
@EnableFeignClients
public class Application {
  ...
}
```

Create a client for external service restful api:

```java
@FeignClient("police-service")
interface PoliceOfficeClient {

    @RequestMapping(method = RequestMethod.POST, value = "/Gotham-City/villains/{villainName}")
    void catched(@PathVariable("villainName") String villainName);
}
```

Then use it in business logic in the controller class:

```java
@Autowired
PoliceOfficeClient policeOfficeClient;

@RequestMapping(method = RequestMethod.POST)
@ResponseBody void move(@PathVariable String heroName, @RequestBody Villain villain) {

  System.out.println("I " + heroName + " move!");

  Resource resource = villainClient.get(villain.getName());

  System.out.println(villain.getName() + ", catch you!");

  policeOfficeClient.catched(villain.getName());
}
```

### Test Feign Clients

Startup all of the services, Hero Villain and Police services, the send a request to url:

*http://localhost:8060/xman*

you will get messages

Villain service:

```
I xman am here!
```

Police service:

```
This is Gotham-City, you xman just wait!
Have catched xman
```

Hero service:

```
I Batman move!
xman, catch you!
```

You can get the complete application sourcecode for this step on [Github](https://github.com/tiven-wang/try-cf/tree/service-discovery-2)

## Dockerization
Probably you have learnt [docker][docker] in your past project experiences, now we apply docker in our these services:

> Before run docker runtime container, you need package the applications by `mvn package`
{: .Notes}

1 Startup the eureka server `docker run --rm --name eureka-service -h eureka-service -p 8761:8761 -v \<project-path\>/eureka-service:/data -w /data openjdk java -jar ./target/eureka-service.jar`

2 Startup a mongodb server `docker run --rm --name mongoserver -d -p 27017:27017 mongo`

Because applications run own OS container separately, so thy can't use `localhost` to access another service.
Change the service url for eureka server in all the applications and add mongodb server host address:

```yaml
eureka:
  client:
    serviceUrl:
      defaultZone: http://eureka-service:8761/eureka/

---
spring:
  data:
    mongodb:
      host: mongoserver
      port: 27017
```

3 `docker run --rm --name villain-service -h villain-service -p 8060:8060 --link eureka-service --link mongoserver -v \<project-path\>/villain-service:/data -w /data openjdk java -jar ./target/villain-service.jar`

Add a docker `--link` as `eureka-service` in the command to enable this container be able to access the *eureka service* container. And the `mongoserver` be too.

4 `docker run --rm --name police-service -h police-service -p 9098:9098 --link eureka-service --link hero-service -v \<project-path\>/police-service:/data -w /data openjdk java -jar ./target/police-service.jar`

5 `docker run --rm --name hero-service -h hero-service -p 8050:8050 --link eureka-service --link police-service --link villain-service -v \<project-path\>/hero-service:/data -w /data openjdk java -jar ./target/hero-service.jar`

> You can't startup the applications, because the container link dependencies.
{: .Warning}

So next

### Docker Network

> [What is Docker networking](https://www.oreilly.com/learning/what-is-docker-networking)
{: .Quotes}

[Docker container networking][docker-networking]

Create a network for our microservices:

`docker network create microservices-net`

And add the network into all the Docker containers:

`docker run --rm --name eureka-service -h eureka-service --network microservices-net -p 8761:8761 -v \<project-path\>/eureka-service:/data -w /data openjdk java -jar ./target/eureka-service.jar`

`docker run --rm --name mongoserver -d --network microservices-net -p 27017:27017 mongo`

`docker run --rm --name villain-service -h villain-service --network microservices-net -p 8060:8060 -v \<project-path\>/villain-service:/data -w /data openjdk java -jar ./target/villain-service.jar`

`docker run --rm --name police-service -h police-service --network microservices-net -p 9098:9098 -v \<project-path\>/police-service:/data -w /data openjdk java -jar ./target/police-service.jar`

`docker run --rm --name hero-service -h hero-service --network microservices-net -p 8050:8050 -v \<project-path\>/hero-service:/data -w /data openjdk java -jar ./target/hero-service.jar`

You can get the complete application sourcecode for this step on [Github](https://github.com/tiven-wang/try-cf/tree/service-discovery-dockerization)

## Cloud Foundry

In production, how to deploy our services to CloudFoundry platform? Please refer to [Try Cloud Foundry 10 - Service Discovery](/articles/try-cf-10-service-discovery/).

[eureka]:https://github.com/Netflix/eureka
[spring-cloud-netflix]:https://cloud.spring.io/spring-cloud-netflix/
[h2database]:http://www.h2database.com
[docker]:https://www.docker.com/
[docker-networking]:https://docs.docker.com/engine/userguide/networking/


text:
import sys
import math
import random
import subprocess

class Point:
    def __init__(self, coords, id = None):
        self.coords = coords
        self.n = len(coords)
        if not id is None:
            self.id = id

    def __repr__(self):
        return str(self.coords)

class Cluster:
    def __init__(self, points):
        if len(points) == 0: raise Exception("ILLEGAL: empty cluster")
        self.points = points
        self.n = points[0].n
        for p in points:
            if p.n != self.n: raise Exception("ILLEGAL: wrong dimensions")

        self.centroid = self.calculateCentroid()

    def __repr__(self):
        return str(self.points)

    def update(self, points):
        old_centroid = self.centroid
        self.points = points
        self.centroid = self.calculateCentroid()
        shift = getDistance(old_centroid, self.centroid)
        return shift

    def calculateCentroid(self):
        numPoints = len(self.points)
        coords = [p.coords for p in self.points]
        unzipped = zip(*coords)
        centroid_coords = [math.fsum(dList)/numPoints for dList in unzipped]
        return Point(centroid_coords)

def kmeans(points, k, cutoff, verbose = True):

    initial = random.sample(points, k)
    clusters = [Cluster([p]) for p in initial]
    print "clustering results"
    loopCounter = 0
    while True:
        lists = [ [] for c in clusters]
        clusterCount = len(clusters)

        loopCounter += 1
        if verbose is True:
            print "k means iter: ", loopCounter
        for p in points:
            smallest_distance = getDistance(p, clusters[0].centroid)
            clusterIndex = 0

            for i in range(clusterCount - 1):
                distance = getDistance(p, clusters[i+1].centroid)
                if distance < smallest_distance:
                    smallest_distance = distance
                    clusterIndex = i+1
            lists[clusterIndex].append(p)

        biggest_shift = 0.0

        for i in range(clusterCount):
            shift = clusters[i].update(lists[i])
            biggest_shift = max(biggest_shift, shift)

        if biggest_shift < cutoff:
            print "Converged after %s iterations" % loopCounter
            break
    return clusters

def getDistance(a, b):
    if a.n != b.n:
        raise Exception("ILLEGAL: non comparable points")

    ret = reduce(lambda x,y: x + pow((a.coords[y]-b.coords[y]), 2),range(a.n),0.0)
    return math.sqrt(ret)


text:
from __future__ import unicode_literals

import django.contrib.postgres.fields.jsonb
from django.db import migrations, models

class Migration(migrations.Migration):

    initial = True

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Visualization',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('title', models.CharField(max_length=512)),
                ('description', models.TextField(blank=True, default='')),
                ('hidden', models.BooleanField(default=False)),
                ('info', django.contrib.postgres.fields.jsonb.JSONField(default={})),
                ('view_name', models.CharField(max_length=100)),
            ],
        ),
    ]