text:
/*
 * Contributors:
 * - Liryna <liryna.stark@gmail.com>
 */

namespace WebSocketSharp
{
    using System;
    using System.Collections.Generic;
    using System.Collections.Specialized;
    using System.IO;
    using System.IO.Compression;
    using System.Linq;
    using System.Net;
    using System.Net.Sockets;
    using System.Text;
    using System.Threading.Tasks;

    using WebSocketSharp.Net;
    using WebSocketSharp.Net.WebSockets;
    using WebSocketSharp.Server;

    using CookieCollection = WebSocketSharp.Net.CookieCollection;
    using HttpStatusCode = WebSocketSharp.Net.HttpStatusCode;

    /// <summary>
    ///     Provides a set of static methods for websocket-sharp.
    /// </summary>
    internal static class Ext
    {
        private const string Tspecials = "()<>@,;:\\\"/[]?={} \t";

        public static async Task<Stream> Compress(this Stream stream)
        {
            stream.Seek(0, SeekOrigin.Begin);
            var compressed = new MemoryStream();
            var deflate = new DeflateStream(compressed, CompressionLevel.Optimal, true);
            await stream.CopyToAsync(deflate).ConfigureAwait(false);
            await deflate.FlushAsync().ConfigureAwait(false);
            deflate.Close();
            await compressed.FlushAsync().ConfigureAwait(false);

            compressed.Seek(0, SeekOrigin.Begin);

            return compressed;
        }

        /// <summary>
        ///     Determines whether the specified <see cref="string" /> contains any of characters
        ///     in the specified array of <see cref="char" />.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if <paramref name="value" /> contains any of <paramref name="chars" />;
        ///     otherwise, <c>false</c>.
        /// </returns>
        /// <param name="value">
        ///     A <see cref="string" /> to test.
        /// </param>
        /// <param name="chars">
        ///     An array of <see cref="char" /> that contains characters to find.
        /// </param>
        public static bool Contains(this string value, params char[] chars)
        {
            return chars.Length == 0 || (!string.IsNullOrEmpty(value) && value.IndexOfAny(chars) > -1);
        }

        /// <summary>
        ///     Determines whether the specified <see cref="NameValueCollection" /> contains the entry
        ///     with the specified both <paramref name="name" /> and <paramref name="value" />.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if <paramref name="collection" /> contains the entry with both
        ///     <paramref name="name" /> and <paramref name="value" />; otherwise, <c>false</c>.
        /// </returns>
        /// <param name="collection">
        ///     A <see cref="NameValueCollection" /> to test.
        /// </param>
        /// <param name="name">
        ///     A <see cref="string" /> that represents the key of the entry to find.
        /// </param>
        /// <param name="value">
        ///     A <see cref="string" /> that represents the value of the entry to find.
        /// </param>
        public static bool Contains(this NameValueCollection collection, string name, string value)
        {
            if (collection == null || collection.Count == 0)
            {
                return false;
            }

            var vals = collection[name];
            if (vals == null)
            {
                return false;
            }

            return vals.Split(',').Any(val => val.Trim().Equals(value, StringComparison.OrdinalIgnoreCase));
        }

        /// <summary>
        ///     Gets the collection of the HTTP cookies from the specified HTTP <paramref name="headers" />.
        /// </summary>
        /// <returns>
        ///     A <see cref="Net.CookieCollection" /> that receives a collection of the HTTP cookies.
        /// </returns>
        /// <param name="headers">
        ///     A <see cref="NameValueCollection" /> that contains a collection of the HTTP headers.
        /// </param>
        /// <param name="response">
        ///     <c>true</c> if <paramref name="headers" /> is a collection of the response headers;
        ///     otherwise, <c>false</c>.
        /// </param>
        public static CookieCollection GetCookies(this NameValueCollection headers, bool response)
        {
            var name = response ? "Set-Cookie" : "Cookie";
            return headers != null && headers.Contains(name)
                       ? CookieCollection.Parse(headers[name], response)
                       : new CookieCollection();
        }

        /// <summary>
        ///     Gets the description of the specified HTTP status <paramref name="code" />.
        /// </summary>
        /// <returns>
        ///     A <see cref="string" /> that represents the description of the HTTP status code.
        /// </returns>
        /// <param name="code">
        ///     One of <see cref="Net.HttpStatusCode" /> enum values, indicates the HTTP status code.
        /// </param>
        public static string GetDescription(this HttpStatusCode code)
        {
            return ((int)code).GetStatusDescription();
        }

        /// <summary>
        ///     Determines whether the specified <see cref="string" /> is enclosed in the specified
        ///     <see cref="char" />.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if <paramref name="value" /> is enclosed in <paramref name="c" />;
        ///     otherwise, <c>false</c>.
        /// </returns>
        /// <param name="value">
        ///     A <see cref="string" /> to test.
        /// </param>
        /// <param name="c">
        ///     A <see cref="char" /> that represents the character to find.
        /// </param>
        public static bool IsEnclosedIn(this string value, char c)
        {
            return value != null && value.Length > 1 && value[0] == c && value[value.Length - 1] == c;
        }

        /// <summary>
        ///     Determines whether the specified <see cref="System.Net.IPAddress" /> represents
        ///     the local IP address.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if <paramref name="address" /> represents the local IP address;
        ///     otherwise, <c>false</c>.
        /// </returns>
        /// <param name="address">
        ///     A <see cref="System.Net.IPAddress" /> to test.
        /// </param>
        /// <exception cref="ArgumentNullException">
        ///     <paramref name="address" /> is <see langword="null" />.
        /// </exception>
        public static bool IsLocal(this IPAddress address)
        {
            if (address == null)
            {
                throw new ArgumentNullException(nameof(address));
            }

            if (address.Equals(IPAddress.Any) || IPAddress.IsLoopback(address))
            {
                return true;
            }

            var host = Dns.GetHostName();
            var addrs = Dns.GetHostAddresses(host);
            return addrs.Contains(address);
        }

        /// <summary>
        ///     Determines whether the specified <see cref="string" /> is a URI string.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if <paramref name="value" /> may be a URI string; otherwise, <c>false</c>.
        /// </returns>
        /// <param name="value">
        ///     A <see cref="string" /> to test.
        /// </param>
        public static bool MaybeUri(this string value)
        {
            if (string.IsNullOrEmpty(value))
            {
                return false;
            }

            var i = value.IndexOf(':');
            if (i == -1)
            {
                return false;
            }

            if (i >= 10)
            {
                return false;
            }

            return value.Substring(0, i).IsPredefinedScheme();
        }

        /// <summary>
        ///     Retrieves a sub-array from the specified <paramref name="array" />.
        ///     A sub-array starts at the specified element position in <paramref name="array" />.
        /// </summary>
        /// <returns>
        ///     An array of T that receives a sub-array, or an empty array of T
        ///     if any problems with the parameters.
        /// </returns>
        /// <param name="array">
        ///     An array of T from which to retrieve a sub-array.
        /// </param>
        /// <param name="startIndex">
        ///     An <see cref="int" /> that represents the zero-based starting position of
        ///     a sub-array in <paramref name="array" />.
        /// </param>
        /// <param name="length">
        ///     An <see cref="int" /> that represents the number of elements to retrieve.
        /// </param>
        /// <typeparam name="T">
        ///     The type of elements in <paramref name="array" />.
        /// </typeparam>
        public static T[] SubArray<T>(this T[] array, int startIndex, int length)
        {
            int len;
            if (array == null || (len = array.Length) == 0)
            {
                return new T[0];
            }

            if (startIndex < 0 || length <= 0 || startIndex + length > len)
            {
                return new T[0];
            }

            if (startIndex == 0 && length == len)
            {
                return array;
            }

            var subArray = new T[length];
            Array.Copy(array, startIndex, subArray, 0, length);

            return subArray;
        }

        /// <summary>
        ///     Converts the specified <see cref="string" /> to a <see cref="Uri" />.
        /// </summary>
        /// <returns>
        ///     A <see cref="Uri" /> converted from <paramref name="uriString" />, or <see langword="null" />
        ///     if <paramref name="uriString" /> isn't successfully converted.
        /// </returns>
        /// <param name="uriString">
        ///     A <see cref="string" /> to convert.
        /// </param>
        public static Uri ToUri(this string uriString)
        {
            Uri res;
            return Uri.TryCreate(uriString, uriString.MaybeUri() ? UriKind.Absolute : UriKind.Relative, out res)
                       ? res
                       : null;
        }

        /// <summary>
        ///     URL-decodes the specified <see cref="string" />.
        /// </summary>
        /// <returns>
        ///     A <see cref="string" /> that receives the decoded string, or the <paramref name="value" />
        ///     if it's <see langword="null" /> or empty.
        /// </returns>
        /// <param name="value">
        ///     A <see cref="string" /> to decode.
        /// </param>
        public static string UrlDecode(this string value)
        {
            return !string.IsNullOrEmpty(value) ? HttpUtility.UrlDecode(value) : value;
        }

        /// <summary>
        ///     URL-encodes the specified <see cref="string" />.
        /// </summary>
        /// <returns>
        ///     A <see cref="string" /> that receives the encoded string, or <paramref name="value" />
        ///     if it's <see langword="null" /> or empty.
        /// </returns>
        /// <param name="value">
        ///     A <see cref="string" /> to encode.
        /// </param>
        public static string UrlEncode(this string value)
        {
            return !string.IsNullOrEmpty(value) ? HttpUtility.UrlEncode(value) : value;
        }

        internal static byte[] Append(this ushort code, string reason)
        {
            return code.InternalToByteArray(ByteOrder.Big).Take(2).Concat(Encoding.UTF8.GetBytes(reason)).ToArray();
        }

        internal static string CheckIfClosable(this WebSocketState state)
        {
            return state == WebSocketState.Closing
                       ? "While closing the WebSocket connection."
                       : state == WebSocketState.Closed ? "The WebSocket connection has already been closed." : null;
        }

        internal static string CheckIfConnectable(this WebSocketState state)
        {
            return state == WebSocketState.Open || state == WebSocketState.Closing
                       ? "A WebSocket connection has already been established."
                       : null;
        }

        internal static string CheckIfOpen(this WebSocketState state)
        {
            return state == WebSocketState.Connecting
                       ? "A WebSocket connection isn't established."
                       : state == WebSocketState.Closing
                             ? "While closing the WebSocket connection."
                             : state == WebSocketState.Closed
                                   ? "The WebSocket connection has already been closed."
                                   : null;
        }

        internal static string CheckIfStart(this ServerState state)
        {
            return state == ServerState.Ready
                       ? "The server hasn't yet started."
                       : state == ServerState.ShuttingDown
                             ? "The server is shutting down."
                             : state == ServerState.Stop ? "The server has already stopped." : null;
        }

        internal static string CheckIfStartable(this ServerState state)
        {
            return state == ServerState.Start
                       ? "The server has already started."
                       : state == ServerState.ShuttingDown ? "The server is shutting down." : null;
        }

        internal static string CheckIfValidControlData(this byte[] data, string paramName)
        {
            return data.Length > 125 ? $"'{paramName}' is greater than the allowable max size." : null;
        }

        internal static string CheckIfValidProtocols(this string[] protocols)
        {
            return protocols.Contains(protocol => string.IsNullOrEmpty(protocol) || !protocol.IsToken())
                       ? "Contains an invalid value."
                       : protocols.ContainsTwice() ? "Contains a value twice." : null;
        }

        internal static string CheckIfValidSendData(this Stream data)
        {
            return IsDataNull(data);
        }

        internal static string CheckIfValidSendData(this byte[] data)
        {
            return IsDataNull(data);
        }

        internal static string CheckIfValidSendData(this string data)
        {
            return IsDataNull(data);
        }

        internal static string CheckIfValidServicePath(this string path)
        {
            return string.IsNullOrEmpty(path)
                       ? "'path' is null or empty."
                       : path[0] != '/'
                             ? "'path' isn't an absolute path."
                             : path.IndexOfAny(new[] { '?', '#' }) > -1
                                   ? "'path' includes either or both query and fragment components."
                                   : null;
        }

        internal static string CheckIfValidSessionId(this string id)
        {
            return string.IsNullOrEmpty(id) ? "'id' is null or empty." : null;
        }

        internal static string CheckIfValidWaitTime(this TimeSpan time)
        {
            return time <= TimeSpan.Zero ? "A wait time is zero or less." : null;
        }

        internal static Task<Stream> Compress(this Stream stream, long length)
        {
            var subStream = new SubStream(stream, length);
            return subStream.Compress();
        }

        internal static bool Contains<T>(this IEnumerable<T> source, Func<T, bool> condition)
        {
            return source.Any(condition);
        }

        /// <summary>
        ///     Determines whether the specified <see cref="int" /> equals the specified <see cref="char" />,
        ///     and invokes the specified Action<int> delegate at the same time.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if <paramref name="value" /> equals <paramref name="c" />;
        ///     otherwise, <c>false</c>.
        /// </returns>
        /// <param name="value">
        ///     An <see cref="int" /> to compare.
        /// </param>
        /// <param name="c">
        ///     A <see cref="char" /> to compare.
        /// </param>
        /// <param name="action">
        ///     An Action<int> delegate that references the method(s) called at
        ///     the same time as comparing. An <see cref="int" /> parameter to pass to
        ///     the method(s) is <paramref name="value" />.
        /// </param>
        /// <exception cref="ArgumentOutOfRangeException">
        ///     <paramref name="value" /> isn't between 0 and 255.
        /// </exception>
        internal static bool EqualsWith(this int value, char c, Action<int> action)
        {
            if (value < 0 || value > 255)
            {
                throw new ArgumentOutOfRangeException(nameof(value));
            }

            action(value);
            return value == c - 0;
        }

        /// <summary>
        ///     Gets the absolute path from the specified <see cref="Uri" />.
        /// </summary>
        /// <returns>
        ///     A <see cref="string" /> that represents the absolute path if it's successfully found;
        ///     otherwise, <see langword="null" />.
        /// </returns>
        /// <param name="uri">
        ///     A <see cref="Uri" /> that represents the URI to get the absolute path from.
        /// </param>
        internal static string GetAbsolutePath(this Uri uri)
        {
            if (uri.IsAbsoluteUri)
            {
                return uri.AbsolutePath;
            }

            var original = uri.OriginalString;
            if (original[0] != '/')
            {
                return null;
            }

            var i = original.IndexOfAny(new[] { '?', '#' });
            return i > 0 ? original.Substring(0, i) : original;
        }

        internal static string GetMessage(this CloseStatusCode code)
        {
            return code == CloseStatusCode.ProtocolError
                       ? "A WebSocket protocol error has occurred."
                       : code == CloseStatusCode.IncorrectData
                             ? "An incorrect data has been received."
                             : code == CloseStatusCode.Abnormal
                                   ? "An exception has occurred."
                                   : code == CloseStatusCode.InconsistentData
                                         ? "An inconsistent data has been received."
                                         : code == CloseStatusCode.PolicyViolation
                                               ? "A policy violation has occurred."
                                               : code == CloseStatusCode.TooBig
                                                     ? "A too big data has been received."
                                                     : code == CloseStatusCode.IgnoreExtension
                                                           ? "WebSocket client didn't receive expected extension(s)."
                                                           : code == CloseStatusCode.ServerError
                                                                 ? "WebSocket server got an internal error."
                                                                 : code == CloseStatusCode.TlsHandshakeFailure
                                                                       ? "An error has occurred while handshaking."
                                                                       : string.Empty;
        }

        /// <summary>
        ///     Gets the value from the specified <see cref="string" /> that contains a pair of name and
        ///     value separated by a separator character.
        /// </summary>
        /// <returns>
        ///     A <see cref="string" /> that represents the value if any; otherwise, <c>null</c>.
        /// </returns>
        /// <param name="nameAndValue">
        ///     A <see cref="string" /> that contains a pair of name and value separated by a separator
        ///     character.
        /// </param>
        /// <param name="separator">
        ///     A <see cref="char" /> that represents the separator character.
        /// </param>
        internal static string GetValue(this string nameAndValue, char separator)
        {
            var i = nameAndValue.IndexOf(separator);
            return i > -1 && i < nameAndValue.Length - 1 ? nameAndValue.Substring(i + 1).Trim() : null;
        }

        internal static string GetValue(this string nameAndValue, char separator, bool unquote)
        {
            var i = nameAndValue.IndexOf(separator);
            if (i < 0 || i == nameAndValue.Length - 1)
            {
                return null;
            }

            var val = nameAndValue.Substring(i + 1).Trim();
            return unquote ? val.Unquote() : val;
        }

        internal static TcpListenerWebSocketContext GetWebSocketContext(
            this TcpClient tcpClient,
            string protocol,
            bool secure,
            ServerSslConfiguration sslConfiguration)
        {
            return new TcpListenerWebSocketContext(tcpClient, protocol, secure, sslConfiguration);
        }

        internal static bool IncludesReservedCloseStatusCode(this byte[] byteArray)
        {
            return byteArray.Length > 1 && byteArray.SubArray(0, 2).ToUInt16(ByteOrder.Big).IsReserved();
        }

        internal static byte[] InternalToByteArray(this ushort value, ByteOrder order)
        {
            var bytes = BitConverter.GetBytes(value);
            if (!order.IsHostOrder())
            {
                Array.Reverse(bytes);
            }

            return bytes;
        }

        internal static byte[] InternalToByteArray(this ulong value, ByteOrder order)
        {
            var bytes = BitConverter.GetBytes(value);
            if (!order.IsHostOrder())
            {
                Array.Reverse(bytes);
            }

            return bytes;
        }

        internal static bool IsCompressionExtension(this string value)
        {
            return value.StartsWith("permessage-");
        }

        internal static bool IsPortNumber(this int value)
        {
            return value > 0 && value < 65536;
        }

        internal static bool IsReserved(this ushort code)
        {
            return code == (ushort)CloseStatusCode.Undefined || code == (ushort)CloseStatusCode.NoStatusCode
                   || code == (ushort)CloseStatusCode.Abnormal || code == (ushort)CloseStatusCode.TlsHandshakeFailure;
        }

        internal static bool IsReserved(this CloseStatusCode code)
        {
            return code == CloseStatusCode.Undefined || code == CloseStatusCode.NoStatusCode
                   || code == CloseStatusCode.Abnormal || code == CloseStatusCode.TlsHandshakeFailure;
        }

        internal static bool IsText(this string value)
        {
            var len = value.Length;
            for (var i = 0; i < len; i++)
            {
                var c = value[i];
                if (c < 0x20 && !"\r\n\t".Contains(c))
                {
                    return false;
                }

                if (c == 0x7f)
                {
                    return false;
                }

                if (c == '\n' && ++i < len)
                {
                    c = value[i];
                    if (!" \t".Contains(c))
                    {
                        return false;
                    }
                }
            }

            return true;
        }

        internal static bool IsToken(this string value)
        {
            return value.All(c => c >= 0x20 && c < 0x7f && !Tspecials.Contains(c));
        }

        internal static string Quote(this string value)
        {
            return $"\"{value.Replace("\"", "\\\"")}\"";
        }

        internal static async Task<byte[]> ReadBytes(this Stream stream, long length, int bufferLength)
        {
            using (var res = new MemoryStream())
            {
                var cnt = length / bufferLength;
                var rem = (int)(length % bufferLength);

                var buff = new byte[bufferLength];
                var end = false;
                for (long i = 0; i < cnt; i++)
                {
                    if (!await stream.ReadBytes(buff, 0, bufferLength, res).ConfigureAwait(false))
                    {
                        end = true;
                        break;
                    }
                }

                if (!end && rem > 0)
                {
                    await stream.ReadBytes(new byte[rem], 0, rem, res).ConfigureAwait(false);
                }

                res.Close();
                return res.ToArray();
            }
        }

        internal static async Task<byte[]> ReadBytes(this Stream stream, int length)
        {
            var buff = new byte[length];

            var len = await stream.ReadAsync(buff, 0, length).ConfigureAwait(false);

            var bytes = len < 1
                            ? new byte[0]
                            : len < length
                                  ? await stream.ReadBytes(buff, len, length - len).ConfigureAwait(false)
                                  : buff;

            return bytes;
        }

        internal static string RemovePrefix(this string value, params string[] prefixes)
        {
            var i = (from prefix in prefixes where value.StartsWith(prefix) select prefix.Length).FirstOrDefault();

            return i > 0 ? value.Substring(i) : value;
        }

        internal static IEnumerable<string> SplitHeaderValue(this string value, params char[] separators)
        {
            var len = value.Length;
            var seps = new string(separators);

            var buff = new StringBuilder(32);
            var escaped = false;
            var quoted = false;

            for (var i = 0; i < len; i++)
            {
                var c = value[i];
                if (c == '"')
                {
                    if (escaped)
                    {
                        escaped = false;
                    }
                    else
                    {
                        quoted = !quoted;
                    }
                }
                else if (c == '\\')
                {
                    if (i < len - 1 && value[i + 1] == '"')
                    {
                        escaped = true;
                    }
                }
                else if (seps.Contains(c))
                {
                    if (!quoted)
                    {
                        yield return buff.ToString();
                        buff.Length = 0;

                        continue;
                    }
                }

                buff.Append(c);
            }

            if (buff.Length > 0)
            {
                yield return buff.ToString();
            }
        }

        internal static async Task<byte[]> ToByteArray(this Stream stream)
        {
            if (stream == null)
            {
                return null;
            }

            using (var output = new MemoryStream())
            {
                //stream.Seek(0, SeekOrigin.Begin);
                await stream.CopyToAsync(output).ConfigureAwait(false);
                await output.FlushAsync().ConfigureAwait(false);

                return output.ToArray();
            }
        }

        internal static CompressionMethod ToCompressionMethod(this string value)
        {
            return
                Enum.GetValues(typeof(CompressionMethod))
                    .Cast<CompressionMethod>()
                    .FirstOrDefault(method => method.ToExtensionString() == value);
        }

        internal static string ToExtensionString(this CompressionMethod method)
        {
            return method != CompressionMethod.None ? $"permessage-{method.ToString().ToLower()}" : string.Empty;
        }

        internal static ushort ToUInt16(this byte[] source, ByteOrder sourceOrder)
        {
            return BitConverter.ToUInt16(source.ToHostOrder(sourceOrder), 0);
        }

        internal static ulong ToUInt64(this byte[] source, ByteOrder sourceOrder)
        {
            return BitConverter.ToUInt64(source.ToHostOrder(sourceOrder), 0);
        }

        internal static string TrimEndSlash(this string value)
        {
            value = value.TrimEnd('/');
            return value.Length > 0 ? value : "/";
        }

        /// <summary>
        ///     Tries to create a <see cref="Uri" /> for WebSocket with the specified
        ///     <paramref name="uriString" />.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if a <see cref="Uri" /> is successfully created; otherwise, <c>false</c>.
        /// </returns>
        /// <param name="uriString">
        ///     A <see cref="string" /> that represents the WebSocket URL to try.
        /// </param>
        /// <param name="result">
        ///     When this method returns, a <see cref="Uri" /> that represents the WebSocket URL
        ///     if <paramref name="uriString" /> is valid; otherwise, <see langword="null" />.
        /// </param>
        /// <param name="message">
        ///     When this method returns, a <see cref="string" /> that represents the error message
        ///     if <paramref name="uriString" /> is invalid; otherwise, <see cref="String.Empty" />.
        /// </param>
        internal static bool TryCreateWebSocketUri(this string uriString, out Uri result, out string message)
        {
            result = null;
            if (uriString.Length == 0)
            {
                message = "An empty string.";
                return false;
            }

            var uri = uriString.ToUri();
            if (!uri.IsAbsoluteUri)
            {
                message = "Not an absolute URI: " + uriString;
                return false;
            }

            var schm = uri.Scheme;
            if (schm != "ws" && schm != "wss")
            {
                message = "The scheme part isn't 'ws' or 'wss': " + uriString;
                return false;
            }

            if (uri.Fragment.Length > 0)
            {
                message = "Includes the fragment component: " + uriString;
                return false;
            }

            var port = uri.Port;
            if (port > 0)
            {
                if (port > 65535)
                {
                    message = "The port part is greater than 65535: " + uriString;
                    return false;
                }

                if ((schm == "ws" && port == 443) || (schm == "wss" && port == 80))
                {
                    message = "An invalid pair of scheme and port: " + uriString;
                    return false;
                }
            }
            else
            {
                uri = new Uri($"{schm}://{uri.Host}:{(schm == "ws" ? 80 : 443)}{uri.PathAndQuery}");
            }

            result = uri;
            message = string.Empty;

            return true;
        }

        internal static Task WriteBytes(this Stream stream, byte[] bytes)
        {
            using (var input = new MemoryStream(bytes))
            {
                return input.CopyToAsync(stream);
            }
        }

        /// <summary>
        ///     Determines whether the specified <see cref="NameValueCollection" /> contains the entry
        ///     with the specified <paramref name="name" />.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if <paramref name="collection" /> contains the entry
        ///     with <paramref name="name" />; otherwise, <c>false</c>.
        /// </returns>
        /// <param name="collection">
        ///     A <see cref="NameValueCollection" /> to test.
        /// </param>
        /// <param name="name">
        ///     A <see cref="string" /> that represents the key of the entry to find.
        /// </param>
        private static bool Contains(this NameValueCollection collection, string name)
        {
            return collection != null && collection.Count > 0 && collection[name] != null;
        }

        private static bool ContainsTwice(this string[] values)
        {
            var len = values.Length;

            Func<int, bool> contains = null;
            contains = idx =>
                {
                    if (idx < len - 1)
                    {
                        for (var i = idx + 1; i < len; i++)
                        {
                            if (values[i] == values[idx])
                            {
                                return true;
                            }
                        }

                        return contains(++idx);
                    }

                    return false;
                };

            return contains(0);
        }

        /// <summary>
        ///     Gets the description of the specified HTTP status <paramref name="code" />.
        /// </summary>
        /// <returns>
        ///     A <see cref="string" /> that represents the description of the HTTP status code.
        /// </returns>
        /// <param name="code">
        ///     An <see cref="int" /> that represents the HTTP status code.
        /// </param>
        private static string GetStatusDescription(this int code)
        {
            switch (code)
            {
                case 100:
                    return "Continue";
                case 101:
                    return "Switching Protocols";
                case 102:
                    return "Processing";
                case 200:
                    return "OK";
                case 201:
                    return "Created";
                case 202:
                    return "Accepted";
                case 203:
                    return "Non-Authoritative Information";
                case 204:
                    return "No Content";
                case 205:
                    return "Reset Content";
                case 206:
                    return "Partial Content";
                case 207:
                    return "Multi-Status";
                case 300:
                    return "Multiple Choices";
                case 301:
                    return "Moved Permanently";
                case 302:
                    return "Found";
                case 303:
                    return "See Other";
                case 304:
                    return "Not Modified";
                case 305:
                    return "Use Proxy";
                case 307:
                    return "Temporary Redirect";
                case 400:
                    return "Bad Request";
                case 401:
                    return "Unauthorized";
                case 402:
                    return "Payment Required";
                case 403:
                    return "Forbidden";
                case 404:
                    return "Not Found";
                case 405:
                    return "Method Not Allowed";
                case 406:
                    return "Not Acceptable";
                case 407:
                    return "Proxy Authentication Required";
                case 408:
                    return "Request Timeout";
                case 409:
                    return "Conflict";
                case 410:
                    return "Gone";
                case 411:
                    return "Length Required";
                case 412:
                    return "Precondition Failed";
                case 413:
                    return "Request Entity Too Large";
                case 414:
                    return "Request-Uri Too Long";
                case 415:
                    return "Unsupported Media Type";
                case 416:
                    return "Requested Range Not Satisfiable";
                case 417:
                    return "Expectation Failed";
                case 422:
                    return "Unprocessable Entity";
                case 423:
                    return "Locked";
                case 424:
                    return "Failed Dependency";
                case 500:
                    return "Internal Server Error";
                case 501:
                    return "Not Implemented";
                case 502:
                    return "Bad Gateway";
                case 503:
                    return "Service Unavailable";
                case 504:
                    return "Gateway Timeout";
                case 505:
                    return "Http Version Not Supported";
                case 507:
                    return "Insufficient Storage";
            }

            return string.Empty;
        }

        private static string IsDataNull(object data)
        {
            return data == null ? "'data' is null." : null;
        }

        /// <summary>
        ///     Determines whether the specified <see cref="ByteOrder" /> is host (this computer
        ///     architecture) byte order.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if <paramref name="order" /> is host byte order; otherwise, <c>false</c>.
        /// </returns>
        /// <param name="order">
        ///     One of the <see cref="ByteOrder" /> enum values, to test.
        /// </param>
        private static bool IsHostOrder(this ByteOrder order)
        {
            // true : !(true ^ true)  or !(false ^ false)
            // false: !(true ^ false) or !(false ^ true)
            return !(BitConverter.IsLittleEndian ^ (order == ByteOrder.Little));
        }

        /// <summary>
        ///     Determines whether the specified <see cref="string" /> is a predefined scheme.
        /// </summary>
        /// <returns>
        ///     <c>true</c> if <paramref name="value" /> is a predefined scheme; otherwise, <c>false</c>.
        /// </returns>
        /// <param name="value">
        ///     A <see cref="string" /> to test.
        /// </param>
        private static bool IsPredefinedScheme(this string value)
        {
            if (value == null || value.Length < 2)
            {
                return false;
            }

            var c = value[0];
            if (c == 'h')
            {
                return value == "http" || value == "https";
            }

            if (c == 'w')
            {
                return value == "ws" || value == "wss";
            }

            if (c == 'f')
            {
                return value == "file" || value == "ftp";
            }

            if (c == 'n')
            {
                c = value[1];
                return c == 'e' ? value == "news" || value == "net.pipe" || value == "net.tcp" : value == "nntp";
            }

            return (c == 'g' && value == "gopher") || (c == 'm' && value == "mailto");
        }

        private static async Task<byte[]> ReadBytes(this Stream stream, byte[] buffer, int offset, int length)
        {
            var len = 0;
            try
            {
                len = await stream.ReadAsync(buffer, offset, length).ConfigureAwait(false);
                if (len < 1)
                {
                    return buffer.SubArray(0, offset);
                }

                while (len < length)
                {
                    var readLen = await stream.ReadAsync(buffer, offset + len, length - len).ConfigureAwait(false);
                    if (readLen < 1)
                    {
                        break;
                    }

                    len += readLen;
                }
            }
            catch
            {
            }

            return len < length ? buffer.SubArray(0, offset + len) : buffer;
        }

        private static async Task<bool> ReadBytes(
            this Stream stream,
            byte[] buffer,
            int offset,
            int length,
            Stream destination)
        {
            var bytes = await stream.ReadBytes(buffer, offset, length).ConfigureAwait(false);
            var len = bytes.Length;
            await destination.WriteAsync(bytes, 0, len).ConfigureAwait(false);

            return len == offset + length;
        }

        /// <summary>
        ///     Converts the order of the specified array of <see cref="byte" /> to the host byte order.
        /// </summary>
        /// <returns>
        ///     An array of <see cref="byte" /> converted from <paramref name="source" />.
        /// </returns>
        /// <param name="source">
        ///     An array of <see cref="byte" /> to convert.
        /// </param>
        /// <param name="sourceOrder">
        ///     One of the <see cref="ByteOrder" /> enum values, indicates the byte order of
        ///     <paramref name="source" />.
        /// </param>
        /// <exception cref="ArgumentNullException">
        ///     <paramref name="source" /> is <see langword="null" />.
        /// </exception>
        private static byte[] ToHostOrder(this byte[] source, ByteOrder sourceOrder)
        {
            if (source == null)
            {
                throw new ArgumentNullException(nameof(source));
            }

            return source.Length > 1 && !sourceOrder.IsHostOrder() ? source.Reverse().ToArray() : source;
        }

        private static string Unquote(this string value)
        {
            var start = value.IndexOf('"');
            if (start < 0)
            {
                return value;
            }

            var end = value.LastIndexOf('"');
            var len = end - start - 1;

            return len < 0 ? value : len == 0 ? string.Empty : value.Substring(start + 1, len).Replace("\\\"", "\"");
        }
    }
}


text:
/* eslint no-unused-vars: 0 */
import lib from './lib'
import cli from './cli'

redpajama_set_name: RedPajamaGithub


text:
package com.elastisys.scale.cloudpool.api.types;

import static com.elastisys.scale.cloudpool.api.types.TestUtils.ips;
import static com.elastisys.scale.cloudpool.api.types.TestUtils.machineNoIp;
import static com.elastisys.scale.cloudpool.api.types.TestUtils.pool;
import static com.elastisys.scale.commons.json.JsonUtils.parseJsonString;
import static java.util.Arrays.asList;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.junit.Assert.assertThat;

import java.io.IOException;
import java.util.Arrays;

import org.joda.time.DateTime;
import org.junit.Test;

import com.elastisys.scale.commons.json.JsonUtils;
import com.elastisys.scale.commons.util.time.UtcTime;

/**
 * Exercises the {@link MachinePool} class.
 *
 *
 *
 */
public class TestMachinePool {

    /** Resource path containing Json representation of a machine pool. */
    private final static String emptyPoolJson = "json/empty-pool.json";
    /** Resource path containing Json representation of a machine pool. */
    private final static String singleMachinePoolJson = "json/pool1.json";
    /** Resource path containing Json representation of a machine pool. */
    private final static String multiMachinePoolJson = "json/pool2.json";

    @Test
    public void testGetTimestamp() {
        DateTime now = UtcTime.now();
        MachinePool pool = MachinePool.emptyPool(now);
        assertThat(pool.getTimestamp(), is(now));
    }

    /**
     * Exercise {@link MachinePool#getMachines()}
     */
    @Test
    public void testGetMachines() {
        DateTime now = UtcTime.now();

        // on empty pool
        MachinePool pool = MachinePool.emptyPool(now);
        assertThat(pool.getMachines().size(), is(0));

        // on a pool with a mix of machines in different states
        Machine requested1 = Machine.builder().id("i-1").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine requested2 = Machine.builder().id("i-2").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine pending1 = Machine.builder().id("i-3").machineState(MachineState.PENDING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.BOOTING).requestTime(now)
                .launchTime(now).build();
        Machine running1 = Machine.builder().id("i-4").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.IN_SERVICE).requestTime(now)
                .launchTime(now).publicIps(asList("1.2.3.4")).build();
        Machine running2 = Machine.builder().id("i-5").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.UNHEALTHY).requestTime(now)
                .launchTime(now).publicIps(asList("1.2.3.4")).build();
        Machine running3 = Machine.builder().id("i-5.1").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.OUT_OF_SERVICE).requestTime(now)
                .launchTime(now).publicIps(asList("1.2.3.5")).build();
        Machine terminating = Machine.builder().id("i-6").machineState(MachineState.TERMINATING)
                .cloudProvider("AWS-EC2").region("us-east-1").machineSize("m1.small").build();
        Machine terminated1 = Machine.builder().id("i-7").machineState(MachineState.TERMINATED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine terminated2 = Machine.builder().id("i-8").machineState(MachineState.TERMINATED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        pool = new MachinePool(Arrays.asList(requested1, requested2, pending1, running1, running2, running3,
                terminating, terminated1, terminated2), now);
        // verify machines
        assertThat(pool.getMachines().size(), is(9));
        assertThat(pool.getMachines(), is(asList(requested1, requested2, pending1, running1, running2, running3,
                terminating, terminated1, terminated2)));
    }

    /**
     * Exercise {@link MachinePool#getAllocatedMachines()}
     */
    @Test
    public void testGetAllocatedMachines() {
        DateTime now = UtcTime.now();

        // on empty pool
        MachinePool pool = MachinePool.emptyPool(now);
        assertThat(pool.getAllocatedMachines().size(), is(0));

        // on a pool with a mix of machines in different states
        Machine requested1 = Machine.builder().id("i-1").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine requested2 = Machine.builder().id("i-2").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine pending1 = Machine.builder().id("i-3").machineState(MachineState.PENDING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.BOOTING).requestTime(now)
                .launchTime(now).build();
        Machine running1 = Machine.builder().id("i-4").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.IN_SERVICE).requestTime(now)
                .launchTime(now).publicIps(asList("1.2.3.4")).build();
        Machine running2 = Machine.builder().id("i-5").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.UNHEALTHY).requestTime(now)
                .launchTime(now).publicIps(asList("1.2.3.4")).build();
        Machine running3 = Machine.builder().id("i-5.1").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.OUT_OF_SERVICE).requestTime(now)
                .launchTime(now).publicIps(asList("1.2.3.5")).build();
        Machine terminating = Machine.builder().id("i-6").machineState(MachineState.TERMINATING)
                .cloudProvider("AWS-EC2").region("us-east-1").machineSize("m1.small").build();
        Machine terminated1 = Machine.builder().id("i-7").machineState(MachineState.TERMINATED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine terminated2 = Machine.builder().id("i-8").machineState(MachineState.TERMINATED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        pool = new MachinePool(Arrays.asList(requested1, requested2, pending1, running1, running2, running3,
                terminating, terminated1, terminated2), now);
        // verify allocated machines: 2 requested + 1 pending + 3 running
        assertThat(pool.getAllocatedMachines().size(), is(6));
        assertThat(pool.getAllocatedMachines(),
                is(asList(requested1, requested2, pending1, running1, running2, running3)));
    }

    /**
     * Exercise {@link MachinePool#getActiveMachines()}
     */
    @Test
    public void testGetActiveMachines() {
        DateTime now = UtcTime.now();

        // on empty pool
        MachinePool pool = MachinePool.emptyPool(now);
        assertThat(pool.getActiveMachines().size(), is(0));

        // on a pool with a mix of machines in different states
        Machine requested1 = Machine.builder().id("i-1").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine requested2 = Machine.builder().id("i-2").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine pending1 = Machine.builder().id("i-3").machineState(MachineState.PENDING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.BOOTING).requestTime(now)
                .launchTime(now).build();
        Machine running1 = Machine.builder().id("i-4").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.IN_SERVICE)
                .membershipStatus(MembershipStatus.blessed()).requestTime(now).launchTime(now)
                .publicIps(asList("1.2.3.4")).build();
        Machine running2 = Machine.builder().id("i-5").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.UNHEALTHY).requestTime(now)
                .launchTime(now).publicIps(asList("1.2.3.4")).build();
        Machine running3 = Machine.builder().id("i-5.1").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.OUT_OF_SERVICE)
                .membershipStatus(MembershipStatus.awaitingService()).requestTime(now).launchTime(now)
                .publicIps(asList("1.2.3.5")).build();
        Machine terminating = Machine.builder().id("i-6").machineState(MachineState.TERMINATING)
                .cloudProvider("AWS-EC2").region("us-east-1").machineSize("m1.small").build();
        Machine terminated1 = Machine.builder().id("i-7").machineState(MachineState.TERMINATED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine terminated2 = Machine.builder().id("i-8").machineState(MachineState.TERMINATED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        pool = new MachinePool(Arrays.asList(requested1, requested2, pending1, running1, running2, running3,
                terminating, terminated1, terminated2), now);
        // verify effective machines: 2 requested + 1 pending + 2 running that
        // don't have an inactive membership status
        assertThat(pool.getActiveMachines().size(), is(5));
        assertThat(pool.getActiveMachines(), is(asList(requested1, requested2, pending1, running1, running2)));
    }

    /**
     * Exercise {@link MachinePool#getStartedMachines()}
     */
    @Test
    public void testGetStartedMachines() {
        DateTime now = UtcTime.now();

        // on empty pool
        MachinePool pool = MachinePool.emptyPool(now);
        assertThat(pool.getActiveMachines().size(), is(0));

        // on a pool with a mix of machines in different states
        Machine requested1 = Machine.builder().id("i-1").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine requested2 = Machine.builder().id("i-2").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine pending1 = Machine.builder().id("i-3").machineState(MachineState.PENDING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.BOOTING).requestTime(now)
                .launchTime(now).build();
        Machine running1 = Machine.builder().id("i-4").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.IN_SERVICE)
                .membershipStatus(MembershipStatus.blessed()).requestTime(now).launchTime(now)
                .publicIps(asList("1.2.3.4")).build();
        Machine running2 = Machine.builder().id("i-5").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.UNHEALTHY).requestTime(now)
                .launchTime(now).publicIps(asList("1.2.3.4")).build();
        Machine running3 = Machine.builder().id("i-5.1").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.OUT_OF_SERVICE)
                .membershipStatus(MembershipStatus.awaitingService()).requestTime(now).launchTime(now)
                .publicIps(asList("1.2.3.5")).build();
        Machine terminating = Machine.builder().id("i-6").machineState(MachineState.TERMINATING)
                .cloudProvider("AWS-EC2").region("us-east-1").machineSize("m1.small").build();
        Machine terminated1 = Machine.builder().id("i-7").machineState(MachineState.TERMINATED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        Machine terminated2 = Machine.builder().id("i-8").machineState(MachineState.TERMINATED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        pool = new MachinePool(Arrays.asList(requested1, requested2, pending1, running1, running2, running3,
                terminating, terminated1, terminated2), now);
        // verify started machines: 1 pending + 3 running that
        // don't have an inactive membership status
        assertThat(pool.getStartedMachines().size(), is(4));
        assertThat(pool.getStartedMachines(), is(asList(pending1, running1, running2, running3)));
    }

    /**
     * Test equality comparisons.
     */
    @Test
    public void testEquality() throws IOException {
        MachinePool empty1 = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"));
        MachinePool empty1Clone = pool(DateTime.parse("2014-01-13T12:00:00.000Z"));
        // different timestamp
        MachinePool empty2 = pool(UtcTime.parse("2014-01-14T15:00:00.000Z"));

        // single-member pool
        MachinePool singlePool = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"),
                machineNoIp("m1", MachineState.PENDING, UtcTime.parse("2014-01-13T11:00:00.000Z")));
        MachinePool singlePoolClone = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"),
                machineNoIp("m1", MachineState.PENDING, UtcTime.parse("2014-01-13T11:00:00.000Z")));

        // multi-member pool
        MachinePool multiPool = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"),
                machineNoIp("m1", MachineState.RUNNING, UtcTime.parse("2014-01-13T11:00:00.000Z")),
                machineNoIp("m2", MachineState.REQUESTED, null));
        MachinePool multiPoolClone = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"),
                machineNoIp("m1", MachineState.RUNNING, UtcTime.parse("2014-01-13T11:00:00.000Z")),
                machineNoIp("m2", MachineState.REQUESTED, null));

        assertThat(empty1, is(empty1Clone));
        assertThat(empty1, is(not(empty2)));

        assertThat(singlePool, is(singlePoolClone));
        assertThat(singlePool, is(not(empty1)));
        assertThat(singlePool, is(not(multiPool)));

        assertThat(multiPool, is(multiPoolClone));
        assertThat(multiPool, is(not(empty1)));
        assertThat(multiPool, is(not(singlePool)));

        // a bit far-fetched but test with null timestamps (*could* occur if a
        // bad machine pool was parsed from json)
        MachinePool noTimestamp = JsonUtils
                .toObject(JsonUtils.parseJsonResource("json/invalidpool-missing-timestamp.json"), MachinePool.class);
        MachinePool noTimestampClone = JsonUtils
                .toObject(JsonUtils.parseJsonResource("json/invalidpool-missing-timestamp.json"), MachinePool.class);
        assertThat(noTimestamp, is(noTimestampClone));
        assertThat(empty1, is(not(noTimestamp)));
        assertThat(singlePool, is(not(noTimestamp)));
        assertThat(multiPool, is(not(noTimestamp)));
        assertThat(noTimestamp, is(not(multiPool)));
    }

    /**
     * Test hashcode.
     */
    @Test
    public void testHashcode() throws IOException {
        MachinePool empty1 = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"));
        MachinePool empty1Clone = pool(DateTime.parse("2014-01-13T12:00:00.000Z"));
        // different timestamp
        MachinePool empty2 = pool(UtcTime.parse("2014-01-14T15:00:00.000Z"));

        // single-member pool
        MachinePool singlePool = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"),
                machineNoIp("m1", MachineState.PENDING, UtcTime.parse("2014-01-13T11:00:00.000Z")));
        MachinePool singlePoolClone = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"),
                machineNoIp("m1", MachineState.PENDING, UtcTime.parse("2014-01-13T11:00:00.000Z")));

        // multi-member pool
        MachinePool multiPool = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"),
                machineNoIp("m1", MachineState.RUNNING, UtcTime.parse("2014-01-13T11:00:00.000Z")),
                machineNoIp("m2", MachineState.REQUESTED, null));
        MachinePool multiPoolClone = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"),
                machineNoIp("m1", MachineState.RUNNING, UtcTime.parse("2014-01-13T11:00:00.000Z")),
                machineNoIp("m2", MachineState.REQUESTED, null));

        assertThat(empty1.hashCode(), is(empty1Clone.hashCode()));
        assertThat(empty1.hashCode(), is(not(empty2.hashCode())));

        assertThat(singlePool.hashCode(), is(singlePoolClone.hashCode()));
        assertThat(singlePool.hashCode(), is(not(empty1.hashCode())));
        assertThat(singlePool.hashCode(), is(not(multiPool.hashCode())));

        assertThat(multiPool.hashCode(), is(multiPoolClone.hashCode()));
        assertThat(multiPool.hashCode(), is(not(empty1.hashCode())));
        assertThat(multiPool.hashCode(), is(not(singlePool.hashCode())));
    }

    /**
     * Verifies proper parsing of a {@link MachinePool} from its JSON
     * representation.
     */
    @Test
    public void parseEmptyMachinePoolFromJson() throws IOException {
        MachinePool expectedPool = pool(DateTime.parse("2014-01-13T12:00:00.000Z"));

        MachinePool parsedPool = MachinePool.fromJson(loadJson(emptyPoolJson));
        assertThat(parsedPool, is(expectedPool));
    }

    /**
     * Verifies proper conversion of an empty {@link MachinePool} to its JSON
     * representation.
     *
     * @throws IOException
     */
    @Test
    public void convertEmptyMachinePoolToJson() throws IOException {
        MachinePool pool = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"));
        assertThat(pool.toJson(), is(JsonUtils.parseJsonResource(emptyPoolJson)));
    }

    /**
     * Verifies proper parsing of a {@link MachinePool} containing a single
     * {@link Machine}.
     *
     * @throws IOException
     */
    @Test
    public void parseSingleMachinePoolFromJson() throws IOException {
        Machine machine1 = Machine.builder().id("m1").machineState(MachineState.PENDING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.IN_SERVICE)
                .requestTime(UtcTime.parse("2014-01-13T11:00:00.000Z"))
                .launchTime(UtcTime.parse("2014-01-13T11:00:00.000Z")).publicIps(ips("1.2.3.4"))
                .privateIps(ips("1.2.3.5")).metadata(parseJsonString("{\"k1\": \"v1\"}").getAsJsonObject()).build();
        MachinePool expectedPool = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"), machine1);

        MachinePool parsedPool = MachinePool.fromJson(loadJson(singleMachinePoolJson));
        assertThat(parsedPool, is(expectedPool));
    }

    /**
     * Verifies proper JSON conversion of a {@link MachinePool} containing a
     * single {@link Machine}.
     *
     * @throws IOException
     */
    @Test
    public void convertSingleMachinePoolToJson() throws IOException {
        Machine machine1 = Machine.builder().id("m1").machineState(MachineState.PENDING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").serviceState(ServiceState.IN_SERVICE)
                .requestTime(UtcTime.parse("2014-01-13T11:00:00.000Z"))
                .launchTime(UtcTime.parse("2014-01-13T11:00:00.000Z")).publicIps(ips("1.2.3.4"))
                .privateIps(ips("1.2.3.5")).metadata(parseJsonString("{\"k1\": \"v1\"}").getAsJsonObject()).build();
        MachinePool pool = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"), machine1);

        assertThat(pool.toJson(), is(JsonUtils.parseJsonResource(singleMachinePoolJson)));
    }

    /**
     * Verifies proper parsing of a {@link MachinePool} containing several
     * {@link Machine}s, where one {@link Machine} doesn't have a launch time
     * and one {@link Machine} has meta data.
     *
     * @throws IOException
     */
    @Test
    public void parseMultiMachinePoolFromJson() throws IOException {
        Machine machine1 = Machine.builder().id("m1").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").membershipStatus(new MembershipStatus(true, false))
                .serviceState(ServiceState.IN_SERVICE).requestTime(UtcTime.parse("2014-01-13T11:00:00.000Z"))
                .launchTime(UtcTime.parse("2014-01-13T11:00:00.000Z")).publicIps(ips("1.2.3.4"))
                .metadata(parseJsonString("{\"k1\": \"v1\"}").getAsJsonObject()).build();
        Machine machine2 = Machine.builder().id("m2").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        MachinePool expectedPool = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"), machine1, machine2);

        MachinePool parsedPool = MachinePool.fromJson(loadJson(multiMachinePoolJson));
        assertThat(parsedPool, is(expectedPool));
    }

    /**
     * Verifies proper JSON conversion of a {@link MachinePool} containing
     * several {@link Machine}s, where one {@link Machine} doesn't have a launch
     * time and one {@link Machine} has meta data.
     *
     * @throws IOException
     */
    @Test
    public void convertMultiMachinePoolToJson() throws IOException {
        Machine machine1 = Machine.builder().id("m1").machineState(MachineState.RUNNING).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").membershipStatus(new MembershipStatus(true, false))
                .serviceState(ServiceState.IN_SERVICE).requestTime(UtcTime.parse("2014-01-13T11:00:00.000Z"))
                .launchTime(UtcTime.parse("2014-01-13T11:00:00.000Z")).publicIps(ips("1.2.3.4"))
                .metadata(parseJsonString("{\"k1\": \"v1\"}").getAsJsonObject()).build();
        Machine machine2 = Machine.builder().id("m2").machineState(MachineState.REQUESTED).cloudProvider("AWS-EC2")
                .region("us-east-1").machineSize("m1.small").build();
        MachinePool pool = pool(UtcTime.parse("2014-01-13T12:00:00.000Z"), machine1, machine2);

        assertThat(pool.toJson(), is(JsonUtils.parseJsonResource(multiMachinePoolJson)));
    }

    @Test(expected = NullPointerException.class)
    public void parseInvalidPoolMissingMachines() throws IOException {
        MachinePool.fromJson(loadJson("json/invalidpool-missing-machines.json"));
    }

    @Test(expected = NullPointerException.class)
    public void parseInvalidPoolMissingTimestamp() throws IOException {
        MachinePool.fromJson(loadJson("json/invalidpool-missing-timestamp.json"));
    }

    @Test(expected = NullPointerException.class)
    public void parseInvalidPoolWithMachineMissingId() throws IOException {
        MachinePool.fromJson(loadJson("json/invalidpool-machine-missing-id.json"));
    }

    @Test(expected = NullPointerException.class)
    public void parseInvalidPoolWithMachineMissingMachineState() throws IOException {
        MachinePool.fromJson(loadJson("json/invalidpool-machine-missing-machinestate.json"));
    }

    private String loadJson(String resourcePath) {
        try {
            return JsonUtils.toString(JsonUtils.parseJsonResource(resourcePath));
        } catch (Exception e) {
            throw new RuntimeException(
                    String.format("failed to parse JSON resource '%s': %s", resourcePath, e.getMessage()), e);
        }
    }

}


text:
package edu.virginia.engine.tween;

//object storing information relevant to one parameter being tweened.
public class TweenParam implements Comparable<TweenParam>{
	TweenableParam paramToTween;
	double startVal;
	double endVal;
	double tweenTime;

	public TweenParam(TweenableParam paramToTween, double startVal, double endVal, double tweenTime) {
		super();
		this.paramToTween = paramToTween;
		this.startVal = startVal;
		this.endVal = endVal;
		this.tweenTime = tweenTime;
	}

	public TweenableParam getParamToTween() {
		return paramToTween;
	}
	public void setParamToTween(TweenableParam paramToTween) {
		this.paramToTween = paramToTween;
	}
	public double getStartVal() {
		return startVal;
	}
	public void setStartVal(double startVal) {
		this.startVal = startVal;
	}
	public double getEndVal() {
		return endVal;
	}
	public void setEndVal(double endVal) {
		this.endVal = endVal;
	}
	public double getTweenTime() {
		return tweenTime;
	}
	public void setTweenTime(double tweenTime) {
		this.tweenTime = tweenTime;
	}

	@Override
	public int compareTo(TweenParam other) {
		// TODO Auto-generated method stub
		return Double.compare(this.getTweenTime(), ((TweenParam) other).getTweenTime());
	}

	@Override
	public String toString() {
		return "TweenParam [" + "startVal=" + startVal + ", endVal=" + endVal
				+ ", tweenTime=" + tweenTime +", paramToTween=" + this.paramToTween + "]";
	}

}


text:
/*
* generated by Xtext
*/
package org.xtext.example.mydsl.referencesuntyped.parser.antlr;

import java.io.InputStream;
import org.eclipse.xtext.parser.antlr.IAntlrTokenFileProvider;

public class MyDslAntlrTokenFileProvider implements IAntlrTokenFileProvider {

	public InputStream getAntlrTokenFile() {
		ClassLoader classLoader = getClass().getClassLoader();
    	return classLoader.getResourceAsStream("org/xtext/example/mydsl/referencesuntyped/parser/antlr/internal/InternalMyDsl.tokens");
	}
}


text:
Domain Crawl
============

This [Docker Swarm Stack](https://docs.docker.com/engine/reference/commandline/stack/)
definition for our domain crawler, along with basic instructions on how to run it.

Crawl Server Checklist
-----------------------

Before using a service for crawls, we need:

- Docker Swarm installed, using a speedy Docker storage implementation?
- GlusterFS mounted as `/data/` on all Swarm nodes.
- An ethernet connection with access to the world wide web.
- A DNS service that can cope with very heavy usage (we usually use Google DNS servers).
- The `nofiles` limit set to be reasonably high (e.g. 10240 instead of the default 1024).
- Docker Swarm initialised.
- ...???...

Deploying the crawl services
----------------------------

First, set up the following environment variable, which defines the server
hostname that runs the containers that provide the dockerised services. As Docker Swarm is used here, dockerised services can be accessible from other servers, so this identifies the accessible name (as opposed to the hostnames internal to the dockerised services).
E.g.

    export EXTERNAL_HOSTNAME=crawler04.n45.bl.uk

Then, you should be able to start the stack deployment:

    docker stack deploy -c docker-compose.yml dc

If you run `docker service ls` you should eventually see all services have started up
the requested number of replicas. If any are stuck, you can use:

    `docker service ps <service name> --no-trunc`

and this will list the recent attempts to start the service and any errors,
e.g. if the volume the script attempts to mount is not present.

If this looks good, you can also check some of the services directly with your web browser:

- The [Trifecta](https://github.com/ldaniels528/trifecta) Kafka UI on http://EXTERNAL_HOSTNAME:9000/
- The [OutbackCDX](https://github.com/nla/outbackcdx) service on http://EXTERNAL_HOSTNAME:9090/
- The [Heritrix metrics exporter](https://github.com/ukwa/ukwa-monitor/tree/master/heritrix3_exporter) on http://EXTERNAL_HOSTNAME:9118/metrics
- The [OpenWayback](https://github.com/ukwa/waybacks) service on http://EXTERNAL_HOSTNAME:8080/

You can also use `docker service logs` to inspect the logs of a given service.

TODO Add example, using the `-t` flag to make the output sortable.

### Hooking up Prometheus

Once the monitoring endpoint is up, it should be added to `dev-monitor`

    ssh dev-monitor
    cd github/ukwa-monitor/metrics
    vim prometheus/prometheus.yml

Now add the endpoint to the section that looks like this:

```
   # Crawl Metrics
  - job_name: 'crawl-h3'
    static_configs:
      - targets: ['crawler04.n45.bl.uk:9118']
```

e.g.

```
   # Crawl Metrics
  - job_name: 'crawl-h3'
    static_configs:
      - targets: ['crawler04.n45.bl.uk:9118']
```

(note that because our n45.bl.uk addresses are not in DNS, these aliases have to be added to the `ukwa-monitor` Docker configuration)

Now you can run:

    sh prometheus-reload-cfg.sh

to update the configuration and should see a `HTTP 200` response if the configuration is valid.

If you go to http://dev-monitor.n45.wa.bl.uk:9090/targets the new target
should be listed and should be validated after a short while. From now on
any activity on this crawler should show up in the main dashboard.

Run a test crawl
----------------

The `http://data.webarchive.org.uk/crawl-test-site/` can be used to run
a small crawl through the engine and verify things are working as expected.

You can see the options for the `launch` command, like this:

    docker run ukwa/ukwa-manage launch --help

For example you can run the test crawl by running this command:

    docker run ukwa/ukwa-manage launch -k ${EXTERNAL_HOSTNAME}:9094 uris.tocrawl.dc http://data.webarchive.org.uk/crawl-test-site/

Which means enqueue the crawl test site URL on the `uris.tocrawl.dc` Kafka queue. For the usual frequent crawl we'd have to mark the URL as a seed (using `-S`) so that the scope of the crawl is widened and the page gets rendered, but for the domain crawl the scope is already wide enough and there is no page rendering.

A stream of discovered URLs should show up in Kafka, and the logs and WARCs should start to grow. The page itself should also quickly become visible in the internal Wayback instance (unless the WARCs have been moved off).

Data will end up in `/data/dc/`

Launching the full crawl
------------------------

TODO We also need to revisit the full-domain seeding protocol

Feed in all known hosts. Pre-filter to avoid lots of empty queues in H3? Run through web render separately?


text:
<?xml version="1.0"?>
<TestCases system="Authenticated">
  <Has name="server" value="/"/>
  <Has name="title" value="Authenticated"/>
  <Has name="version" value="0.0.0"/>
  <Function name="GET_resource">
    <Has name="operation" value="GET"/>
    <Has name="path" value="/resource"/>
    <Has name="server" value="/"/>
    <Has name="title" value="Authenticated"/>
    <Has name="version" value="0.0.0"/>
    <TestCase id="0" name="Auth.apiHttpBearer.Defined='Yes'">
      <Has name="operation" value="GET"/>
      <Has name="path" value="/resource"/>
      <Has name="properties" value="secRecApiHttpBearer,secRecApiKeyQuery"/>
      <Has name="server" value="/"/>
      <Has name="title" value="Authenticated"/>
      <Has name="version" value="0.0.0"/>
      <Input type="security">
        <Var name="Auth.apiHttpBearer.Defined" value="Yes"/>
        <Var name="Auth.apiHttpBearer.Type" value="http"/>
        <Var name="Auth.apiHttpBearer.Scheme" value="bearer"/>
        <Var name="Auth.apiKeyQuery.Defined" value="Yes"/>
        <Var name="Auth.apiKeyQuery.Type" value="apiKey"/>
        <Var name="Auth.apiKeyQuery.Location" value="query"/>
        <Var name="Auth.apiKeyQuery.Name" value="apiKey"/>
      </Input>
    </TestCase>
    <TestCase id="1" failure="true" name="Auth.apiHttpBearer.Defined='No'">
      <Has name="operation" value="GET"/>
      <Has name="path" value="/resource"/>
      <Has name="properties" value="secRecApiKeyQuery"/>
      <Has name="server" value="/"/>
      <Has name="title" value="Authenticated"/>
      <Has name="version" value="0.0.0"/>
      <Input type="security">
        <Var name="Auth.apiHttpBearer.Defined" value="No" failure="true">
          <Has name="authFailure" value="true"/>
        </Var>
        <Var name="Auth.apiHttpBearer.Type" NA="true"/>
        <Var name="Auth.apiHttpBearer.Scheme" NA="true"/>
        <Var name="Auth.apiKeyQuery.Defined" value="Yes"/>
        <Var name="Auth.apiKeyQuery.Type" value="apiKey"/>
        <Var name="Auth.apiKeyQuery.Location" value="query"/>
        <Var name="Auth.apiKeyQuery.Name" value="apiKey"/>
      </Input>
    </TestCase>
    <TestCase id="2" failure="true" name="Auth.apiKeyQuery.Defined='No'">
      <Has name="operation" value="GET"/>
      <Has name="path" value="/resource"/>
      <Has name="properties" value="secRecApiHttpBearer"/>
      <Has name="server" value="/"/>
      <Has name="title" value="Authenticated"/>
      <Has name="version" value="0.0.0"/>
      <Input type="security">
        <Var name="Auth.apiHttpBearer.Defined" value="Yes"/>
        <Var name="Auth.apiHttpBearer.Type" value="http"/>
        <Var name="Auth.apiHttpBearer.Scheme" value="bearer"/>
        <Var name="Auth.apiKeyQuery.Defined" value="No" failure="true">
          <Has name="authFailure" value="true"/>
        </Var>
        <Var name="Auth.apiKeyQuery.Type" NA="true"/>
        <Var name="Auth.apiKeyQuery.Location" NA="true"/>
        <Var name="Auth.apiKeyQuery.Name" NA="true"/>
      </Input>
    </TestCase>
  </Function>
</TestCases>


text:
package com.sleepycat.db.test;

import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;

import com.sleepycat.db.*;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;

import com.sleepycat.db.test.TestUtils;
public class PartialGetTest {
    public static final String PARTIALGETTEST_DBNAME = "partialgettest.db";
    public static final byte[] data_64chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890<>".getBytes();
    @BeforeClass public static void ClassInit() {
        TestUtils.loadConfig(null);
        TestUtils.check_file_removed(TestUtils.getDBFileName(PARTIALGETTEST_DBNAME), true, true);
        TestUtils.removeall(true, true, TestUtils.BASETEST_DBDIR, TestUtils.getDBFileName(PARTIALGETTEST_DBNAME));
    }

    @AfterClass public static void ClassShutdown() {
        TestUtils.check_file_removed(TestUtils.getDBFileName(PARTIALGETTEST_DBNAME), true, true);
        TestUtils.removeall(true, true, TestUtils.BASETEST_DBDIR, TestUtils.getDBFileName(PARTIALGETTEST_DBNAME));
    }

    @Before public void PerTestInit()
        throws Exception {
    }

    @After public void PerTestShutdown()
        throws Exception {
        TestUtils.check_file_removed(TestUtils.getDBFileName(PARTIALGETTEST_DBNAME), true, true);
    }
    /*
     * Test case implementations.
     * To disable a test mark it with @Ignore
     * To set a timeout(ms) notate like: @Test(timeout=1000)
     * To indicate an expected exception notate like: (expected=Exception)
     */

    /*
     * Simple partial gets on a record which is on a single page.
     */
    @Test public void test1()
        throws DatabaseException, FileNotFoundException
    {
        DatabaseEntry key = new DatabaseEntry("key".getBytes());
        Database db = setupDb1(key, data_64chars);

        StringEntry partialData = new StringEntry();
        partialData.setPartial(true);
        partialData.setPartial(0, 12, true);

        if (db.get(null, key, partialData, LockMode.DEFAULT) !=
            OperationStatus.SUCCESS)
            fail("Failed doing partial retrieval, first part of entry on single page.");
        // Validate the data.
        if (!MatchData(data_64chars, partialData.getString(), 12))
            fail("Data mismatch from partial get.");

        partialData.setPartial(12, 12, true);
        if (db.get(null, key, partialData, LockMode.DEFAULT) !=
            OperationStatus.SUCCESS)
            fail("Failed doing partial retrieval, second part of entry on single page.");
        // Validate the data.
        if (!MatchData(new String(data_64chars, 12, 12), partialData.getString(), 12))
            fail("Data mismatch from partial get.");

        db.close(false);
    }

    /*
     * Retrieve entry using different DB_DBT_alloc flags.
     * Verify results.
     */
    @Test public void test2()
        throws DatabaseException, FileNotFoundException
    {
        DatabaseEntry key = new DatabaseEntry("key".getBytes());
        Database db = setupDb1(key, data_64chars);
        StringEntry partialData = new StringEntry();
        partialData.setPartial(true);
        partialData.setPartial(0, 12, true);

        if (db.get(null, key, partialData, LockMode.DEFAULT) !=
            OperationStatus.SUCCESS)
            fail("Failed doing partial retrieval.");
        // Validate the data.
        if (!MatchData(data_64chars, partialData.getString(), 12))
            fail("Data mismatch from partial get.");

        partialData.setReuseBuffer(true);
        if (db.get(null, key, partialData, LockMode.DEFAULT) !=
            OperationStatus.SUCCESS)
        if (!MatchData(data_64chars, partialData.getString(), 12))
            fail("Data mismatch from partial get.");

        partialData.setReuseBuffer(false);
        partialData.setUserBuffer(64, true);
        partialData.setData(new byte[64]);
        if (db.get(null, key, partialData, LockMode.DEFAULT) !=
            OperationStatus.SUCCESS)
        if (!MatchData(data_64chars, partialData.getString(), 12))
            fail("Data mismatch from partial get.");

        partialData.setPartial(12, 12, true);
        if (db.get(null, key, partialData, LockMode.DEFAULT) !=
            OperationStatus.SUCCESS)
            fail("Failed doing partial retrieval.");
        // Validate the data.
        if (!MatchData(new String(data_64chars, 12, 12), partialData.getString(), 12))
            fail("Data mismatch from partial get.");

        db.close(false);
    }

    /* Retrieve entry that spans multiple pages. */

    @Test public void test3()
        throws DatabaseException, FileNotFoundException
    {
        DatabaseEntry key = new DatabaseEntry("key".getBytes());
        StringBuffer sb = new StringBuffer(1024*100);
        for(int i = 0; i < 1024; i++) {
            sb.append("abcdefghijklmnopqrstuvwxyz1234567890!@#$%^&*()_+-=");
            sb.append("abcdefghijklmnopqrstuvwxyz1234567890!@#$%^&*()_+-=");
        }
        Database db = setupDb1(key, sb.toString().getBytes());

        StringEntry partialData = new StringEntry();
        partialData.setPartial(true);
        partialData.setPartial(0, 12, true);

        if (db.get(null, key, partialData, LockMode.DEFAULT) !=
            OperationStatus.SUCCESS)
            fail("Failed doing partial retrieval.");
        // Validate the data.
        if (!MatchData(data_64chars, partialData.getString(), 12))
            fail("Data mismatch from partial get.");

        // retrieve a chunk larger than a page size, starting at offset 0.
        partialData.setPartial(0, 2048, true);
        if (db.get(null, key, partialData, LockMode.DEFAULT) !=
            OperationStatus.SUCCESS)
            fail("Failed doing partial retrieval.");
        // Validate the data.
        if (!MatchData(sb.substring(0, 2048), partialData.getString(), 2048))
            fail("Data mismatch from partial get.");

        // retrieve a chunk larger than a page size, starting at offset greater than 0.
        partialData.setPartial(10, 2048, true);
        if (db.get(null, key, partialData, LockMode.DEFAULT) !=
            OperationStatus.SUCCESS)
            fail("Failed doing partial retrieval.");
        // Validate the data.
        if (!MatchData(sb.substring(10, 2048+10), partialData.getString(), 12))
            fail("Data mismatch from partial get.");

        db.close(false);
    }

    /*
     * Test partial retrieval using a cursor.
     */
    @Test public void test4()
        throws DatabaseException, FileNotFoundException
    {
    }

    /*
     * Test partial retrieval using different DB types.
     */
    @Test public void test5()
        throws DatabaseException, FileNotFoundException
    {
    }

    /*
     * Test partial retrieval .
     */
    @Test public void test6()
        throws DatabaseException, FileNotFoundException
    {
    }

    /*
     * Helper methods and classes follow.
     */

    private Database setupDb1(DatabaseEntry key, byte[] dataData)
        throws DatabaseException, FileNotFoundException
    {
        DatabaseConfig dbConfig = new DatabaseConfig();
        dbConfig.setErrorStream(TestUtils.getErrorStream());
        dbConfig.setErrorPrefix("PartialGetTest");
        dbConfig.setType(DatabaseType.BTREE);
        dbConfig.setPageSize(1024);
        dbConfig.setAllowCreate(true);
        Database db = new Database(TestUtils.getDBFileName(PARTIALGETTEST_DBNAME), null, dbConfig);

        DatabaseEntry data = new DatabaseEntry(dataData);

        if(db.putNoOverwrite(null, key, data) != OperationStatus.SUCCESS)
            TestUtils.ERR("Failed to create standard entry in database.");

        return db;
    }

    /* Save converting String to do data comparisons. */
    private boolean MatchData(byte[] data1, byte[] data2, int len)
    {
        return MatchData(new String(data1), new String(data2), len);
    }
    private boolean MatchData(String data1, byte[] data2, int len)
    {
        return MatchData(data1, new String(data2), len);
    }
    private boolean MatchData(byte[] data1, String data2, int len)
    {
        return MatchData(new String(data1), data2, len);
    }
    private boolean MatchData(String data1, String data2, int len)
    {
        if(data1.length() < len || data2.length() < len)
            return false;
        TestUtils.DEBUGOUT(0, "data1: " +data1.substring(0, 12));
        TestUtils.DEBUGOUT(0, "data2: " +data2.substring(0, 12));
        return data1.regionMatches(0, data2, 0, len);
    }

    static /*inner*/
    class StringEntry extends DatabaseEntry {
        StringEntry() {
        }

        StringEntry (String value) {
            setString(value);
        }

        void setString(String value) {
            byte[] data = value.getBytes();
            setData(data);
            setSize(data.length);
        }

        String getString() {
            return new String(getData(), getOffset(), getSize());
        }
    }
}


text:
package demo.controller;

import demo.domain.Person;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.Date;

/**
 * Created by Administrator on 2017-07-25.
 *
 * @RestController 等价于 @controller 和 @RequestBody
 */
@RestController
public class HelloController {

    @RequestMapping("/hello")
    public String hello() {
        return "hello";
    }

    @RequestMapping("/hello2")
    public String hello2() {
        return "hello-2";
    }

    @RequestMapping("/hello3")
    public String hello3() {
        return "hello-3";
    }

    @RequestMapping("/getPerson")
    public Person getPerson() {
        Person p = new Person();
        p.setId(1);
        p.setName("Robin");
        p.setCreateTime(new Date());

        return p;
    }

}


text:
Deploy your `./dist/` directory to the Livefyre CDN.

It will read from the `name` and `version` in your package.json

You will need to set the following environment variables

    export LF_CDN_S3_KEY='...'
    export LF_CDN_S3_SECRET='...'

Install with npm.

    npm install -g git+ssh://git@github.com:Livefyre/lfcdn

Note the `-g`. It will add `lfcdn` to your PATH.

## `lfcdn -h`

    Usage: lfcdn -e {dev|qa|staging|prod} -c [/path/to/config.json] [--reporter (url|none)]

`--reporter urls` will print URLs to stdout for each file

## `lfcdn -c [/path/to/config.json]`

Load config from a JSON config file. Supported config options:

* `dir` - S3 key to deploy files to. Defaults to a path derrived from the `name` in `package.json`

## `lfcdn -e {dev|qa|staging|prod}`

Environment to deploy to. Defaults to `dev`.

    livefyre-cdn-dev: deploying streamhub-sdk
    [gulp] [cache]  libs/streamhub-sdk/2.7.5/streamhub-sdk.min.css
    [gulp] [cache]  libs/streamhub-sdk/2.7.5/streamhub-sdk.min.js
    [gulp] [cache]  libs/streamhub-sdk/2.7.5/streamhub-sdk.min.js.map

## `lfcdn -f`

By default lfcdn will not update published files. One can, however, overwrite published files with the force option.


text:
<?xml version="1.0" encoding="UTF-8"?>

<rootTag>
  <Award>
    <AwardTitle>Universality Limits, Orthogonal Polynomials and Weighted Polynomial Approximation</AwardTitle>
    <AwardEffectiveDate>05/15/2007</AwardEffectiveDate>
    <AwardExpirationDate>09/30/2011</AwardExpirationDate>
    <AwardAmount>189001</AwardAmount>
    <AwardInstrument>
      <Value>Continuing grant</Value>
    </AwardInstrument>
    <Organization>
      03040000
      <Directorate>
        <LongName>Direct For Mathematical & Physical Scien</LongName>
      </Directorate>
      <Division>
        <LongName>Division Of Mathematical Sciences</LongName>
      </Division>
    </Organization>
    <ProgramOfficer>
      <SignBlockName>Bruce P. Palka</SignBlockName>
    </ProgramOfficer>
    <AbstractNarration>We have developed a new approach to proving universality limits involving orthogonal polynomials. We allow, for example, the weight to be positive a.e. globally, and continuous at a given point. That is enough for universality for a fixed weight. We intend to explore the limits of this method for weights on compact and non-compact sets, as well as for varying weights.<br/><br/>Universality limits of this type arise in the theory of random matrices and its applications, which include various topics in mathematical physics. The significance is that previously strong global hypotheses were required, such as analyticity. Now only weak local and global assumptions are required.</AbstractNarration>
    <MinAmdLetterDate>04/10/2007</MinAmdLetterDate>
    <MaxAmdLetterDate>03/24/2009</MaxAmdLetterDate>
    <ARRAAmount/>
    <AwardID>0700427</AwardID>
    <Investigator>
      <FirstName>Doron</FirstName>
      <LastName>Lubinsky</LastName>
      <EmailAddress>lubinsky@math.gatech.edu</EmailAddress>
      <StartDate>04/10/2007</StartDate>
      <EndDate/>
      <RoleCode>Principal Investigator</RoleCode>
    </Investigator>
    <Institution>
      <Name>Georgia Tech Research Corporation</Name>
      <CityName>Atlanta</CityName>
      <ZipCode>303320420</ZipCode>
      <PhoneNumber>4048944819</PhoneNumber>
      <StreetAddress>Office of Sponsored Programs</StreetAddress>
      <CountryName>United States</CountryName>
      <StateName>Georgia</StateName>
      <StateCode>GA</StateCode>
    </Institution>
    <FoaInformation>
      0000099
      <Name>Other Applications NEC</Name>
    </FoaInformation>
  </Award>
</rootTag>


text:
'use strict';

angular.module('core').controller('AboutUsController', ['$scope',
	function($scope) {
	    document.body.style.background = 'none';
	}
]);


text:
-
class Direction
end
-


text:
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.

[assembly: AssemblyTitle("ZBrad's Library for Microsoft WCF")]
[assembly: AssemblyDescription(@"Provides classes for use with Microsoft WCF.
      Current supports creating services and clients using TCP and REST protocols.
      Also provides classes for creating a WCF router for TCP or REST.")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Brad Merrill")]
[assembly: AssemblyProduct("ZBrad.WcfLib")]
[assembly: AssemblyCopyright("Copyright Brad Merrill © 2015")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]

// Setting ComVisible to false makes the types in this assembly not visible
// to COM components.  If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]

// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("31c09892-446e-442a-b5d8-648ba6021f77")]

// Version information for an assembly consists of the following four values:
//
//      Major Version
//      Minor Version
//      Build Number
//      Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
[assembly: AssemblyInformationalVersion("1.0.0")]


text:
namespace base {
class Clock;
class Value;
}

namespace chromeos {

class NetworkState;
class ManagedNetworkConfigurationHandler;

namespace internal {
struct NetworkAndMatchingCert;
struct MatchingCertAndResolveStatus;
}  // namespace internal

// Observes the known networks. If a network is configured with a client
// certificate pattern, this class searches for a matching client certificate.
// Each time it finds a match, it configures the network accordingly.
class COMPONENT_EXPORT(CHROMEOS_NETWORK) ClientCertResolver
    : public NetworkStateHandlerObserver,
      public NetworkCertLoader::Observer,
      public NetworkPolicyObserver {
 public:
  class Observer {
   public:
    Observer& operator=(const Observer&) = delete;

    // Called every time resolving of client certificate patterns finishes,
    // no resolve requests are pending and no tasks are running.
    // |network_properties_changed| will be true if any network properties were
    // changed by this resolver since the last notification.
    virtual void ResolveRequestCompleted(bool network_properties_changed) = 0;

   protected:
    virtual ~Observer() {}
  };

  ClientCertResolver();

  ClientCertResolver(const ClientCertResolver&) = delete;
  ClientCertResolver& operator=(const ClientCertResolver&) = delete;

  ~ClientCertResolver() override;

  void Init(NetworkStateHandler* network_state_handler,
            ManagedNetworkConfigurationHandler* managed_network_config_handler);

  void AddObserver(Observer* observer);
  void RemoveObserver(Observer* observer);

  // Returns true if any resolve tasks are running. Every time a task finishes
  // and no further requests are pending, a notification is sent, see
  // |Observer|.
  bool IsAnyResolveTaskRunning() const;

  // Sets the clock for testing. This clock is used when checking the
  // certificates for expiration.
  void SetClockForTesting(base::Clock* clock);

  // Returns true and sets the Shill properties that have to be configured in
  // |shill_properties| if the client certificate could be resolved according to
  // |client_cert_config|.
  // Returns false otherwise and sets empty Shill properties to clear the
  // certificate configuration.
  // Note that it uses the global clock when checking the certificates for
  // expiration.
  static bool ResolveClientCertificateSync(
      const client_cert::ConfigType client_cert_type,
      const client_cert::ClientCertConfig& client_cert_config,
      base::Value* shill_properties);

  // Allows overwriting the function which gets the client certificate
  // provisioning profile id of a certificate. This is necessary for unit tests,
  // because there we use an NSS soft token which does not support the custom
  // attributes used for storing the id. Calling this will overwrite the
  // behavior until the returned ScopedClosureRunner is destructed, which will
  // reset to the original behavior.
  using ProvisioningProfileIdGetter =
      base::RepeatingCallback<std::string(CERTCertificate* cert)>;
  static base::ScopedClosureRunner SetProvisioningIdForCertGetterForTesting(
      ProvisioningProfileIdGetter getter);

 private:
  // NetworkStateHandlerObserver overrides
  void NetworkListChanged() override;
  void NetworkConnectionStateChanged(const NetworkState* network) override;

  // NetworkCertLoader::Observer overrides
  void OnCertificatesLoaded() override;

  // NetworkPolicyObserver overrides
  void PolicyAppliedToNetwork(const std::string& service_path) override;

  // Forget the resolved certificate of |network| - ensures that the next
  // ResolveNetwork run will reconfigure the certificate (and EAP Identity
  // field) even if the certificate has not changed.
  void ForgetResolvedCert(const NetworkState* network);

  // Check which networks of |networks| are configured with a client certificate
  // pattern. Search for certificates, on the worker thread, and configure the
  // networks for which a matching cert is found (see ConfigureCertificates).
  void ResolveNetworks(const NetworkStateHandler::NetworkStateList& networks);

  // Resolves certificates for the pending networks. This will always trigger a
  // ResolveRequestCompleted notification, even if the queue is empty.
  void ResolvePendingNetworks();

  // |matches| contains networks for which a matching certificate was found.
  // Configures these networks.
  void ConfigureCertificates(
      std::vector<internal::NetworkAndMatchingCert> matches);

  // Trigger a ResolveRequestCompleted event on all observers.
  void NotifyResolveRequestCompleted();

  // Returns Time::Now() unless a mock clock has been installed with
  // SetClockForTesting, in which case the time according to that clock is used
  // instead.
  base::Time Now() const;

  base::ObserverList<Observer, true>::Unchecked observers_;

  // Stores the current client certificate resolution status for each network.
  // The key is the network's service path. If a network is not present, it is
  // not known yet (or disappeared from the list of known networks again).
  base::flat_map<std::string, internal::MatchingCertAndResolveStatus>
      networks_status_;

  // The list of network paths that still have to be resolved.
  std::set<std::string> queued_networks_to_resolve_;

  // True if currently a resolve task is running.
  bool resolve_task_running_;

  // True if any network properties were changed since the last notification to
  // observers.
  bool network_properties_changed_;

  // Unowned associated (global or test) instance.
  NetworkStateHandler* network_state_handler_;

  // Unowned associated (global or test) instance.
  ManagedNetworkConfigurationHandler* managed_network_config_handler_;

  // Can be set for testing.
  base::Clock* testing_clock_;

  SEQUENCE_CHECKER(sequence_checker_);

  base::WeakPtrFactory<ClientCertResolver> weak_ptr_factory_{this};
};

}  // namespace chromeos

#endif  // CHROMEOS_NETWORK_CLIENT_CERT_RESOLVER_H_


text:
<?xml version="1.0" encoding="utf-8"?>
<layer-list xmlns:android="http://schemas.android.com/apk/res/android">
    <item>
        <shape
                android:shape="rectangle">
            <solid android:color="@color/dark_feed_background_selected_start"/>
        </shape>
    </item>
    <item
            android:top="0.5dp"
            android:bottom="0.5dp">
        <shape android:shape="rectangle">
            <solid android:color="@color/dark_feed_background_selected_end"/>
        </shape>
    </item>
</layer-list>