<|start_filename|>src/NLU.DevOps.LuisV3/ILuisTestClient.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Luis
{
    using System;
    using System.Threading;
    using System.Threading.Tasks;
    using Microsoft.Azure.CognitiveServices.Language.LUIS.Runtime.Models;

    /// <summary>
    /// LUIS interface for testing operations.
    /// </summary>
    public interface ILuisTestClient : IDisposable
    {
        /// <summary>
        /// Queries the LUIS app to extract intent and entities.
        /// </summary>
        /// <returns>Task to await the LUIS results.</returns>
        /// <param name="predictionRequest">Prediction request.</param>
        /// <param name="cancellationToken">Cancellation token.</param>
        Task<PredictionResponse> QueryAsync(PredictionRequest predictionRequest, CancellationToken cancellationToken);

        /// <summary>
        /// Performs intent recognition from speech using the given audio file.
        /// </summary>
        /// <returns>Task to await the LUIS results.</returns>
        /// <param name="speechFile">Path to file.</param>
        /// <param name="predictionRequest">Prediction request.</param>
        /// <param name="cancellationToken">Cancellation token.</param>
        Task<SpeechPredictionResponse> RecognizeSpeechAsync(string speechFile, PredictionRequest predictionRequest, CancellationToken cancellationToken);
    }
}


<|start_filename|>src/NLU.DevOps.MockProvider/Program.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.MockProvider
{
    using System;

    internal class Program
    {
        private static void Main()
        {
        }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine/NLUClientFactory.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine
{
    using System;
    using Microsoft.Extensions.Configuration;
    using Models;

    internal class NLUClientFactory
    {
        public static INLUTrainClient CreateTrainInstance(BaseOptions options, IConfiguration configuration, string settingsPath = null)
        {
            if (!ServiceResolver.TryResolve<INLUClientFactory>(options, out var serviceFactory))
            {
                throw new InvalidOperationException($"Invalid NLU provider type '{options.Service}'.");
            }

            return serviceFactory.CreateTrainInstance(configuration, settingsPath);
        }

        public static INLUTestClient CreateTestInstance(BaseOptions options, IConfiguration configuration, string settingsPath = null)
        {
            if (!ServiceResolver.TryResolve<INLUClientFactory>(options, out var serviceFactory))
            {
                throw new InvalidOperationException($"Invalid NLU provider type '{options.Service}'.");
            }

            return serviceFactory.CreateTestInstance(configuration, settingsPath);
        }
    }
}


<|start_filename|>src/NLU.DevOps.ModelPerformance/ConfusionMatrixResultKind.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.ModelPerformance
{
    /// <summary>
    /// Enumeration of confusion matrix result types.
    /// </summary>
    public enum ConfusionMatrixResultKind
    {
        /// <summary>
        /// Default <see cref="ConfusionMatrixResultKind"/>.
        /// </summary>
        None,

        /// <summary>
        /// True positive.
        /// </summary>
        TruePositive = 1,

        /// <summary>
        /// True negative.
        /// </summary>
        TrueNegative,

        /// <summary>
        /// False positive.
        /// </summary>
        FalsePositive,

        /// <summary>
        /// False negative.
        /// </summary>
        FalseNegative,
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine/Train/TrainOptions.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Train
{
    using global::CommandLine;

    [Verb("train", HelpText = "Trains the NLU model.")]
    internal class TrainOptions : BaseOptions
    {
        [Option('u', "utterances", HelpText = "Path to utterances.", Required = false)]
        public string UtterancesPath { get; set; }

        [Option('m', "model-settings", HelpText = "Path to NLU model settings.", Required = false)]
        public string SettingsPath { get; set; }

        [Option('a', "save-appsettings", HelpText = "Flag to save NLU model instance appsettings.", Required = false)]
        public bool SaveAppSettings { get; set; }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine.Tests/Clean/CleanCommandMock.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Tests.Clean
{
    using CommandLine.Clean;
    using Models;

    internal class CleanCommandMock : CleanCommand
    {
        public CleanCommandMock(CleanOptions options)
            : base(options)
        {
        }

        protected override INLUTrainClient CreateNLUTrainClient() =>
            new MockNLUTrainClient();
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine/Test/TestOptions.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Test
{
    using global::CommandLine;

    [Verb("test", HelpText = "Runs test cases against the NLU model.")]
    internal class TestOptions : BaseOptions
    {
        [Option('u', "utterances", HelpText = "Path to utterances.", Required = true)]
        public string UtterancesPath { get; set; }

        [Option('m', "model-settings", HelpText = "Path to NLU model settings.", Required = false)]
        public string SettingsPath { get; set; }

        [Option('d', "speech-directory", HelpText = "Path to recordings directory.", Required = false)]
        public string SpeechFilesDirectory { get; set; }

        [Option('t', "transcriptions", HelpText = "Path to input transcriptions cache.", Required = false)]
        public string TranscriptionsFile { get; set; }

        [Option('o', "output", HelpText = "Path to labeled results output.", Required = false)]
        public string OutputPath { get; set; }

        [Option('p', "parallelism", HelpText = "Numeric value to determine the numer of parallel tests.  Default value is 3.", Required = false)]
        public int Parallelism { get; set; } = 3;
    }
}


<|start_filename|>src/NLU.DevOps.ModelPerformance/NLUCompareResults.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.ModelPerformance
{
    using System;
    using System.Collections.Generic;
    using System.Linq;

    /// <summary>
    /// NLU compare results.
    /// </summary>
    public class NLUCompareResults
    {
        /// <summary>
        /// Initializes a new instance of the <see cref="NLUCompareResults"/> class.
        /// </summary>
        /// <param name="testCases">Test cases.</param>
        public NLUCompareResults(IReadOnlyList<TestCase> testCases)
        {
            this.TestCases = testCases;
            this.LazyStatistics = new Lazy<NLUStatistics>(this.CalculateStatistics);
        }

        /// <summary>
        /// Gets the test cases.
        /// </summary>
        public IReadOnlyList<TestCase> TestCases { get; }

        /// <summary>
        /// Gets the test case statistics.
        /// </summary>
        public NLUStatistics Statistics => this.LazyStatistics.Value;

        private Lazy<NLUStatistics> LazyStatistics { get; }

        private static ConfusionMatrix CalculateConfusionMatrix(IEnumerable<TestCase> testCases)
        {
            var groups = testCases
                .GroupBy(testCase => testCase.ResultKind)
                .ToDictionary(
                    group => group.Key,
                    group => group.Count());

            return new ConfusionMatrix(
                GetValueOrDefault(groups, ConfusionMatrixResultKind.TruePositive),
                GetValueOrDefault(groups, ConfusionMatrixResultKind.TrueNegative),
                GetValueOrDefault(groups, ConfusionMatrixResultKind.FalsePositive),
                GetValueOrDefault(groups, ConfusionMatrixResultKind.FalseNegative));
        }

        private static TValue GetValueOrDefault<TKey, TValue>(IReadOnlyDictionary<TKey, TValue> dictionary, TKey key)
        {
            return dictionary.TryGetValue(key, out var value)
                ? value
                : default(TValue);
        }

        private NLUStatistics CalculateStatistics()
        {
            return new NLUStatistics(
                CalculateConfusionMatrix(this.TestCases.Where(testCase => testCase.TargetKind == ComparisonTargetKind.Text)),
                CalculateConfusionMatrix(this.TestCases.Where(testCase => testCase.TargetKind == ComparisonTargetKind.Intent)),
                CalculateConfusionMatrix(this.TestCases.Where(testCase => testCase.TargetKind == ComparisonTargetKind.Entity)),
                CalculateConfusionMatrix(this.TestCases.Where(testCase => testCase.TargetKind == ComparisonTargetKind.EntityValue)),
                this.TestCases
                    .Where(testCase => testCase.Group != null)
                    .Where(testCase => testCase.TargetKind == ComparisonTargetKind.Intent)
                    .GroupBy(testCase => testCase.Group)
                    .ToDictionary(group => group.Key, group => CalculateConfusionMatrix(group)),
                this.TestCases
                    .Where(testCase => testCase.Group != null)
                    .Where(testCase => testCase.TargetKind == ComparisonTargetKind.Entity)
                    .GroupBy(testCase => testCase.Group)
                    .ToDictionary(group => group.Key, group => CalculateConfusionMatrix(group)),
                this.TestCases
                    .Where(testCase => testCase.Group != null)
                    .Where(testCase => testCase.TargetKind == ComparisonTargetKind.EntityValue)
                    .GroupBy(testCase => testCase.Group)
                    .ToDictionary(group => group.Key, group => CalculateConfusionMatrix(group)));
        }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine.Tests/Clean/CleanOptionsTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Tests.Clean
{
    using System.Collections.Generic;
    using CommandLine.Clean;
    using FluentAssertions;
    using global::CommandLine;
    using NUnit.Framework;

    [TestFixture]
    internal class CleanOptionsTests
    {
        private List<string> options;

        [SetUp]
        public void SetUp()
        {
            this.options = new List<string>();
            this.options.Add("-s");
            this.options.Add("luis");
        }

        [Test]
        public void SettingForCleanOptionsWhenSet()
        {
            this.options.Add("-a");
            var args = this.options.ToArray();
            var parser = Parser.Default.ParseArguments<CleanOptions>(args)
                .WithParsed(o => o.DeleteAppSettings.Should().BeTrue())
                .WithNotParsed(o => Assert.Fail("Could not Parse Options"));
        }

        [Test]
        public void SettingForCleanOptionsWhenNotSet()
        {
            var args = this.options.ToArray();
            var parser = Parser.Default.ParseArguments<CleanOptions>(args)
                .WithParsed(o => o.DeleteAppSettings.Should().BeFalse())
                .WithNotParsed(o => Assert.Fail("Could not Parse Options"));
        }
    }
}


<|start_filename|>src/NLU.DevOps.Lex/LexTestClient.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Lex
{
    using System;
    using System.Threading;
    using System.Threading.Tasks;
    using Amazon;
    using Amazon.Lex;
    using Amazon.Lex.Model;
    using Amazon.Runtime;
    using Logging;
    using Microsoft.Extensions.Logging;

    internal sealed class LexTestClient : ILexTestClient
    {
        /// <summary>
        /// Number of retries per Lex model building service request.
        /// </summary>
        /// <remarks>
        /// Experimental results show 5 retries is sufficient for avoiding <see cref="Amazon.LexModelBuildingService.Model.ConflictException"/>.
        /// </remarks>
        private const int RetryCount = 5;

        private static readonly TimeSpan RetryConflictDelay = TimeSpan.FromSeconds(10);
        private static readonly TimeSpan RetryLimitExceededDelay = TimeSpan.FromSeconds(1);

        public LexTestClient(AWSCredentials credentials, RegionEndpoint regionEndpoint)
        {
            this.AmazonLexClient = new AmazonLexClient(credentials, regionEndpoint);
        }

        private static ILogger Logger => LazyLogger.Value;

        private static Lazy<ILogger> LazyLogger { get; } = new Lazy<ILogger>(() => ApplicationLogger.LoggerFactory.CreateLogger<LexNLUTrainClient>());

        private AmazonLexClient AmazonLexClient { get; }

        public Task<PostContentResponse> PostContentAsync(PostContentRequest request, CancellationToken cancellationToken)
        {
            return RetryAsync(this.AmazonLexClient.PostContentAsync, request, cancellationToken);
        }

        public Task<PostTextResponse> PostTextAsync(PostTextRequest request, CancellationToken cancellationToken)
        {
            return RetryAsync(this.AmazonLexClient.PostTextAsync, request, cancellationToken);
        }

        public void Dispose()
        {
            this.AmazonLexClient.Dispose();
        }

        private static Task RetryAsync<TRequest>(Func<TRequest, CancellationToken, Task> actionAsync, TRequest request, CancellationToken cancellationToken)
        {
            async Task<bool> asyncActionWrapper(TRequest requestParam, CancellationToken cancellationTokenParam)
            {
                await actionAsync(requestParam, cancellationTokenParam).ConfigureAwait(false);
                return true;
            }

            return RetryAsync(asyncActionWrapper, request, cancellationToken);
        }

        private static async Task<TResponse> RetryAsync<TRequest, TResponse>(Func<TRequest, CancellationToken, Task<TResponse>> actionAsync, TRequest request, CancellationToken cancellationToken)
        {
            Task onErrorAsync(Exception ex, TimeSpan delay, string name)
            {
                Logger.LogWarning(ex, $"Encountered '{name}', retrying.");
                return Task.Delay(delay, cancellationToken);
            }

            var count = 0;
            while (count++ < RetryCount)
            {
                try
                {
                    return await actionAsync(request, cancellationToken).ConfigureAwait(false);
                }
                catch (ConflictException ex)
                when (count < RetryCount)
                {
                    await onErrorAsync(ex, RetryConflictDelay, "ConflictException").ConfigureAwait(false);
                }
                catch (LimitExceededException ex)
                when (count < RetryCount)
                {
                    await onErrorAsync(ex, RetryLimitExceededDelay, "LimitExceededException").ConfigureAwait(false);
                }
            }

            throw new InvalidOperationException("Exception will be rethrown before reaching this point.");
        }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine.Tests/Test/TestOptionsTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Tests.Test
{
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;
    using CommandLine.Test;
    using FluentAssertions;
    using global::CommandLine;
    using NUnit.Framework;

    [TestFixture]
    internal class TestOptionsTests
    {
        private List<string> options;

        [SetUp]
        public void SetUp()
        {
            this.options = new List<string>
            {
                "-s",
                "luis"
            };
        }

        [Test]
        public void SettingForTestOptionsWhenSet()
        {
            this.options.Add("-u");
            this.options.Add("utterances");
            this.options.Add("-m");
            this.options.Add("servicesettings");
            this.options.Add("-d");
            this.options.Add("speechdirectory");
            this.options.Add("-t");
            this.options.Add("transcriptionfile");
            this.options.Add("-o");
            this.options.Add("outputpath");
            this.options.Add("-p");
            this.options.Add("5");
            var args = this.options.ToArray();
            var parser = Parser.Default.ParseArguments<TestOptions>(args)
                .WithParsed(o =>
                {
                    o.UtterancesPath.Should().Be("utterances");
                    o.SettingsPath.Should().Be("servicesettings");
                    o.SpeechFilesDirectory.Should().Be("speechdirectory");
                    o.TranscriptionsFile.Should().Be("transcriptionfile");
                    o.OutputPath.Should().Be("outputpath");
                    o.Parallelism.Should().Be(5);
                })
                .WithNotParsed(o => Assert.Fail("Could not Parse Options"));
        }

        [Test]
        public void SettingForTestOptionsWhenLongNameSet()
        {
            this.options.Add("--utterances");
            this.options.Add("utterances");
            this.options.Add("--model-settings");
            this.options.Add("servicesettings");
            this.options.Add("--speech-directory");
            this.options.Add("speechdirectory");
            this.options.Add("--transcriptions");
            this.options.Add("transcriptionfile");
            this.options.Add("--output");
            this.options.Add("outputpath");
            this.options.Add("--parallelism");
            this.options.Add("5");
            var args = this.options.ToArray();
            var parser = Parser.Default.ParseArguments<TestOptions>(args).WithParsed<TestOptions>(o =>
            {
                o.UtterancesPath.Should().Be("utterances");
                o.SettingsPath.Should().Be("servicesettings");
                o.SpeechFilesDirectory.Should().Be("speechdirectory");
                o.TranscriptionsFile.Should().Be("transcriptionfile");
                o.OutputPath.Should().Be("outputpath");
                o.Parallelism.Should().Be(5);
            }).WithNotParsed(o => Assert.Fail("Could not Parse Options"));
        }

        [Test]
        public void VerifyUtterancesIsRequired()
        {
            var args = this.options.ToArray();
            var parser = Parser.Default.ParseArguments<TestOptions>(args);
            parser.Tag.Should().Be(ParserResultType.NotParsed);
            var error = parser.As<NotParsed<TestOptions>>().Errors.First();
            error.As<MissingRequiredOptionError>().NameInfo.LongName.Should().Be("utterances");
        }

        [Test]
        public void SettingForTestOptionsWhenNotSet()
        {
            this.options.Add("-u");
            this.options.Add("utterances");
            var args = this.options.ToArray();
            var parser = Parser.Default.ParseArguments<TestOptions>(args)
                .WithParsed(o =>
                {
                    o.UtterancesPath.Should().Be("utterances");
                    o.SettingsPath.Should().Be(null);
                    o.SpeechFilesDirectory.Should().Be(null);
                    o.TranscriptionsFile.Should().Be(null);
                    o.OutputPath.Should().Be(null);
                    o.Parallelism.Should().Be(3);
                })
                .WithNotParsed(o => Assert.Fail("Could not Parse Options"));
        }

        [Test]
        public void SettingforParallelismHelp()
        {
            var helpWriter = new StringWriter();
            var parser = new Parser(config => config.HelpWriter = helpWriter)
            .ParseArguments<TestOptions>(new[] { "--help" });

            var text = helpWriter.GetStringBuilder().ToString();

            text.Should().ContainAll("-p", "--parallelism");
        }
    }
}


<|start_filename|>src/NLU.DevOps.Luis.Tests.Shared/LuisNLUClientFactoryTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Luis.Tests
{
    using System;
    using FluentAssertions;
    using NUnit.Framework;

    [TestFixture]
    internal static class LuisNLUClientFactoryTests
    {
        [Test]
        public static void ThrowsArgumentNull()
        {
            var factory = new LuisNLUClientFactory();
            Action nullTrainConfiguration = () => factory.CreateTrainInstance(null, string.Empty);
            Action nullTestConfiguration = () => factory.CreateTestInstance(null, string.Empty);
            nullTrainConfiguration.Should().Throw<ArgumentNullException>().And.ParamName.Should().Be("configuration");
            nullTestConfiguration.Should().Throw<ArgumentNullException>().And.ParamName.Should().Be("configuration");
        }
    }
}


<|start_filename|>src/NLU.DevOps.MockProvider/MockNLUClientFactory.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.MockProvider
{
    using System.Composition;
    using Microsoft.Extensions.Configuration;
    using Models;

    [Export("mock", typeof(INLUClientFactory))]
    internal class MockNLUClientFactory : INLUClientFactory
    {
        public INLUTrainClient CreateTrainInstance(IConfiguration configuration, string settingsPath)
        {
            return new MockNLUClient(configuration["trainedUtterances"]);
        }

        public INLUTestClient CreateTestInstance(IConfiguration configuration, string settingsPath)
        {
            return new MockNLUClient(configuration["trainedUtterances"]);
        }
    }
}


<|start_filename|>src/NLU.DevOps.ModelPerformance/ConfusionMatrixTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.ModelPerformance
{
    using NUnit.Framework;

    [TestFixture]
    [Parallelizable(ParallelScope.All)]
    internal static class ConfusionMatrixTests
    {
        [Test]
        [TestCaseSource(typeof(TestCaseSource), "PassingTests")]
        public static void Pass(string because)
        {
            Assert.Pass(because);
        }

        [Test]
        [TestCaseSource(typeof(TestCaseSource), "FailingTests")]
        public static void Fail(string because)
        {
            Assert.Fail(because);
        }
    }
}


<|start_filename|>src/NLU.DevOps.Lex/ILexTestClient.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Lex
{
    using System;
    using System.Threading;
    using System.Threading.Tasks;
    using Amazon.Lex.Model;

    /// <summary>
    /// Lex client interface for testing operations.
    /// </summary>
    public interface ILexTestClient : IDisposable
    {
        /// <summary>
        /// Calls the 'PostContent' action on Lex.
        /// </summary>
        /// <returns>Task to await the bot response.</returns>
        /// <param name="request">Request.</param>
        /// <param name="cancellationToken">Cancellation token.</param>
        Task<PostContentResponse> PostContentAsync(PostContentRequest request, CancellationToken cancellationToken);

        /// <summary>
        /// Calls the 'PostText' action on Lex.
        /// </summary>
        /// <returns>Task to await the bot response.</returns>
        /// <param name="request">Request.</param>
        /// <param name="cancellationToken">Cancellation token.</param>
        Task<PostTextResponse> PostTextAsync(PostTextRequest request, CancellationToken cancellationToken);
    }
}


<|start_filename|>src/NLU.DevOps.Luis.Shared/JSONEntityWithRole.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Luis
{
    using Microsoft.Azure.CognitiveServices.Language.LUIS.Authoring.Models;
    using Newtonsoft.Json;

    /// <summary>
    /// <see cref="JSONEntity"/> with additional role property.
    /// </summary>
    public class JSONEntityWithRole : JSONEntity
    {
        /// <summary>
        /// Initializes a new instance of the <see cref="JSONEntityWithRole"/> class.
        /// </summary>
        /// <param name="startPos">Start position of the entity.</param>
        /// <param name="endPos">End position of the entity.</param>
        /// <param name="entity">Entity type.</param>
        /// <param name="role">Entity role.</param>
        public JSONEntityWithRole(int startPos, int endPos, string entity, string role)
            : base(startPos, endPos, entity)
        {
            this.Role = role;
        }

        /// <summary>
        /// Gets or sets the role for the entity.
        /// </summary>
        /// <remarks>
        /// Do not change the setter to private, as this will prevent the
        /// LUIS client serializer from serializing this property.
        /// </remarks>
        [JsonProperty("role")]
        public string Role { get; set; }
    }
}


<|start_filename|>src/NLU.DevOps.Luis.Tests.Shared/TestLuisConfigurationTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Luis.Tests
{
    using System;
    using FluentAssertions;
    using Microsoft.Extensions.Configuration;
    using NUnit.Framework;

    [TestFixture]
    internal static class TestLuisConfigurationTests
    {
        [Test]
        public static void MissingAppIdThrowsInvalidOperation()
        {
            var luisConfiguration = new TestLuisConfiguration(new ConfigurationBuilder().Build());
            Func<string> missingAppId = () => luisConfiguration.AppId;
            missingAppId.Should().Throw<InvalidOperationException>().And.Message.Should().Contain("luisAppId");
        }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine.Tests/SerializerTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Tests
{
    using System.IO;
    using System.Linq;
    using FluentAssertions;
    using Newtonsoft.Json.Linq;
    using NUnit.Framework;

    [TestFixture]
    internal class SerializerTests
    {
        [Test]
        public void WriteTrucatesFile()
        {
            var path = Path.GetRandomFileName();
            File.WriteAllText(path, string.Concat(string.Empty, Enumerable.Repeat("!", 1000)));
            Serializer.Write(path, 42);
            var content = File.ReadAllText(path);
            var json = JToken.Parse(content);
            json.Type.Should().Be(JTokenType.Integer);
            json.Value<int>().Should().Be(42);
        }
    }
}


<|start_filename|>src/NLU.DevOps.Dialogflow.Tests/DialogflowNLUTestClientFactoryTests.cs<|end_filename|>
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Dialogflow.Tests
{
    using System;
    using FluentAssertions;
    using Microsoft.Extensions.Configuration;
    using NUnit.Framework;

    [TestFixture]
    internal static class DialogflowNLUTestClientFactoryTests
    {
        [Test]
        public static void ThrowsArgumentNull()
        {
            var factory = new DialogflowNLUClientFactory();
            Action nullConfiguration = () => factory.CreateTestInstance(null, null);
            nullConfiguration.Should().Throw<ArgumentNullException>().And.ParamName.Should().Be("configuration");
        }

        [Test]
        public static void CreateTrainInstanceShouldThrowNotSupported()
        {
            var factory = new DialogflowNLUClientFactory();
            Action createTrainInstance = () => factory.CreateTrainInstance(null, null);
            createTrainInstance.Should().Throw<NotSupportedException>();
        }

        [Test]
        public static void CreatesTestInstance()
        {
            var factory = new DialogflowNLUClientFactory();
            var configuration = new ConfigurationBuilder().Build();
            using (var client = factory.CreateTestInstance(configuration, null))
            {
                client.Should().NotBeNull();
            }
        }
    }
}

<|start_filename|>src/NLU.DevOps.Dialogflow/DialogflowNLUClientFactory.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Dialogflow
{
    using System;
    using System.Composition;
    using Microsoft.Extensions.Configuration;
    using Models;

    /// <summary>
    /// Dialogflow NLU client factory.
    /// </summary>
    [Export("dialogflow", typeof(INLUClientFactory))]
    public class DialogflowNLUClientFactory : INLUClientFactory
    {
        /// <inheritdoc />
        public INLUTestClient CreateTestInstance(IConfiguration configuration, string settingsPath)
        {
            return new DialogflowNLUTestClient(configuration ?? throw new ArgumentNullException(nameof(configuration)));
        }

        /// <inheritdoc />
        public INLUTrainClient CreateTrainInstance(IConfiguration configuration, string settingsPath)
        {
            throw new NotSupportedException("The Dialogflow NLU.DevOps package does not currently support training.");
        }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine.Tests/Train/TrainCommandMock.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Tests.Train
{
    using CommandLine.Train;
    using Models;

    internal class TrainCommandMock : TrainCommand
    {
        public TrainCommandMock(TrainOptions options)
            : base(options)
        {
        }

        protected override INLUTrainClient CreateNLUTrainClient() =>
            new MockNLUTrainClient();
    }
}


<|start_filename|>src/NLU.DevOps.Core/INLUClientFactory.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Models
{
    using Microsoft.Extensions.Configuration;

    /// <summary>
    /// NLU client factory.
    /// </summary>
    public interface INLUClientFactory
    {
        /// <summary>
        /// Create the NLU training client instance.
        /// </summary>
        /// <returns>The instance.</returns>
        /// <param name="configuration">Configuration.</param>
        /// <param name="settingsPath">Path to NLU provider settings.</param>
        INLUTrainClient CreateTrainInstance(IConfiguration configuration, string settingsPath);

        /// <summary>
        /// Create the NLU testing client instance.
        /// </summary>
        /// <returns>The instance.</returns>
        /// <param name="configuration">Configuration.</param>
        /// <param name="settingsPath">Path to NLU provider settings.</param>
        INLUTestClient CreateTestInstance(IConfiguration configuration, string settingsPath);
    }
}


<|start_filename|>src/NLU.DevOps.LuisV3/SpeechPredictionResponse.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Luis
{
    using Microsoft.Azure.CognitiveServices.Language.LUIS.Runtime.Models;

    /// <summary>
    /// LUIS response including speech confidence score.
    /// </summary>
    public class SpeechPredictionResponse
    {
        /// <summary>
        /// Initializes a new instance of the <see cref="SpeechPredictionResponse"/> class.
        /// </summary>
        /// <param name="predictionResponse">LUIS prediction response.</param>
        /// <param name="textScore">Text score.</param>
        public SpeechPredictionResponse(PredictionResponse predictionResponse, double? textScore)
        {
            this.PredictionResponse = predictionResponse;
            this.TextScore = textScore;
        }

        /// <summary>
        /// Gets the LUIS prediction response.
        /// </summary>
        public PredictionResponse PredictionResponse { get; }

        /// <summary>
        /// Gets the text score.
        /// </summary>
        public double? TextScore { get; }
    }
}

<|start_filename|>src/NLU.DevOps.Core/IJsonExtension.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Core
{
    using System.Collections.Generic;
    using Newtonsoft.Json;

    /// <summary>
    /// Internal interface for manipulating properties from <see cref="JsonExtensionDataAttribute"/>.
    /// </summary>
    internal interface IJsonExtension
    {
        /// <summary>
        /// Gets the additional properties.
        /// </summary>
        IDictionary<string, object> AdditionalProperties { get; }
    }
}


<|start_filename|>src/NLU.DevOps.Luis/SpeechLuisResult.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Luis
{
    using Microsoft.Azure.CognitiveServices.Language.LUIS.Runtime.Models;

    /// <summary>
    /// LUIS response including speech confidence score.
    /// </summary>
    public class SpeechLuisResult
    {
        /// <summary>
        /// Initializes a new instance of the <see cref="SpeechLuisResult"/> class.
        /// </summary>
        /// <param name="luisResult">Luis result.</param>
        /// <param name="textScore">Text score.</param>
        public SpeechLuisResult(LuisResult luisResult, double? textScore)
        {
            this.LuisResult = luisResult;
            this.TextScore = textScore;
        }

        /// <summary>
        /// Gets the LUIS result.
        /// </summary>
        public LuisResult LuisResult { get; }

        /// <summary>
        /// Gets the text score.
        /// </summary>
        public double? TextScore { get; }
    }
}

<|start_filename|>src/NLU.DevOps.Lex/EntityType.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Lex
{
    using Newtonsoft.Json.Linq;

    /// <summary>
    /// Entity types class.
    /// </summary>
    public class EntityType
    {
        /// <summary>
        /// Initializes a new instance of the <see cref="EntityType"/> class.
        /// </summary>
        /// <param name="name">Entity type name.</param>
        /// <param name="kind">Entity type kind.</param>
        /// <param name="data">Entity type data.</param>
        public EntityType(string name, string kind, JToken data)
        {
            this.Name = name;
            this.Kind = kind;
            this.Data = data;
        }

        /// <summary>
        /// Gets the entity type name.
        /// </summary>
        public string Name { get; }

        /// <summary>
        /// Gets the entity type kind.
        /// </summary>
        public string Kind { get; }

        /// <summary>
        /// Gets the entity type data.
        /// </summary>
        public JToken Data { get; }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine/BaseOptions.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine
{
    using global::CommandLine;

    internal class BaseOptions
    {
        [Option('s', "service", HelpText = "NLU service to run against. One of 'lex' or 'luis'.", Required = true)]
        public string Service { get; set; }

        [Option('q', "quiet", HelpText = "Suppress log output.", Required = false)]
        public bool Quiet { get; set; }

        [Option('v', "verbose", HelpText = "Verbose log output.", Required = false)]
        public bool Verbose { get; set; }

        [Option('i', "include", HelpText = "Path to search for NLU.DevOps.CommandLine extensions.", Required = false)]
        public string IncludePath { get; set; }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine.Tests/BaseCommandTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Tests
{
    using System.Collections.Generic;
    using System.Linq;
    using CommandLine.Train;
    using FluentAssertions;
    using global::CommandLine;
    using Microsoft.Extensions.Logging;
    using NUnit.Framework;

    /// <summary>
    /// Unit tests for <see cref="BaseCommand{T}"/>
    /// </summary>
    /// <remarks>
    /// Known bug with Microsoft.Extesions.Logging.Console where lowest log level setting must be run first.
    /// </remarks>
    [TestFixture]
    internal class BaseCommandTests
    {
        private List<string> options;

        [SetUp]
        public void SetUp()
        {
            this.options = new List<string>
            {
                "-s",
                "luis",
                "-i",
                "../"
            };
        }

        [Test]
        public void WhenBaseOptionsAreSetACommandIsReturned()
        {
            var args = this.options.ToArray();
            var results = Parser.Default.ParseArguments<TrainOptions>(args);
            results.Tag.Should().Be(ParserResultType.Parsed);
        }

        [Test]
        public void WhenInvalidOptionAppearsParsingFails()
        {
            this.options.Add("-z");
            var args = this.options.ToArray();
            var results = Parser.Default.ParseArguments<TrainOptions>(args);
            results.Tag.Should().Be(ParserResultType.NotParsed);
            results.As<NotParsed<TrainOptions>>().Errors.First().Tag.Should().Be(ErrorType.UnknownOptionError);
        }

        [Test]
        [Order(1)]
        public void WhenLoggingOptionsAreSetToQuietLogLevelIsWarning()
        {
            this.options.Add("-q");
            var args = this.options.ToArray();
            var results = Parser.Default.ParseArguments<BaseOptions>(args);
            var logOptions = results.As<Parsed<BaseOptions>>();
            var command = new BaseCommandMock(logOptions.Value);
            var logger = command.Logger;
            logger.IsEnabled(LogLevel.Information).Should().BeFalse();
            logger.IsEnabled(LogLevel.Warning).Should().BeTrue();
        }

        [Test]
        [Order(2)]
        public void WhenLoggingOptionsAreNotSetLogLevelIsInformation()
        {
            var args = this.options.ToArray();
            var results = Parser.Default.ParseArguments<BaseOptions>(args);
            var logOptions = results.As<Parsed<BaseOptions>>();
            var command = new BaseCommandMock(logOptions.Value);
            var logger = command.Logger;
            logger.IsEnabled(LogLevel.Debug).Should().BeFalse();
            logger.IsEnabled(LogLevel.Information).Should().BeTrue();
        }

        [Test]
        [Order(3)]
        public void WhenLoggingOptionsAreSetToVerboseLogLevelIsTrace()
        {
            this.options.Add("-v");
            var args = this.options.ToArray();
            var results = Parser.Default.ParseArguments<BaseOptions>(args);
            var logOptions = results.As<Parsed<BaseOptions>>();
            var command = new BaseCommandMock(logOptions.Value);
            var logger = command.Logger;
            logger.IsEnabled(LogLevel.Trace).Should().BeTrue();
        }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine.Tests/NLUServiceFactoryTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Tests
{
    using System;
    using CommandLine.Train;
    using FluentAssertions;
    using global::CommandLine;
    using Microsoft.Extensions.Configuration;
    using NUnit.Framework;

    [TestFixture]
    internal static class NLUServiceFactoryTests
    {
        [Test]
        public static void SettingUnknownServiceThrowsException()
        {
            var options = default(BaseOptions);
            var args = new[]
            {
                "nlu",
                "command",
                "-s",
                "foo"
            };
            var parser = Parser.Default.ParseArguments<TrainOptions>(args).WithParsed<TrainOptions>(o =>
            {
                options = o;
            });

            IConfiguration configuration = new ConfigurationBuilder()
                .Build();

            Action createTrainInstance = () => NLUClientFactory.CreateTrainInstance(options, configuration);
            createTrainInstance.Should().Throw<InvalidOperationException>().WithMessage("Invalid NLU provider type 'foo'.");
            Action createTestInstance = () => NLUClientFactory.CreateTestInstance(options, configuration);
            createTestInstance.Should().Throw<InvalidOperationException>().WithMessage("Invalid NLU provider type 'foo'.");
        }
    }
}


<|start_filename|>src/NLU.DevOps.Luis.Shared/TestLuisConfiguration.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Luis
{
    using Microsoft.Extensions.Configuration;

    /// <summary>
    /// LUIS configuration for testing.
    /// </summary>
    public sealed class TestLuisConfiguration : LuisConfiguration
    {
        /// <summary>
        /// Initializes a new instance of the <see cref="TestLuisConfiguration"/> class.
        /// </summary>
        /// <param name="configuration">Configuration.</param>
        public TestLuisConfiguration(IConfiguration configuration)
            : base(configuration)
        {
        }

        /// <inheritdoc />
        public override string AppId => this.EnsureConfigurationString(LuisAppIdConfigurationKey);
    }
}


<|start_filename|>src/NLU.DevOps.Core/DefaultQuery.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.Core
{
    using System;

    /// <summary>
    /// Default NLU test query.
    /// </summary>
    public class DefaultQuery
    {
        /// <summary>
        /// Initializes a new instance of the <see cref="DefaultQuery"/> class.
        /// </summary>
        /// <param name="text">Utterance text.</param>
        public DefaultQuery(string text)
        {
            this.Text = text ?? throw new ArgumentNullException(nameof(text));
        }

        /// <summary>
        /// Gets the text.
        /// </summary>
        public string Text { get; }
    }
}


<|start_filename|>src/NLU.DevOps.ModelPerformance/ComparisonTargetKind.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.ModelPerformance
{
    /// <summary>
    /// Comparison target kind.
    /// </summary>
    public enum ComparisonTargetKind
    {
        /// <summary>
        /// Default <see cref="ComparisonTargetKind"/>.
        /// </summary>
        None,

        /// <summary>
        /// Text.
        /// </summary>
        Text,

        /// <summary>
        /// Intent.
        /// </summary>
        Intent,

        /// <summary>
        /// Entity.
        /// </summary>
        Entity,

        /// <summary>
        /// Entity value.
        /// </summary>
        EntityValue
    }
}


<|start_filename|>src/NLU.DevOps.Models/GlobalSuppressions.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA1040:Avoid empty interfaces", Justification = "The interface is used to identify a set of types at compile time.", Scope = "type", Target = "~T:NLU.DevOps.Models.INLUQuery")]


<|start_filename|>src/NLU.DevOps.CommandLine.Tests/BaseOptionsTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Tests
{
    using System;
    using System.Collections.Generic;
    using System.Linq;
    using CommandLine.Train;
    using FluentAssertions;
    using global::CommandLine;
    using NUnit.Framework;

    [TestFixture]
    internal static class BaseOptionsTests
    {
        [Test]
        public static void VerifyServiceMissingInBaseOptionsRaisesError()
        {
            var args = Array.Empty<string>();
            var parser = Parser.Default.ParseArguments<TrainOptions>(args);
            parser.Tag.Should().Be(ParserResultType.NotParsed);
            var error = parser.As<NotParsed<TrainOptions>>().Errors.First();
            error.As<MissingRequiredOptionError>().NameInfo.LongName.Should().Be("service");
        }

        [Test]
        public static void VerifyServiceExistsInBaseDoesNotRaisesError()
        {
            var options = new List<string>();
            options.Add("-s");
            options.Add("luis");
            var args = options.ToArray();
            var parser = Parser.Default.ParseArguments<TrainOptions>(args)
                .WithParsed(o =>
                {
                    o.Service.Should().Be("luis");
                })
                .WithNotParsed(o => Assert.Fail("Could not Parse Options"));
        }

        [Test]
        public static void VerifyAllSettingsCanBeSet()
        {
            var options = new List<string>();
            options.Add("-s");
            options.Add("service");
            options.Add("-q");
            options.Add("true");
            options.Add("-v");
            options.Add("true");
            options.Add("-i");
            options.Add("includepath");
            var args = options.ToArray();
            var parser = Parser.Default.ParseArguments<TrainOptions>(args)
                .WithParsed(o =>
                {
                    o.Service.Should().Be("service");
                    o.Quiet.Should().BeTrue();
                    o.Verbose.Should().BeTrue();
                    o.IncludePath.Should().Be("includepath");
                })
                .WithNotParsed(o => Assert.Fail("Could not Parse Options"));
        }
    }
}


<|start_filename|>src/NLU.DevOps.CommandLine.Tests/Train/TrainOptionsTests.cs<|end_filename|>
﻿// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

namespace NLU.DevOps.CommandLine.Tests.Train
{
    using System.Collections.Generic;
    using FluentAssertions;
    using global::CommandLine;
    using NLU.DevOps.CommandLine.Train;
    using NUnit.Framework;

    [TestFixture]
    internal class TrainOptionsTests
    {
        private List<string> options;

        [SetUp]
        public void SetUp()
        {
            this.options = new List<string>();
            this.options.Add("-s");
            this.options.Add("luis");
        }

        [Test]
        public void SettingForTrainOptionsWhenSet()
        {
            this.options.Add("-u");
            this.options.Add("utterances");
            this.options.Add("-m");
            this.options.Add("settings");
            this.options.Add("-a");
            var args = this.options.ToArray();
            var parser = Parser.Default.ParseArguments<TrainOptions>(args)
                .WithParsed(o =>
                {
                    o.UtterancesPath.Should().Be("utterances");
                    o.SettingsPath.Should().Be("settings");
                    o.SaveAppSettings.Should().BeTrue();
                })
                .WithNotParsed(o => Assert.Fail("Could not Parse Options"));
        }

        [Test]
        public void SettingForTrainOptionsWhenNotSet()
        {
            var args = this.options.ToArray();
            var parser = Parser.Default.ParseArguments<TrainOptions>(args)
                .WithParsed(o =>
                {
                    o.UtterancesPath.Should().Be(null);
                    o.SettingsPath.Should().Be(null);
                    o.SaveAppSettings.Should().BeFalse();
                })
                .WithNotParsed(o => Assert.Fail("Could not Parse Options"));
        }
    }
}


<|start_filename|>extensions/tasks/NLUTrainV0/package.json<|end_filename|>
{
  "dependencies": {
    "azure-pipelines-task-lib": "~2.8.0",
    "nlu-devops-common": "file:../../nlu-devops-common-0.1.5.tgz"
  }
}
