text:
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Composition;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.Editor.Shared.Utilities;
using Microsoft.CodeAnalysis.ErrorReporting;
using Microsoft.CodeAnalysis.Host.Mef;
using Microsoft.CodeAnalysis.Notification;
using Microsoft.CodeAnalysis.Packaging;
using Microsoft.CodeAnalysis.PooledObjects;
using Microsoft.CodeAnalysis.Shared.TestHooks;
using Microsoft.CodeAnalysis.Shared.Utilities;
using Microsoft.CodeAnalysis.SymbolSearch;
using Microsoft.VisualStudio.Editor;
using Microsoft.VisualStudio.LanguageServices.Implementation.ProjectSystem;
using Microsoft.VisualStudio.LanguageServices.SymbolSearch;
using Microsoft.VisualStudio.LanguageServices.Utilities;
using Microsoft.VisualStudio.Shell.Interop;
using Microsoft.VisualStudio.Shell.ServiceBroker;
using Microsoft.VisualStudio.Threading;
using NuGet.VisualStudio;
using NuGet.VisualStudio.Contracts;
using Roslyn.Utilities;
using SVsServiceProvider = Microsoft.VisualStudio.Shell.SVsServiceProvider;

namespace Microsoft.VisualStudio.LanguageServices.Packaging
{
    using Workspace = Microsoft.CodeAnalysis.Workspace;

    /// <summary>
    /// Free threaded wrapper around the NuGet.VisualStudio STA package installer interfaces.
    /// We want to be able to make queries about packages from any thread.  For example, the
    /// add-NuGet-reference feature wants to know what packages a project already has
    /// references to.  NuGet.VisualStudio provides this information, but only in a COM STA
    /// manner.  As we don't want our background work to bounce and block on the UI thread
    /// we have this helper class which queries the information on the UI thread and caches
    /// the data so it can be read from the background.
    /// </summary>
    [ExportWorkspaceService(typeof(IPackageInstallerService)), Shared]
    internal partial class PackageInstallerService : AbstractDelayStartedService, IPackageInstallerService, IVsSearchProviderCallback
    {
        private readonly VisualStudioWorkspaceImpl _workspace;
        private readonly SVsServiceProvider _serviceProvider;
        private readonly Shell.IAsyncServiceProvider _asyncServiceProvider;
        private readonly IVsEditorAdaptersFactoryService _editorAdaptersFactoryService;

        private readonly Lazy<IVsPackageInstallerServices>? _packageInstallerServices;
        private readonly Lazy<IVsPackageInstaller2>? _packageInstaller;
        private readonly Lazy<IVsPackageUninstaller>? _packageUninstaller;
        private readonly Lazy<IVsPackageSourceProvider>? _packageSourceProvider;

        private IVsPackage? _nugetPackageManager;

        /// <summary>
        /// Used to keep track of what types of changes we've seen so we can then determine what to refresh on the UI
        /// thread.  If we hear about project changes, we only refresh that project.  If we hear about a solution level
        /// change, we'll refresh all projects.
        /// </summary>
        /// <remarks>
        /// <c>solutionChanged == true iff changedProject == null</c> and <c>solutionChanged == false iff changedProject
        /// != null</c>. So technically having both values is redundant.  However, i like the clarity of having both.
        /// </remarks>
        private readonly AsyncBatchingWorkQueue<(bool solutionChanged, ProjectId? changedProject)>? _workQueue;

        private readonly ConcurrentDictionary<ProjectId, ProjectState> _projectToInstalledPackageAndVersion =
            new();

        /// <summary>
        /// Lock used to protect reads and writes of <see cref="_packageSourcesTask"/>.
        /// </summary>
        private readonly object _gate = new();

        /// <summary>
        /// Task uses to compute the set of package sources on demand when asked the first time.  The value will be
        /// computed and cached in the task.  When this value changes, the task will simply be cleared out.
        /// </summary>
        private Task<ImmutableArray<PackageSource>>? _packageSourcesTask;

        [ImportingConstructor]
        [Obsolete(MefConstruction.ImportingConstructorMessage, error: true)]
        public PackageInstallerService(
            IThreadingContext threadingContext,
            IAsynchronousOperationListenerProvider listenerProvider,
            VisualStudioWorkspaceImpl workspace,
            SVsServiceProvider serviceProvider,
            [Import("Microsoft.VisualStudio.Shell.Interop.SAsyncServiceProvider")] object asyncServiceProvider,
            IVsEditorAdaptersFactoryService editorAdaptersFactoryService,
            [Import(AllowDefault = true)] Lazy<IVsPackageInstallerServices>? packageInstallerServices,
            [Import(AllowDefault = true)] Lazy<IVsPackageInstaller2>? packageInstaller,
            [Import(AllowDefault = true)] Lazy<IVsPackageUninstaller>? packageUninstaller,
            [Import(AllowDefault = true)] Lazy<IVsPackageSourceProvider>? packageSourceProvider)
            : base(threadingContext,
                   workspace,
                   SymbolSearchOptions.Enabled,
                   SymbolSearchOptions.SuggestForTypesInReferenceAssemblies,
                   SymbolSearchOptions.SuggestForTypesInNuGetPackages)
        {
            _workspace = workspace;

            _serviceProvider = serviceProvider;
            // MEFv2 doesn't support type based contract for Import above and for this particular contract
            // (SAsyncServiceProvider) actual type cast doesn't work. (https://github.com/microsoft/vs-mef/issues/138)
            // workaround by getting the service as object and cast to actual interface
            _asyncServiceProvider = (Shell.IAsyncServiceProvider)asyncServiceProvider;

            _editorAdaptersFactoryService = editorAdaptersFactoryService;
            _packageInstallerServices = packageInstallerServices;
            _packageInstaller = packageInstaller;
            _packageUninstaller = packageUninstaller;
            _packageSourceProvider = packageSourceProvider;

            // Setup the work queue to allow us to hear about flurries of changes and then respond to them in batches
            // every second.  Note: we pass in EqualityComparer<...>.Default since we don't care about ordering, and
            // since once we hear about changes to a project (or the whole solution), we don't need to keep track if we
            // hear about the same thing in that batch window interval.
            _workQueue = new AsyncBatchingWorkQueue<(bool solutionChanged, ProjectId? changedProject)>(
                TimeSpan.FromSeconds(1),
                this.ProcessWorkQueueAsync,
                equalityComparer: EqualityComparer<(bool solutionChanged, ProjectId? changedProject)>.Default,
                listenerProvider.GetListener(FeatureAttribute.PackageInstaller),
                this.DisposalToken);
        }

        public event EventHandler? PackageSourcesChanged;

        public ImmutableArray<PackageSource> TryGetPackageSources()
        {
            Task<ImmutableArray<PackageSource>> localPackageSourcesTask;
            lock (_gate)
            {
                if (_packageSourcesTask is null)
                    _packageSourcesTask = Task.Run(() => GetPackageSourcesAsync(), this.DisposalToken);

                localPackageSourcesTask = _packageSourcesTask;
            }

            if (localPackageSourcesTask.Status == TaskStatus.RanToCompletion)
            {
                return localPackageSourcesTask.Result;
            }
            else
            {
                // The result was not available yet (or it was canceled/faulted).  Just return an empty result to
                // signify we couldn't get this right now.
                return ImmutableArray<PackageSource>.Empty;
            }
        }

        private async Task<ImmutableArray<PackageSource>> GetPackageSourcesAsync()
        {
            await ThreadingContext.JoinableTaskFactory.SwitchToMainThreadAsync(DisposalToken);

            try
            {
                if (_packageSourceProvider != null)
                    return _packageSourceProvider.Value.GetSources(includeUnOfficial: true, includeDisabled: false).SelectAsArray(r => new PackageSource(r.Key, r.Value));
            }
            catch (Exception ex) when (ex is InvalidDataException || ex is InvalidOperationException)
            {
                // These exceptions can happen when the nuget.config file is broken.
            }
            catch (ArgumentException ae) when (FatalError.ReportAndCatch(ae))
            {
                // This exception can happen when the nuget.config file is broken, e.g. invalid credentials.
                // https://github.com/dotnet/roslyn/issues/40857
            }

            return ImmutableArray<PackageSource>.Empty;
        }

        [MemberNotNullWhen(true, nameof(_packageInstallerServices))]
        [MemberNotNullWhen(true, nameof(_packageInstaller))]
        [MemberNotNullWhen(true, nameof(_packageUninstaller))]
        [MemberNotNullWhen(true, nameof(_packageSourceProvider))]
        private bool IsEnabled
        {
            get
            {
                return _packageInstallerServices != null
                    && _packageInstaller != null
                    && _packageUninstaller != null
                    && _packageSourceProvider != null;
            }
        }

        bool IPackageInstallerService.IsEnabled(ProjectId projectId)
        {
            if (!IsEnabled)
            {
                return false;
            }

            if (_projectToInstalledPackageAndVersion.TryGetValue(projectId, out var state))
            {
                return state.IsEnabled;
            }

            // If we haven't scanned the project yet, assume that we're available for it.
            return true;
        }

        private async ValueTask<IVsPackageSourceProvider> GetPackageSourceProviderAsync()
        {
            Contract.ThrowIfFalse(IsEnabled);

            if (!_packageSourceProvider.IsValueCreated)
            {
                // Switch to background thread for known assembly load
                await TaskScheduler.Default;
            }

            return _packageSourceProvider.Value;
        }

        protected override async Task EnableServiceAsync(CancellationToken cancellationToken)
        {
            if (!IsEnabled)
            {
                return;
            }

            // Continue on captured context since EnableServiceAsync is part of a UI thread initialization sequence
            var packageSourceProvider = await GetPackageSourceProviderAsync().ConfigureAwait(true);

            // Start listening to additional events workspace changes.
            _workspace.WorkspaceChanged += OnWorkspaceChanged;
            packageSourceProvider.SourcesChanged += OnSourceProviderSourcesChanged;
        }

        protected override void StartWorking()
        {
            this.AssertIsForeground();

            if (!this.IsEnabled)
            {
                return;
            }

            OnSourceProviderSourcesChanged(this, EventArgs.Empty);

            Contract.ThrowIfNull(_workQueue, "We should only be called after EnableService is called");

            // Kick off an initial set of work that will analyze the entire solution.
            _workQueue.AddWork((solutionChanged: true, changedProject: null));
        }

        private void OnSourceProviderSourcesChanged(object sender, EventArgs e)
        {
            lock (_gate)
            {
                // If the existing _packageSourcesTask is null, that means no one has asked us about package sources
                // yet.  So no need for us to do anything if that's true.  We'll just continue waiting until first
                // asked.  However, if it's not null, that means we have already been asked.  In that case, proactively
                // get the new set of sources so they're ready for the next time we're asked.
                if (_packageSourcesTask != null)
                    _packageSourcesTask = Task.Run(() => GetPackageSourcesAsync(), this.DisposalToken);
            }

            PackageSourcesChanged?.Invoke(this, EventArgs.Empty);
        }

        public bool TryInstallPackage(
            Workspace workspace,
            DocumentId documentId,
            string source,
            string packageName,
            string versionOpt,
            bool includePrerelease,
            CancellationToken cancellationToken)
        {
            this.AssertIsForeground();

            // The 'workspace == _workspace' line is probably not necessary. However, we include
            // it just to make sure that someone isn't trying to install a package into a workspace
            // other than the VisualStudioWorkspace.
            if (workspace == _workspace && _workspace != null && IsEnabled)
            {
                var projectId = documentId.ProjectId;
                var dte = (EnvDTE.DTE)_serviceProvider.GetService(typeof(SDTE));
                var dteProject = _workspace.TryGetDTEProject(projectId);
                if (dteProject != null)
                {
                    var undoManager = _editorAdaptersFactoryService.TryGetUndoManager(
                        workspace, documentId, cancellationToken);

                    return TryInstallAndAddUndoAction(
                        source, packageName, versionOpt, includePrerelease, dte, dteProject, undoManager);
                }
            }

            return false;
        }

        private bool TryInstallPackage(
            string source,
            string packageName,
            string versionOpt,
            bool includePrerelease,
            EnvDTE.DTE dte,
            EnvDTE.Project dteProject)
        {
            this.AssertIsForeground();
            Contract.ThrowIfFalse(IsEnabled);

            try
            {
                if (!_packageInstallerServices.Value.IsPackageInstalled(dteProject, packageName))
                {
                    dte.StatusBar.Text = string.Format(ServicesVSResources.Installing_0, packageName);

                    if (versionOpt == null)
                    {
                        _packageInstaller.Value.InstallLatestPackage(
                            source, dteProject, packageName, includePrerelease, ignoreDependencies: false);
                    }
                    else
                    {
                        _packageInstaller.Value.InstallPackage(
                            source, dteProject, packageName, versionOpt, ignoreDependencies: false);
                    }

                    var installedVersion = GetInstalledVersion(packageName, dteProject);
                    dte.StatusBar.Text = string.Format(ServicesVSResources.Installing_0_completed,
                        GetStatusBarText(packageName, installedVersion));

                    return true;
                }

                // fall through.
            }
            catch (Exception e) when (FatalError.ReportAndCatch(e))
            {
                dte.StatusBar.Text = string.Format(ServicesVSResources.Package_install_failed_colon_0, e.Message);

                var notificationService = _workspace.Services.GetService<INotificationService>();
                notificationService?.SendNotification(
                    string.Format(ServicesVSResources.Installing_0_failed_Additional_information_colon_1, packageName, e.Message),
                    severity: NotificationSeverity.Error);

                // fall through.
            }

            return false;
        }

        private static string GetStatusBarText(string packageName, string? installedVersion)
            => installedVersion == null ? packageName : $"{packageName} - {installedVersion}";

        private bool TryUninstallPackage(
            string packageName, EnvDTE.DTE dte, EnvDTE.Project dteProject)
        {
            this.AssertIsForeground();
            Contract.ThrowIfFalse(IsEnabled);

            try
            {
                if (_packageInstallerServices.Value.IsPackageInstalled(dteProject, packageName))
                {
                    dte.StatusBar.Text = string.Format(ServicesVSResources.Uninstalling_0, packageName);
                    var installedVersion = GetInstalledVersion(packageName, dteProject);
                    _packageUninstaller.Value.UninstallPackage(dteProject, packageName, removeDependencies: true);

                    dte.StatusBar.Text = string.Format(ServicesVSResources.Uninstalling_0_completed,
                        GetStatusBarText(packageName, installedVersion));

                    return true;
                }

                // fall through.
            }
            catch (Exception e) when (FatalError.ReportAndCatch(e))
            {
                dte.StatusBar.Text = string.Format(ServicesVSResources.Package_uninstall_failed_colon_0, e.Message);

                var notificationService = _workspace.Services.GetService<INotificationService>();
                notificationService?.SendNotification(
                    string.Format(ServicesVSResources.Uninstalling_0_failed_Additional_information_colon_1, packageName, e.Message),
                    severity: NotificationSeverity.Error);

                // fall through.
            }

            return false;
        }

        private string? GetInstalledVersion(string packageName, EnvDTE.Project dteProject)
        {
            this.AssertIsForeground();
            Contract.ThrowIfFalse(IsEnabled);

            try
            {
                var installedPackages = _packageInstallerServices.Value.GetInstalledPackages(dteProject);
                var metadata = installedPackages.FirstOrDefault(m => m.Id == packageName);
                return metadata?.VersionString;
            }
            catch (Exception e) when (FatalError.ReportAndCatch(e))
            {
            }

            return null;
        }

        private void OnWorkspaceChanged(object sender, WorkspaceChangeEventArgs e)
        {
            ThisCanBeCalledOnAnyThread();

            var solutionChanged = false;
            ProjectId? changedProject = null;
            switch (e.Kind)
            {
                default:
                    // Nothing to do for any other events.
                    return;

                case WorkspaceChangeKind.ProjectAdded:
                case WorkspaceChangeKind.ProjectChanged:
                case WorkspaceChangeKind.ProjectReloaded:
                case WorkspaceChangeKind.ProjectRemoved:
                    changedProject = e.ProjectId;
                    break;

                case WorkspaceChangeKind.SolutionAdded:
                case WorkspaceChangeKind.SolutionChanged:
                case WorkspaceChangeKind.SolutionCleared:
                case WorkspaceChangeKind.SolutionReloaded:
                case WorkspaceChangeKind.SolutionRemoved:
                    solutionChanged = true;
                    break;
            }

            Contract.ThrowIfNull(_workQueue, "We should only register for events after having create the WorkQueue");
            _workQueue.AddWork((solutionChanged, changedProject));
        }

        private Task ProcessWorkQueueAsync(
            ImmutableArray<(bool solutionChanged, ProjectId? changedProject)> workQueue, CancellationToken cancellationToken)
        {
            ThisCanBeCalledOnAnyThread();

            Contract.ThrowIfNull(_workQueue, "How could we be processing a workqueue change without a workqueue?");

            // If we've been disconnected, then there's no point proceeding.
            if (_workspace == null || !IsEnabled)
                return Task.CompletedTask;

            return ProcessWorkQueueWorkerAsync(workQueue, cancellationToken);
        }

        [MethodImpl(MethodImplOptions.NoInlining)]
        private async Task ProcessWorkQueueWorkerAsync(
            ImmutableArray<(bool solutionChanged, ProjectId? changedProject)> workQueue, CancellationToken cancellationToken)
        {
            ThisCanBeCalledOnAnyThread();

            var serviceContainer = (IBrokeredServiceContainer?)await _asyncServiceProvider.GetServiceAsync(typeof(SVsBrokeredServiceContainer)).ConfigureAwait(false);
            var serviceBroker = serviceContainer?.GetFullAccessServiceBroker();
            if (serviceBroker == null)
                return;

            // Make sure we are on the thread pool to avoid UI thread dependencies if external code uses ConfigureAwait(true)
            await TaskScheduler.Default;

            var nugetService = await serviceBroker.GetProxyAsync<INuGetProjectService>(NuGetServices.NuGetProjectServiceV1, cancellationToken: cancellationToken).ConfigureAwait(false);

            using (nugetService as IDisposable)
            {
                // If we didn't get a nuget service, there's nothing we can do in terms of querying the solution for
                // nuget info.
                if (nugetService == null)
                    return;

                // Figure out the entire set of projects to process.
                using var _ = PooledHashSet<ProjectId>.GetInstance(out var projectsToProcess);

                var solution = _workspace.CurrentSolution;
                AddProjectsToProcess(workQueue, solution, projectsToProcess);

                // And Process them one at a time.
                foreach (var projectId in projectsToProcess)
                {
                    cancellationToken.ThrowIfCancellationRequested();
                    await ProcessProjectChangeAsync(nugetService, solution, projectId, cancellationToken).ConfigureAwait(false);
                }
            }
        }

        private void AddProjectsToProcess(
            ImmutableArray<(bool solutionChanged, ProjectId? changedProject)> workQueue, Solution solution, HashSet<ProjectId> projectsToProcess)
        {
            ThisCanBeCalledOnAnyThread();

            // If we detected a solution change, then we need to process all projects.
            // This includes all the projects that we already know about, as well as
            // all the projects in the current workspace solution.
            if (workQueue.Any(t => t.solutionChanged))
            {
                projectsToProcess.AddRange(solution.ProjectIds);
                projectsToProcess.AddRange(_projectToInstalledPackageAndVersion.Keys);
            }
            else
            {
                projectsToProcess.AddRange(workQueue.Select(t => t.changedProject).WhereNotNull());
            }
        }

        private async Task ProcessProjectChangeAsync(
            INuGetProjectService nugetService,
            Solution solution,
            ProjectId projectId,
            CancellationToken cancellationToken)
        {
            ThisCanBeCalledOnAnyThread();

            var project = solution.GetProject(projectId);

            // We really only need to know the NuGet status for managed language projects.
            // Also, the NuGet APIs may throw on some projects that don't implement the
            // full set of DTE APIs they expect.  So we filter down to just C# and VB here
            // as we know these languages are safe to build up this index for.
            ProjectState? newState = null;

            if (project?.Language == LanguageNames.CSharp ||
                project?.Language == LanguageNames.VisualBasic)
            {
                var projectGuid = _workspace.GetProjectGuid(projectId);
                if (projectGuid != Guid.Empty)
                {
                    newState = await GetCurrentProjectStateAsync(
                        nugetService, projectGuid, cancellationToken).ConfigureAwait(false);
                }
            }

            // If we weren't able to get the nuget state for the project (i.e. it's not a c#/vb project, or we got a
            // crash attempting to get nuget information).  Mark this project as something that nuget-add-import is not
            // supported for.
            _projectToInstalledPackageAndVersion[projectId] = newState ?? ProjectState.Disabled;
        }

        private static async Task<ProjectState?> GetCurrentProjectStateAsync(
            INuGetProjectService nugetService,
            Guid projectGuid,
            CancellationToken cancellationToken)
        {
            cancellationToken.ThrowIfCancellationRequested();
            try
            {
                var installedPackagesResult = await nugetService.GetInstalledPackagesAsync(projectGuid, cancellationToken).ConfigureAwait(false);

                using var _ = PooledDictionary<string, string>.GetInstance(out var installedPackages);
                if (installedPackagesResult?.Status == InstalledPackageResultStatus.Successful)
                {
                    foreach (var installedPackage in installedPackagesResult.Packages)
                        installedPackages[installedPackage.Id] = installedPackage.Version;
                }

                return new ProjectState(installedPackages.ToImmutableDictionary());
            }
            catch (Exception e) when (FatalError.ReportAndCatchUnlessCanceled(e, cancellationToken))
            {
                return null;
            }
        }

        public bool IsInstalled(Workspace workspace, ProjectId projectId, string packageName)
        {
            ThisCanBeCalledOnAnyThread();
            return _projectToInstalledPackageAndVersion.TryGetValue(projectId, out var installedPackages) &&
                installedPackages.IsInstalled(packageName);
        }

        public ImmutableArray<string> GetInstalledVersions(string packageName)
        {
            ThisCanBeCalledOnAnyThread();

            using var _ = PooledHashSet<string>.GetInstance(out var installedVersions);
            foreach (var state in _projectToInstalledPackageAndVersion.Values)
            {
                if (state.TryGetInstalledVersion(packageName, out var version))
                    installedVersions.Add(version);
            }

            // Order the versions with a weak heuristic so that 'newer' versions come first.
            // Essentially, we try to break the version on dots, and then we use a LogicalComparer
            // to try to more naturally order the things we see between the dots.
            var versionsAndSplits = installedVersions.Select(v => new { Version = v, Split = v.Split('.') }).ToList();

            versionsAndSplits.Sort((v1, v2) =>
            {
                var diff = CompareSplit(v1.Split, v2.Split);
                return diff != 0 ? diff : -v1.Version.CompareTo(v2.Version);
            });

            return versionsAndSplits.Select(v => v.Version).ToImmutableArray();
        }

        private int CompareSplit(string[] split1, string[] split2)
        {
            ThisCanBeCalledOnAnyThread();

            for (int i = 0, n = Math.Min(split1.Length, split2.Length); i < n; i++)
            {
                // Prefer things that look larger.  i.e. 7 should come before 6.
                // Use a logical string comparer so that 10 is understood to be
                // greater than 3.
                var diff = -LogicalStringComparer.Instance.Compare(split1[i], split2[i]);
                if (diff != 0)
                {
                    return diff;
                }
            }

            // Choose the one with more parts.
            return split2.Length - split1.Length;
        }

        public ImmutableArray<Project> GetProjectsWithInstalledPackage(Solution solution, string packageName, string version)
        {
            ThisCanBeCalledOnAnyThread();

            using var _ = ArrayBuilder<Project>.GetInstance(out var result);

            foreach (var (projectId, state) in this._projectToInstalledPackageAndVersion)
            {
                if (state.TryGetInstalledVersion(packageName, out var installedVersion) &&
                    installedVersion == version)
                {
                    var project = solution.GetProject(projectId);
                    if (project != null)
                        result.Add(project);
                }
            }

            return result.ToImmutable();
        }

        public bool CanShowManagePackagesDialog()
            => TryGetOrLoadNuGetPackageManager(out _);

        private bool TryGetOrLoadNuGetPackageManager([NotNullWhen(true)] out IVsPackage? nugetPackageManager)
        {
            this.AssertIsForeground();

            if (_nugetPackageManager != null)
            {
                nugetPackageManager = _nugetPackageManager;
                return true;
            }

            nugetPackageManager = null;
            var shell = (IVsShell)_serviceProvider.GetService(typeof(SVsShell));
            if (shell == null)
            {
                return false;
            }

            var nugetGuid = new Guid("5fcc8577-4feb-4d04-ad72-d6c629b083cc");
            shell.LoadPackage(ref nugetGuid, out nugetPackageManager);
            _nugetPackageManager = nugetPackageManager;
            return nugetPackageManager != null;
        }

        public void ShowManagePackagesDialog(string packageName)
        {
            if (!TryGetOrLoadNuGetPackageManager(out var nugetPackageManager))
            {
                return;
            }

            // We're able to launch the package manager (with an item in its search box) by
            // using the IVsSearchProvider API that the NuGet package exposes.
            //
            // We get that interface for it and then pass it a SearchQuery that effectively
            // wraps the package name we're looking for.  The NuGet package will then read
            // out that string and populate their search box with it.
            var extensionProvider = (IVsPackageExtensionProvider)nugetPackageManager;
            var extensionGuid = new Guid("042C2B4B-C7F7-49DB-B7A2-402EB8DC7892");
            var emptyGuid = Guid.Empty;
            var searchProvider = (IVsSearchProvider)extensionProvider.CreateExtensionInstance(ref emptyGuid, ref extensionGuid);
            var task = searchProvider.CreateSearch(dwCookie: 1, pSearchQuery: new SearchQuery(packageName), pSearchCallback: this);
            task.Start();
        }

        public void ReportProgress(IVsSearchTask pTask, uint dwProgress, uint dwMaxProgress)
        {
        }

        public void ReportComplete(IVsSearchTask pTask, uint dwResultsFound)
        {
        }

        public void ReportResult(IVsSearchTask pTask, IVsSearchItemResult pSearchItemResult)
            => pSearchItemResult.InvokeAction();

        public void ReportResults(IVsSearchTask pTask, uint dwResults, IVsSearchItemResult[] pSearchItemResults)
        {
        }

        private class SearchQuery : IVsSearchQuery
        {
            public SearchQuery(string packageName)
                => this.SearchString = packageName;

            public string SearchString { get; }

            public uint ParseError => 0;

            public uint GetTokens(uint dwMaxTokens, IVsSearchToken[] rgpSearchTokens)
                => 0;
        }
    }
}


text:
/**
 * Module dependencies.
 */

var ads = require('ad-params');
var clone = require('clone');
var cookie = require('cookie');
var extend = require('extend');
var integration = require('analytics.js-integration');
var json = require('segmentio/json@1.0.0');
var localstorage = require('store');
var md5 = require('SparkMD5').hash;
var protocol = require('protocol');
var send = require('send-json');
var topDomain = require('top-domain');
var utm = require('utm-params');
var uuid = require('uuid');

/**
 * Cookie options
 */

var cookieOptions = {
  // 1 year
  maxage: 31536000000,
  secure: false,
  path: '/'
};

/**
 * Expose `Segment` integration.
 */

var Segment = exports = module.exports = integration('Anyhost')
  .option('apiKey', '')
  .option('apiHost', 'api.segment.io/v1');

/**
 * Get the store.
 *
 * @return {Function}
 */

exports.storage = function() {
  return protocol() === 'file:' || protocol() === 'chrome-extension:' ? localstorage : cookie;
};

/**
 * Expose global for testing.
 */

exports.global = window;

/**
 * Initialize.
 *
 * https://github.com/segmentio/segmentio/blob/master/modules/segmentjs/segment.js/v1/segment.js
 *
 * @api public
 */

Segment.prototype.initialize = function() {
  var self = this;
  this.ready();
  this.analytics.on('invoke', function(msg) {
    var action = msg.action();
    var listener = 'on' + msg.action();
    self.debug('%s %o', action, msg);
    if (self[listener]) self[listener](msg);
    self.ready();
  });
};

/**
 * Loaded.
 *
 * @api private
 * @return {boolean}
 */

Segment.prototype.loaded = function() {
  return true;
};

/**
 * Page.
 *
 * @api public
 * @param {Page} page
 */

Segment.prototype.onpage = function(page) {
  this.send('/p', page.json());
};

/**
 * Identify.
 *
 * @api public
 * @param {Identify} identify
 */

Segment.prototype.onidentify = function(identify) {
  this.send('/i', identify.json());
};

/**
 * Group.
 *
 * @api public
 * @param {Group} group
 */

Segment.prototype.ongroup = function(group) {
  this.send('/g', group.json());
};

/**
 * ontrack.
 *
 * TODO: Document this.
 *
 * @api private
 * @param {Track} track
 */

Segment.prototype.ontrack = function(track) {
  var json = track.json();
  // TODO: figure out why we need traits.
  delete json.traits;
  this.send('/t', json);
};

/**
 * Alias.
 *
 * @api public
 * @param {Alias} alias
 */

Segment.prototype.onalias = function(alias) {
  var json = alias.json();
  var user = this.analytics.user();
  json.previousId = json.previousId || json.from || user.id() || user.anonymousId();
  json.userId = json.userId || json.to;
  delete json.from;
  delete json.to;
  this.send('/a', json);
};

/**
 * Normalize the given `msg`.
 *
 * @api private
 * @param {Object} msg
 */

Segment.prototype.normalize = function(msg) {
  this.debug('normalize %o', msg);
  var user = this.analytics.user();
  var global = exports.global;
  var query = global.location.search;
  var ctx = msg.context = msg.context || msg.options || {};
  delete msg.options;
  msg.writeKey = this.options.apiKey;
  ctx.userAgent = navigator.userAgent;
  if (!ctx.library) ctx.library = { name: 'analytics.js', version: this.analytics.VERSION };
  if (query) ctx.campaign = utm(query);
  this.referrerId(query, ctx);
  msg.userId = msg.userId || user.id();
  msg.anonymousId = user.anonymousId();
  msg.sentAt = new Date();
  // add some randomness to the messageId checksum
  msg.messageId = 'ajs-' + md5(json.stringify(msg) + uuid());
  this.debug('normalized %o', msg);
  this.ampId(ctx);
  return msg;
};

/**
 * Add amp id if it exists.
 *
 * @param {Object} ctx
 */

Segment.prototype.ampId = function(ctx){
  var ampId = this.cookie('segment_amp_id');
  if (ampId) ctx.amp = { id: ampId };
};

/**
 * Send `obj` to `path`.
 *
 * @api private
 * @param {string} path
 * @param {Object} obj
 * @param {Function} fn
 */

Segment.prototype.send = function(path, msg, fn) {
  var url = scheme() + '//' + this.options.apiHost + path;
  var headers = { 'Content-Type': 'text/plain' };
  fn = fn || noop;
  var self = this;

  // msg
  msg = this.normalize(msg);

  // send
  send(url, msg, headers, function(err, res) {
    self.debug('sent %O, received %O', msg, arguments);
    if (err) return fn(err);
    res.url = url;
    fn(null, res);
  });
};

/**
 * Gets/sets cookies on the appropriate domain.
 *
 * @api private
 * @param {string} name
 * @param {*} val
 */

Segment.prototype.cookie = function(name, val) {
  var store = Segment.storage();
  if (arguments.length === 1) return store(name);
  var global = exports.global;
  var href = global.location.href;
  var domain = '.' + topDomain(href);
  if (domain === '.') domain = '';
  this.debug('store domain %s -> %s', href, domain);
  var opts = clone(cookieOptions);
  opts.domain = domain;
  this.debug('store %s, %s, %o', name, val, opts);
  store(name, val, opts);
  if (store(name)) return;
  delete opts.domain;
  this.debug('fallback store %s, %s, %o', name, val, opts);
  store(name, val, opts);
};

/**
 * Add referrerId to context.
 *
 * TODO: remove.
 *
 * @api private
 * @param {Object} query
 * @param {Object} ctx
 */

Segment.prototype.referrerId = function(query, ctx) {
  var stored = this.cookie('s:context.referrer');
  var ad;

  if (stored) stored = json.parse(stored);
  if (query) ad = ads(query);

  ad = ad || stored;

  if (!ad) return;
  ctx.referrer = extend(ctx.referrer || {}, ad);
  this.cookie('s:context.referrer', json.stringify(ad));
};

/**
 * Get the scheme.
 *
 * The function returns `http:`
 * if the protocol is `http:` and
 * `https:` for other protocols.
 *
 * @api private
 * @return {string}
 */

function scheme() {
  return protocol() === 'http:' ? 'http:' : 'https:';
}

/**
 * Noop.
 */

function noop() {}

redpajama_set_name: RedPajamaGithub


text:
"""
Created on Thu Oct 23 10:46:02 2014

@author: Parke
"""
from __future__ import division, print_function, absolute_import
from numpy import median, sum, nan, array
import numpy as np
from scipy.stats import shapiro, norm, skewtest, poisson
from math import sqrt, inf, pi
from mypy.my_numpy import splitsum
from scipy.optimize import minimize as _minimize
from scipy.integrate import quad as _quad

def runstest(x, divider=None, passfail=False):
    """
    Perform the Wald-Wolfowitz runs test.

    The Wald-Wolfowitz runs test evaluates the null-hypothesis that the data
    are independent and identically distributed assuming a parametric
    form for the underlying probability distribution function of the data [1].

    Parameters
    ----------
    x : 1-D array-like
        The data to be tested.

    divider : float, optional
        The value to use to assign the data binary values. Data above
        the divider value will be assigned a 1, data below the divider will be
        assigned a 0, and data equal to the divider will be omitted.

        To test a function fit, supply runstest with the fit-subtracted data
        and set divider to 0.0.

        If not supplied, the median value of the data is used.

    passfail : {True|False}, optional
        If set to true, the function simply returns True if the test rejects
        the null hypothesis to > 95% confidence and False otherwise. This was
        implemented for short data series, where the runs statistic deviates
        significantly from a standard normal distribution and a lookup
        table is used to determine whether the test rejects the null hypothesis
        or not instead of computing a p-value.

    Returns
    -------
    z : float
        The test statistic.
    p-value : float
        The two-tailed p-value (probability to exceed) for the hypothesis test.

    References
    ---------
    .. [1] http://www.itl.nist.gov/div898/handbook/eda/section3/eda35d.htm

    """
    #TODO: maybe add this to scipy one day...

    if divider is None:
        divider = median(x)
    x_culled = x[x != divider]

    # Identify the runs
    x_binary = (x_culled > divider)
    run_edges = ((x_binary[1:] - x_binary[:-1]) != 0)
    r = sum(run_edges) + 1
    n = len(x_binary)
    n1 = sum(x_binary)
    n2 = n - n1

    # Use an approximating normal distribution if there is sufficient data
    if n1 > 10 and n2 > 10:
        mu = 2*n1*n2/n + 1
        var = (mu-1)*(mu-2)/(n-1)
        z = abs(r - mu)/sqrt(var)
        if passfail:
            return z > 1.96
        else:
            p = 2*(1 - norm.cdf(z))
            return z, p

    # Otherwise, use table lookup
    else:
        if not passfail:
            raise ValueError("The data in x have too few runs to compute a "
                             "reliable and exact p-value. Consider using "
                             "the function in passfail mode, which uses a "
                             "table lookup to provide a result for any data.")
        else:
            if n1 < 2 and n2 < 2:
                return False

            # From Wall and Jenkins, Practical Statistics for Astronomers,
            # Cambridge University Press, 2003, pp 257, Table A2.8
            # FIXME: the lo array is symmetric and I believe the hi array
            #    should be as well. However, it is asymmetric in the reference.
            #    Need to check against another reference.
            hi = [[nan]*19,
                  [nan]*19,
                  [nan]*3 + [9,9] + [nan]*14,
                  [nan]*2 + [9,10,10,11,11] + [nan]*12,
                  [nan]*2 + [9,10,11,12,12,13,13,13,13] + [nan]*8,
                  [nan]*3 + [11,12,13,13,14,14,14,14,15,15,15] + [nan]*5,
                  [nan]*3 + [11,12,13,14,14,15,15,16,16,16,16,17,17,17,17,17],
                  [nan]*4 + [13,14,14,15,16,16,16,17,17,18,18,18,18,18,18],
                  [nan]*4 + [13,14,15,16,16,17,17,18,18,18,19,19,19,20,20],
                  [nan]*4 + [13,14,15,16,17,17,18,19,19,19,20,20,20,21,21],
                  [nan]*4 + [13,14,16,16,17,18,19,19,20,20,21,21,21,22,22],
                  [nan]*5 + [15,16,17,18,19,19,20,20,21,21,22,22,23,23],
                  [nan]*5 + [15,16,17,18,19,19,20,20,21,21,22,22,23,23],
                  [nan]*5 + [15,16,18,18,19,20,21,22,22,23,23,24,24,25],
                  [nan]*6 + [17,18,19,20,21,21,22,23,23,24,25,25,25],
                  [nan]*6 + [17,18,19,20,21,22,23,23,24,25,25,26,26],
                  [nan]*6 + [17,18,19,20,21,22,23,24,25,25,26,26,27],
                  [nan]*6 + [17,18,20,21,22,23,23,24,25,25,26,26,27],
                  [nan]*6 + [17,18,20,21,22,23,24,25,25,26,27,27,28]]
            lo = [[nan]*10 + [2]*9,
                  [nan]*4 + [2]*9 + [3]*6,
                  [nan]*3 + [2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4],
                  [nan,nan,2,2,3,3,3,3,3,4,4,4,4,4,4,4,5,5,5],
                  [nan,2,2,3,3,3,3,4,4,4,4,5,5,5,5,5,5,6,6],
                  [nan,2,2,3,3,3,4,4,5,5,5,5,5,6,6,6,6,6,6],
                  [nan,2,3,3,3,4,4,5,5,5,6,6,6,6,6,7,7,7,7],
                  [nan,2,3,3,4,4,5,5,5,6,6,6,7,7,7,7,8,8,8],
                  [nan,2,3,3,4,5,5,5,6,6,7,7,7,7,8,8,8,8,9],
                  [nan,2,3,4,4,5,5,6,6,7,7,7,8,8,8,9,9,9,9],
                  [2,2,3,4,4,5,6,6,7,7,7,8,8,8,9,9,9,10,10],
                  [2,2,3,4,5,5,6,6,7,7,8,8,9,9,9,10,10,10,10],
                  [2,2,3,4,5,5,6,7,7,8,8,9,9,9,10,10,10,11,11],
                  [2,3,3,4,5,6,6,7,7,8,8,9,9,10,10,11,11,11,12],
                  [2,3,4,4,5,6,6,7,8,8,9,9,10,10,11,11,11,12,12],
                  [2,3,4,4,5,6,7,7,8,9,9,10,10,11,11,11,12,12,13],
                  [2,3,4,5,5,6,7,8,8,9,9,10,10,11,11,12,12,13,13],
                  [2,3,4,5,6,6,7,8,8,9,10,10,11,11,12,12,13,13,13],
                  [2,3,4,5,6,6,7,8,9,9,10,10,11,12,12,13,13,13,14]]
            hi, lo = list(map(array, [hi, lo]))
            return (r > hi[n1,n2] or r < lo[n1,n2])

def clean(x, tol, test='runs', metric='chi2', trendfit='median', maxiter=1000, printsteps=False):
    """
    Sequentially removes groups of anomalous data until the remaining data
    pases a user-specified test within some tolerance.

    Parameters
    ----------
    x : list or 1-D numpy array
        The data series to be analyzed for anomalies.

    tol : float
        Tolerance. When the value returned by testing the cleaned data is below
        this level, the data are considered successfully cleaned.

    test : {function|string}, optional
        Measures the "cleanliness" of the data, generally by computing the
        probability that the data could have been generated by some expected
        distribution (e.g. the normal distribution).

        String input can be used to specify one of the following built-in
        tests:
            - 'deviation size'|'devsize' : Value of the maximum deviation as
                quantified by "metric." Useful for simply removing all
                anomalies that exceed some threshold. Use tol to set that
                threshold.
            - 'shapiro-wilks'|'sw' : Tests whether the data are normally
                distributed. See scipy.stats.shapiro.
            - 'runs'|'r' : Tests whether the data are independent.
            - 'skew' : Tests whether data skewness is consistent with a normal
                distribution. See scipy.stats.skewtest. Useful when you just
                want the "clean" data to be symmetric about the trend.

        Function input allows the user to define a custom test. The function
        must accept two arguments, the trend removed data and a boolean
        array identifying the "clean" data, and return some measure of
        _un_cleanliness (i.e. the result must be smaller if the data are
        "cleaner").

    metric : {string|function}, optional
        Computes tdeviation values for the data. These values are then summed
        to quantify the deviance of each run.

        Built in metrics currently include:
            - 'len' : the number of points in the run
            - 'chi2' : the chi2 statistic for the run assuming unit variance
            - 'area' : area under the curve for the run

        Function input must accept an single array (the detrended) data and
        return a single array of equal length. For example, the 'area'
        function simply returns the absolute value of each point, and these are
        later summed.

    trendfit : {string|function|None}
        Trend to fit and subtract from the partly clean data at each iteration.

        String input can be used to specify one of the following built-in
        trends:
            - 'mean' : mean value of the data
            - 'median' : median value of the data

        Function input allows the user to define a custom trend. The function
        must accept *only* a boolean vector identifying the "clean" data as
        input and return the value of the trend at each data point.

        None means the function will not attempt to fit any trend.

    maxiter : int, optional
        Limits the number of iterations allowed. Error is thrown if this number
        is exceeded.

    Returns
    -------
    clean : 1-D numpy boolean array
        Boolean array identifying the clean (non-anomalous) data points.

    Description
    -----------
    The algorithm consists of three parts:
        (1) Fitting a trend to the data or partially-cleaned data.
        (2) Computing the result of the-user defined test on the detreneded
            data.
        (3) Flagging and removing the most deviant "runs" of data, where
            deviance is quantified by the user-specified metric.

    "Runs" are defined as groups of points succesively above or below the
    trend. The three steps are iterated until the test result is greater than the
    user-specified tolerance, tol and the points identified as clean are the
    same between two iterations.

    The algorithm was designed with the idea
    that it could be used to find and remove flares in time-series observations
    of stars and emission/absorption lines in spectra.

    Modification History
    --------------------
    2014-10-23 written by Parke Loyd
    2015-04-07 heavy modification
    """
    """
    Bad decisions:
        - "Creeping" in or out the edges of the anomalies after each new trendfit:
            the requisite for loops are SLOW. Way faster to just refind the
            anomalies using array operations.
        - iteratively flagging anoms, then refitting and repeating until no
            new points are identified. Since the first fit is high biased due
            to the big anomalies, lots or all of the "good" data is flagged
            and excluded from future fits. better to refit after each anom is
            removed.
    """
    ## PARSE THE INPUT
    n = len(x)

    # construct built in *statisctical* test function (others also built in)
    builtintests = {'runs':runstest, 'r':runstest, 'shapiro-wilks':shapiro,
                    'shapiro':shapiro, 'sw':shapiro, 'skew':skewtest}
    if test in builtintests:
        testfunc = builtintests[test.lower()]
        def test(x, good):
            return 1.0 - testfunc(x[good])[1]

    # construct built in trend fit function
    builtintrends = {'median' : np.median, 'mean':np.mean}
    if type(trendfit) is str:
        fitfunc = builtintrends[trendfit.lower()]
        def trendfit(good):
            value = fitfunc(x[good])
            return np.ones(x.shape)*value

    # construct built in metric
    builtinmetrics = {'chi2' : (lambda x: x**2),
                      'area' : (lambda x: np.abs(x)),
                      'len'  : (lambda x: np.ones(x.shape))}
    if type(metric) is str: metric = builtinmetrics[metric.lower()]

    ## ITERATIVELY FIT AND FLAG ANOMALIES
    Nanom = 0
    if printsteps:
        print('test value below {} needed to pass'.format(tol))
        print('anomalies % bad test')
        print('--------- ----- ---------')
    while True:
        Ngood_rec = []
        counter = 0
        good, good_old, Ngood = np.ones(n, bool), np.zeros(n, bool), n
        x1 = x
        while True:
            Ngood_rec.append(Ngood)
            Ngood = np.sum(good)
            counter += 1

            # check various conditions for exiting loop
            if Ngood == 0:
                raise ValueError('All data removed before test was passed.')
            if counter > maxiter:
                raise ValueError('Exceeded allowable number of iterations.')
            if np.all(good_old == good):
                # converged!
                break
            if counter > 10:
                if all(n == Ngood_rec[-10] for n in Ngood_rec[-10::2]):
                    # solution is (almost certainly) oscillating
                    break

            # fit the retained data
            fit = trendfit(good) if trendfit is not None else 0.0
            x1 = x - fit

            # sum deviations over each run of positive or negative points
            deviations, splits = __run_deviations(x1, metric)

            good_old = good
            if Nanom > 0:
                # identify the Nanom largest deviations
                order = np.argsort(np.abs(deviations))
                indices = np.arange(len(deviations))
                sorted_indices = indices[order]
                anomalies = sorted_indices[-Nanom:]

                # flag those anomalies
                good = __buildmask(anomalies, splits, n)
            else:
                good = np.ones(n, bool)

        # check if the remaining data is clean according to the desired
        # condition
        if test in ['devsize', 'deviation size']:
            # maximum deviation that hasn't been removed
            result = np.sort(deviations)[-Nanom - 1]
        elif test in ['sigma clip']:
            # maximum deviation in std dev from mean
            clean_devs = np.sort(deviations)[:(len(deviations)-Nanom)]
            result = np.abs(clean_devs[-1] - np.median(clean_devs)) / np.std(clean_devs)
        else:
            result = test(x1, good)
        if printsteps:
            pctbad = (1.0 - float(Ngood) / n) * 100.0
            print('{:9.0f} {:5.2f} {}'.format(Nanom, pctbad, result))
        if result < tol:
            return good
        else:
            Nanom += 1

def excess_noise(y, base_noise, Poisson=False, PDF=False, CDF=False, loglike=False):
    """Computes the maximum likelihood value of the excess noise.

    Specifically, this function assumes the data, y, are each drawn from a
    normal distribution with the same mean but different variances. The
    variance for the PDF from which a given point was drawn is given by
    base_noise that may differ from point to point and some excess noise that
    is the same for all points.

    Modification History:
    2014/05/12 - Created (only for Poisson=False case)
    """

    y, base_noise = np.array(y), np.array(base_noise)
    ymn = np.mean(y)
    yvar = np.var(y)
    base_var = base_noise**2
    x_guess = np.sqrt(yvar - np.mean(base_var)) if yvar > np.mean(base_var) else 0.0

    def log_like(mn,x_noise,log_norm_fac):
        x_var = x_noise**2
        var = x_var + base_var
        terms = -np.log(2*np.pi*var)/2 - (y - mn)**2/2/var
        return np.sum(terms) + log_norm_fac
    if loglike:
        def log_like_for_min(params):
            mn, x_noise = params
            if x_noise < 0:
                return -np.inf
            else:
                return log_like(mn, x_noise, 0)
        return log_like_for_min

    #initial pass at a normalization factor - use peak value
    def neg_like(x):
        mn,x_noise= x
        return -log_like(mn,x_noise,0.0)
    result = _minimize(neg_like, [ymn,x_guess], method='Nelder-Mead')
    log_norm_fac = result.fun
    mn_pk, x_noise_pk = result.x

    def like(mn,x_noise):
        return np.exp(log_like(mn, x_noise, log_norm_fac))

    def xmn_like(xnorm):
        constrained_like = lambda mn: like(mn, xnorm*mn)
        result = _quad(constrained_like, mn_pk, np.inf)[0]
        result += -_quad(constrained_like, mn_pk, 0.0)[0]
        return result

    xmn_pk = x_noise_pk/mn_pk
    result = _minimize(lambda x: -xmn_like(x), xmn_pk, method='Nelder-Mead')
    xmn_pk = result.x[0]
    if xmn_pk < 0.0: xmn_pk = 0.0

    area = _quad(xmn_like, xmn_pk, np.inf)[0]
    area += -_quad(xmn_like, xmn_pk, 0.0)[0]

    if PDF:
        pdf = lambda x: xmn_like(x)/area if x >= 0.0 else 0.0
    if CDF:
        def cdf(x):
            if x <= xmn_pk:
                return -_quad(xmn_like, x, 0.0)[0]/area
            else:
                y = -_quad(xmn_like, xmn_pk, 0.0)[0]/area
                y += _quad(xmn_like, xmn_pk, x)[0]/area
                return y

    if PDF and CDF:
        return xmn_pk, pdf, cdf
    elif PDF:
        return xmn_pk, pdf
    elif CDF:
        return xmn_pk, cdf
    else:
        return xmn_pk

# backward compatible
def excess_noise_PDF(y, base_noise, Poisson=False):
    xmn_pk, pdf = excess_noise(y, base_noise, Poisson=Poisson, PDF=True)
    return pdf, xmn_pk

def poisson_to_skewnorm(rate):
    """
    Approximate a poisson distribution of given rate with a skew normal distribution.

    Parameters
    ----------
    rate
        Rate for Poisson distribution. Must be > 1.0.

    Returns
    -------
    a, loc, scale :
        Parameters for use with scipy.stats.skewnorm. Note that the mean = loc + scale*delta*sqrt(2/pi); var =
        scale**2*(1 - 2*delta**2/pi); skew = (4 - pi)/2 * (delta*sqrt(2/pi))**3/(1 - 2*delta**2/pi)**(3/2); where
        delta = a/sqrt(1 + a**2)
    """
    if rate <= 1.0:
        raise ValueError('Only defined for rate >= 1.0.')

    mean, var, skew = poisson.stats(rate, moments='mvs')

    delta = np.sqrt(np.pi/2*skew**(2./3)/(skew**(2./3) + ((4-np.pi)/2)**(2./3)))
    a = delta/np.sqrt(1 - delta**2)
    scale = np.sqrt(var/(1 - 2*delta**2/np.pi))
    loc = mean - scale*delta*np.sqrt(2/np.pi)
    return a, loc, scale

def __run_deviations(x, metric):
    pospts = (x > 0)
    splits = np.nonzero(pospts[1:] - pospts[:-1])[0] + 1
    ptdevs = metric(x)
    deviations = splitsum(ptdevs, splits)
    return deviations, splits

def __buildmask(flags, splits, n):
    # split a vector of all True values up by run
    alltrue = np.ones(n, bool)
    runs = np.array(np.split(alltrue, splits))
    # then mark the flags as false
    runs[flags] = runs[flags]*False
    # and reconstruct the vector
    good = np.hstack(runs)
    return good

def multivariate_normal_loglike(x, mean=None, cov=None):
    p, n = x.shape
    if mean is None:
        mean = np.mean(x, axis=1)
    if cov is None:
        cov = np.cov(x)
    inv_cov = np.linalg.inv(cov)
    # ummm time for some gnarly vectorizing to do the matrix sum for each
    # term. this matrix sum is (x_i - mean).T * cov**-1 * (x_i - mean)
    # where the matrices in the product have shapes 1 x n, n x n, n x 1
    xx = x - mean[:,None] # p x n
    element_prod = x[:,None,:]*inv_cov[:,:,None] # p x p x n
    matrix_prod_first = np.sum(element_prod, 0) # p x n
    element_prod = matrix_prod_first * xx # p x n
    terms = matrix_prod_final = np.sum(element_prod, 0)
    result = -n*p/2*np.log(2*np.pi) - n/2*np.log(np.linalg.det(cov)) \
             - 0.5*np.sum(terms)
    return result

def draw_arbitrary(n, icdf):
    """
    Draw random variables from an arbitrary 1D distribution given the
    inverse cdf. The heavy lifting is on the user to determine the inverse
    cdf function.

    Parameters
    ----------
    n : integer
    icdf : function
        When given a cumulative probability on [0,1], should return the value
        corresponding to that probability (i.e. gives the x value for a plot
        where y is the cdf).

    Returns
    -------
    values of n random draws
    """

    y = np.random.rand(n)
    try:
        x = icdf(y)
    except ValueError:
        x = list(map(icdf, y))
    return x

def line_path_error_function(x, y, xerr, yerr, xa=-inf, xb=inf, samplefac=10):
    """Compute the probability that the data points actually lie along a
    line given the x and y value of the data point, the errors
    on the data, and the line parameters. If desired, compute only for the
    segment extending from xa to xb.

    samplefac gives how many points per sigma. Will pick either x or y sigma to
    determine this depending on the steepness of the line and size of the sigmas.
    """

    if xa == -inf:
        xa = x - 5*xerr
    if xb == inf:
        xb = x + 5*xerr

    # make both x and y grids to sample appropriately according to the steepness of the line
    # e.g. for steep lines need to make sure there are enough y points to properly sample PDF
    dx = xerr/samplefac
    dy = yerr/samplefac
    xgrid = np.arange(xa, xb + dx, dx)
    ya = y - 5*yerr
    yb = y + 5*yerr
    ygrid = np.arange(ya, yb + dy, dy)

    norm = lambda xx, yy: 1/2/pi/sqrt(xerr*yerr) * np.exp(-(xx - x)**2/2/xerr**2 - (yy - y)**2/2/yerr**2)

    def path_errfunc(m, b):
        mnorm = m * xerr / yerr
        if abs(mnorm) > 1:
            # line covers more ground in y than x relative to errors
            # so invert and create a function using y as indep variable
            # y = mx + b => x = (y - b)/m = 1/m*y - b/m
            ya = m*xa + b
            yb = m*xb + b
            use = (ygrid > ya) & (ygrid < yb)
            yy = ygrid[use]
            xx = (yy - b)/m
            ddx = dy/m
            ddy = dy
        else:
            xx = xgrid
            yy = m*xx + b
            ddx = dx*m
            ddy = dy

        norms = norm(xx, yy)
        ds = sqrt(ddx**2 + ddy**2)
        return np.sum(norms*ds)

    return path_errfunc


text:
..  Copyright (C)  Brad Miller, David Ranum, Jeffrey Elkner, Peter Wentworth, Allen B. Downey, Chris
    Meyers, and Dario Mitchell.  Permission is granted to copy, distribute
    and/or modify this document under the terms of the GNU Free Documentation
    License, Version 1.3 or any later version published by the Free Software
    Foundation; with Invariant Sections being Forward, Prefaces, and
    Contributor List, no Front-Cover Texts, and no Back-Cover Texts.  A copy of
    the license is included in the section entitled "GNU Free Documentation
    License".

Introduction: Strings
=======

.. index:: compound data type, character, subscript operator, index


text:
require "uri"

require_relative "helper"

describe "#update_secret", :secret_manager_snippet do
  it "updates the secret" do
    sample = SampleLoader.load "update_secret.rb"

    refute_nil secret

    out, _err = capture_io do
      sample.run project_id: project_id, secret_id: secret_id
    end

    assert_match(/Updated secret/, out)
    assert_match(/New label: rocks/, out)
  end
end


text:
<readable><title>241345656_861aacefde</title><content>
A cheerleader dressed in red smiles and cheers .
A smiling blond cheerleader runs with an audience in the background .
A smiling dancer dressed in red is on the football field with fans in the stadium bleachers behind her .
A woman in a red sports outfit is running on the grass in front of a crowd of spectators .
A woman in a red suit is running in a field in front of a crowd .
</content></readable>


text:
namespace igloo {

  namespace detail {
    //
    // Check if the context is marked as "only"
    //
    template <typename T>
    inline bool IsContextMarkedAsOnly()
    {
      return T::IsContextMarkedAsOnly();
    }

    //
    // Specialization for root context. Is never marked "only".
    //
    template <>
    inline bool IsContextMarkedAsOnly<ContextWithAttribute<void> >()
    {
      return false;
    }

    //
    // Check if the context is marked as "skip"
    //
    template <typename T>
    inline bool IsContextMarkedAsSkip()
    {
      return T::IsMarkedAsSkip();
    }

    //
    // Specialization for root context. Is never marked as "skip".
    //
    template <>
    inline bool IsContextMarkedAsSkip<ContextWithAttribute<void> >()
    {
      return false;
    }
  }

  //
  // This class provides functionality for nested contexts.
  //
  // Template arguments:
  //   InnerContext - this is the type of the current context.
  //   OuterContext - the type of the outer (parent) context.
  //                  For root contexts, this is 'void'.
  //   ISONLY       - Marks this context as 'only', which means
  //                  this context and its children are the only
  //                  contexts to be executed.
  //   ISSKIP       - Marks this context as 'skip', which means
  //                  this context and its children should not
  //                  be executed.
  //
  template <typename InnerContext, typename OuterContext, bool ISONLY, bool ISSKIP>
    struct NestedContextProvider: public igloo::ContextWithAttribute<InnerContext>
  {
    //
    // By storing the current context in 'IGLOO_CURRENT_CONTEXT', the macros for
    // registering contexts nested to this will be created with this context as
    // outer context.
    //
    typedef InnerContext IGLOO_CURRENT_CONTEXT;

    static bool IsContextMarkedAsOnly()
    {
      return ISONLY ||
             ContextRegistry<InnerContext>::HasSpecsMarkedAsOnly() ||
             detail::IsContextMarkedAsOnly<OuterContext>();
    }

    static bool IsMarkedAsSkip()
    {
      return ISSKIP || detail::IsContextMarkedAsSkip<OuterContext>();
    }

    virtual OuterContext& Parent()
    {
      if(m_outerContext.get() == 0)
      {
        m_outerContext = std::auto_ptr<OuterContext>(CreateIglooContext<OuterContext>());
      }
      return *(m_outerContext.get());
    }

    virtual void IglooFrameworkSetUp()
    {
      Parent().IglooFrameworkSetUp();
      this->SetUp();
    }

    virtual void IglooFrameworkTearDown()
    {
      this->TearDown();
      Parent().IglooFrameworkTearDown();
    }

    private:
    template <typename ContextType>
      ContextType* CreateIglooContext()
      {
        return new ContextType();
      }

    std::auto_ptr<OuterContext> m_outerContext;
  };

  //
  // This class makes it possible for contexts to retrieve the root context.
  //
  template <typename InnerContext, typename OuterContext, typename RootContext, bool ISONLY, bool ISSKIP>
    struct ContextProvider : public NestedContextProvider<InnerContext, OuterContext, ISONLY, ISSKIP>
    {
      //
      // For non root contexts, we ask our parent to return the root context.
      // This will work recursively up to the root context.
      //
      RootContext& Root()
      {
        return NestedContextProvider<InnerContext, OuterContext, ISONLY, ISSKIP>::Parent().Root();
      }
    };

  //
  // Specialization for the root contexts.
  //
  template <typename InnerContext, bool ISONLY, bool ISSKIP>
    struct ContextProvider<InnerContext, ContextWithAttribute<void>, void, ISONLY, ISSKIP> : public NestedContextProvider<InnerContext, ContextWithAttribute<void>, ISONLY, ISSKIP>
  {
      //
      // By setting this, nested contexts will have access to the type
      // of the root context so that their 'Root()' method can return
      // the correct type.
      //
      typedef InnerContext IGLOO_ROOT_CONTEXT;

      //
      // For the root context, we return a reference to this
      // context.
      //
      InnerContext& Root()
      {
        return *(static_cast<InnerContext*>(this));
      }
  };

}
#endif


text:
// ------------------------------------------------------------------------------
// <auto-generated>
//     This code was generated by a tool.
//     Runtime Version: 10.0.0.0
//
//     Changes to this file may cause incorrect behavior and will be lost if
//     the code is regenerated.
// </auto-generated>
// ------------------------------------------------------------------------------
namespace CyPhyPET.Templates.TestBenchExecutors
{
    using System;
    using System.IO;
    using System.Diagnostics;
    using System.Linq;
    using System.Xml.Linq;
    using System.Collections;
    using System.Collections.Generic;
    using ISIS.GME.Dsml.CyPhyML.Classes;

    #line 1 "C:\META\meta_trunk\src\CyPhyPET\Templates\TestBenchExecutors\Modelica.tt"
    [System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "10.0.0.0")]
    public partial class Modelica : ModelicaBase
    {
        public virtual string TransformText()
        {
            this.Write(" \r\n");
            this.Write(" \r\n");
            this.Write(" \r\n \r\n# =========================================================================" +
                    "==\r\n# Auto generated from Modelica.tt\r\n# =======================================" +
                    "====================================\r\n\"\"\"\r\nFunctions in this file are always cal" +
                    "led from the TestBench directory.\r\n\"\"\"\r\nimport os\r\nimport json\r\nimport logging\r\n" +
                    "import subprocess\r\nimport py_modelica as pym\r\n# Throw this exception for \"expect" +
                    "ed\" errors.\r\nfrom driver_runner import TestBenchExecutionError\r\n\r\n# Reference to" +
                    " instantiated Modelica tool from py_modelica\r\nMODELICA_TOOL = None\r\n\r\n\r\ndef init" +
                    "ial_run():\r\n    \"\"\"\r\n    Instantiates a new tool for simulation,\r\n    compiles a" +
                    "nd makes one default simulation.\r\n    \"\"\"\r\n\r\n    log = logging.getLogger()\r\n    " +
                    "tb_dir = os.getcwd()\r\n    # Read the configuration file to get which Modelica to" +
                    "ol to use.\r\n    config_file = os.path.abspath(os.path.join(\'..\', \'test_bench_mod" +
                    "el_config.json\'))\r\n    with open(config_file, \'r\') as f_in:\r\n        tool_name =" +
                    " json.load(f_in)[\'tool_name\']\r\n    log.info(\'Loaded {0}, defined tool : {1}\'.for" +
                    "mat(config_file, tool_name))\r\n\r\n    os.chdir(\'CyPhy\')\r\n    # Read in the model_c" +
                    "onfig to be passed to the tool.\r\n    with open(\'model_config.json\', \'r\') as f_in" +
                    ":\r\n        model_config = json.load(f_in)\r\n    result_mat_file = model_config[\'r" +
                    "esult_file\']\r\n    del model_config[\'result_file\']\r\n\r\n    # Instantiate Modelica " +
                    "tool.\r\n    global MODELICA_TOOL\r\n    try:\r\n        if tool_name == \'OpenModelica" +
                    "\':\r\n            MODELICA_TOOL = pym.OpenModelica(model_config)\r\n            MODE" +
                    "LICA_TOOL.short_name = result_mat_file\r\n        elif tool_name == \'Dymola\':\r\n   " +
                    "         MODELICA_TOOL = pym.Dymola(model_config)\r\n            MODELICA_TOOL.res" +
                    "ult_mat = \'{0}.mat\'.format(result_mat_file)\r\n        elif tool_name == \'JModelic" +
                    "a\':\r\n            MODELICA_TOOL = pym.JModelica(model_config)\r\n    except pym.Mod" +
                    "elicaError as err:\r\n        raise TestBenchExecutionError(err.message)\r\n\r\n    # " +
                    "Compile the model.\r\n    try:\r\n        MODELICA_TOOL.compile_model()\r\n    except " +
                    "pym.ModelicaError as err:\r\n        raise TestBenchExecutionError(err.message)\r\n\r" +
                    "\n\r\n    # Read in parameters.\r\n    tb_parameters = pym.get_parameters_from_report" +
                    "_json(os.path.join(\'..\', \'testbench_manifest.json\'))\r\n    log.info(\'TestBench Pa" +
                    "rameters : {0}\'.format(tb_parameters))\r\n    if tb_parameters:\r\n        MODELICA_" +
                    "TOOL.change_parameter(tb_parameters)\r\n\r\n    os.chdir(tb_dir)\r\n    # Execute the " +
                    "model once.\r\n    execute()\r\n\r\n\r\ndef update_parameters(parameters):\r\n    \"\"\"\r\n   " +
                    " Uses the passed parameters rather than reading from the testbench_manifest.json" +
                    ". \r\n    The reason is that a PCC-driver might have special PCC-Properties.\r\n    " +
                    "\"\"\"\r\n    tb_dir = os.getcwd()\r\n    if parameters:\r\n        MODELICA_TOOL.change_" +
                    "parameter(parameters)\r\n    os.chdir(tb_dir)\r\n\r\n\r\ndef execute():\r\n    \"\"\"\r\n    Re" +
                    "ads in parameters from testbench_manifest.json and simulates the\r\n    test bench" +
                    ". When done, writes metrics to testbench_manifest.json.\r\n    \"\"\"\r\n    tb_dir = o" +
                    "s.getcwd()\r\n    log = logging.getLogger()\r\n    os.chdir(\'CyPhy\')\r\n    log.debug(" +
                    "\'Changed dir to {0}\'.format(os.getcwd()))\r\n\r\n    # Simulate model\r\n    try:\r\n   " +
                    "     MODELICA_TOOL.simulate_model()\r\n    except pym.ModelicaError as err:\r\n     " +
                    "   raise TestBenchExecutionError(err.message)\r\n\r\n    # Run PostProcessing script" +
                    "s\r\n    pp_failed = os.path.abspath(os.path.join(tb_dir, \'..\', \'_POST_PROCESSING_" +
                    "FAILED.txt\'))\r\n    if os.path.isfile(pp_failed):\r\n        log.info(\'Found {0} fr" +
                    "om previous run, deleting it...\'.format(pp_failed))\r\n        os.remove(pp_failed" +
                    ")\r\n    exception = None\r\n    try:\r\n        t_post = pym.run_post_scripts(MODELIC" +
                    "A_TOOL.result_mat, log)\r\n    except subprocess.CalledProcessError as err:\r\n     " +
                    "   exception = err\r\n    finally:\r\n        pp_output = \'\'\r\n        if exception:\r" +
                    "\n            pp_output = \'Post-script execution returned with error. : {0}\'.form" +
                    "at(exception.returncode)\r\n        if os.path.isfile(pp_failed):\r\n            wit" +
                    "h open(pp_failed, \'r\') as f_in:\r\n                pp_output = \'{0}\\n POST_PROCESS" +
                    "ING_FAILED :\\n {1}\'.format(pp_output, f_in.read())\r\n        if pp_output:\r\n     " +
                    "       raise TestBenchExecutionError(pp_output)\r\n    ## TODO: What are the rules" +
                    " of limit checks?\r\n    os.chdir(tb_dir)\r\n    return None");
            return this.GenerationEnvironment.ToString();
        }
    }

    #line default
    #line hidden
    #region Base class
    /// <summary>
    /// Base class for this transformation
    /// </summary>
    [System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "10.0.0.0")]
    public class ModelicaBase
    {
        #region Fields
        private global::System.Text.StringBuilder generationEnvironmentField;
        private global::System.CodeDom.Compiler.CompilerErrorCollection errorsField;
        private global::System.Collections.Generic.List<int> indentLengthsField;
        private string currentIndentField = "";
        private bool endsWithNewline;
        private global::System.Collections.Generic.IDictionary<string, object> sessionField;
        #endregion
        #region Properties
        /// <summary>
        /// The string builder that generation-time code is using to assemble generated output
        /// </summary>
        protected System.Text.StringBuilder GenerationEnvironment
        {
            get
            {
                if ((this.generationEnvironmentField == null))
                {
                    this.generationEnvironmentField = new global::System.Text.StringBuilder();
                }
                return this.generationEnvironmentField;
            }
            set
            {
                this.generationEnvironmentField = value;
            }
        }
        /// <summary>
        /// The error collection for the generation process
        /// </summary>
        public System.CodeDom.Compiler.CompilerErrorCollection Errors
        {
            get
            {
                if ((this.errorsField == null))
                {
                    this.errorsField = new global::System.CodeDom.Compiler.CompilerErrorCollection();
                }
                return this.errorsField;
            }
        }
        /// <summary>
        /// A list of the lengths of each indent that was added with PushIndent
        /// </summary>
        private System.Collections.Generic.List<int> indentLengths
        {
            get
            {
                if ((this.indentLengthsField == null))
                {
                    this.indentLengthsField = new global::System.Collections.Generic.List<int>();
                }
                return this.indentLengthsField;
            }
        }
        /// <summary>
        /// Gets the current indent we use when adding lines to the output
        /// </summary>
        public string CurrentIndent
        {
            get
            {
                return this.currentIndentField;
            }
        }
        /// <summary>
        /// Current transformation session
        /// </summary>
        public virtual global::System.Collections.Generic.IDictionary<string, object> Session
        {
            get
            {
                return this.sessionField;
            }
            set
            {
                this.sessionField = value;
            }
        }
        #endregion
        #region Transform-time helpers
        /// <summary>
        /// Write text directly into the generated output
        /// </summary>
        public void Write(string textToAppend)
        {
            if (string.IsNullOrEmpty(textToAppend))
            {
                return;
            }
            // If we're starting off, or if the previous text ended with a newline,
            // we have to append the current indent first.
            if (((this.GenerationEnvironment.Length == 0)
                        || this.endsWithNewline))
            {
                this.GenerationEnvironment.Append(this.currentIndentField);
                this.endsWithNewline = false;
            }
            // Check if the current text ends with a newline
            if (textToAppend.EndsWith(global::System.Environment.NewLine, global::System.StringComparison.CurrentCulture))
            {
                this.endsWithNewline = true;
            }
            // This is an optimization. If the current indent is "", then we don't have to do any
            // of the more complex stuff further down.
            if ((this.currentIndentField.Length == 0))
            {
                this.GenerationEnvironment.Append(textToAppend);
                return;
            }
            // Everywhere there is a newline in the text, add an indent after it
            textToAppend = textToAppend.Replace(global::System.Environment.NewLine, (global::System.Environment.NewLine + this.currentIndentField));
            // If the text ends with a newline, then we should strip off the indent added at the very end
            // because the appropriate indent will be added when the next time Write() is called
            if (this.endsWithNewline)
            {
                this.GenerationEnvironment.Append(textToAppend, 0, (textToAppend.Length - this.currentIndentField.Length));
            }
            else
            {
                this.GenerationEnvironment.Append(textToAppend);
            }
        }
        /// <summary>
        /// Write text directly into the generated output
        /// </summary>
        public void WriteLine(string textToAppend)
        {
            this.Write(textToAppend);
            this.GenerationEnvironment.AppendLine();
            this.endsWithNewline = true;
        }
        /// <summary>
        /// Write formatted text directly into the generated output
        /// </summary>
        public void Write(string format, params object[] args)
        {
            this.Write(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args));
        }
        /// <summary>
        /// Write formatted text directly into the generated output
        /// </summary>
        public void WriteLine(string format, params object[] args)
        {
            this.WriteLine(string.Format(global::System.Globalization.CultureInfo.CurrentCulture, format, args));
        }
        /// <summary>
        /// Raise an error
        /// </summary>
        public void Error(string message)
        {
            System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError();
            error.ErrorText = message;
            this.Errors.Add(error);
        }
        /// <summary>
        /// Raise a warning
        /// </summary>
        public void Warning(string message)
        {
            System.CodeDom.Compiler.CompilerError error = new global::System.CodeDom.Compiler.CompilerError();
            error.ErrorText = message;
            error.IsWarning = true;
            this.Errors.Add(error);
        }
        /// <summary>
        /// Increase the indent
        /// </summary>
        public void PushIndent(string indent)
        {
            if ((indent == null))
            {
                throw new global::System.ArgumentNullException("indent");
            }
            this.currentIndentField = (this.currentIndentField + indent);
            this.indentLengths.Add(indent.Length);
        }
        /// <summary>
        /// Remove the last indent that was added with PushIndent
        /// </summary>
        public string PopIndent()
        {
            string returnValue = "";
            if ((this.indentLengths.Count > 0))
            {
                int indentLength = this.indentLengths[(this.indentLengths.Count - 1)];
                this.indentLengths.RemoveAt((this.indentLengths.Count - 1));
                if ((indentLength > 0))
                {
                    returnValue = this.currentIndentField.Substring((this.currentIndentField.Length - indentLength));
                    this.currentIndentField = this.currentIndentField.Remove((this.currentIndentField.Length - indentLength));
                }
            }
            return returnValue;
        }
        /// <summary>
        /// Remove any indentation
        /// </summary>
        public void ClearIndent()
        {
            this.indentLengths.Clear();
            this.currentIndentField = "";
        }
        #endregion
        #region ToString Helpers
        /// <summary>
        /// Utility class to produce culture-oriented representation of an object as a string.
        /// </summary>
        public class ToStringInstanceHelper
        {
            private System.IFormatProvider formatProviderField  = global::System.Globalization.CultureInfo.InvariantCulture;
            /// <summary>
            /// Gets or sets format provider to be used by ToStringWithCulture method.
            /// </summary>
            public System.IFormatProvider FormatProvider
            {
                get
                {
                    return this.formatProviderField ;
                }
                set
                {
                    if ((value != null))
                    {
                        this.formatProviderField  = value;
                    }
                }
            }
            /// <summary>
            /// This is called from the compile/run appdomain to convert objects within an expression block to a string
            /// </summary>
            public string ToStringWithCulture(object objectToConvert)
            {
                if ((objectToConvert == null))
                {
                    throw new global::System.ArgumentNullException("objectToConvert");
                }
                System.Type t = objectToConvert.GetType();
                System.Reflection.MethodInfo method = t.GetMethod("ToString", new System.Type[] {
                            typeof(System.IFormatProvider)});
                if ((method == null))
                {
                    return objectToConvert.ToString();
                }
                else
                {
                    return ((string)(method.Invoke(objectToConvert, new object[] {
                                this.formatProviderField })));
                }
            }
        }
        private ToStringInstanceHelper toStringHelperField = new ToStringInstanceHelper();
        public ToStringInstanceHelper ToStringHelper
        {
            get
            {
                return this.toStringHelperField;
            }
        }
        #endregion
    }
    #endregion
}


text:
package org.onosproject.net.intent.impl.compiler;

import com.google.common.collect.Sets;
import org.onlab.packet.EthType;
import org.onlab.packet.Ethernet;
import org.onlab.packet.MplsLabel;
import org.onlab.packet.VlanId;
import org.onlab.util.Identifier;
import org.onosproject.net.ConnectPoint;
import org.onosproject.net.DeviceId;
import org.onosproject.net.EncapsulationType;
import org.onosproject.net.Link;
import org.onosproject.net.LinkKey;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.criteria.Criterion;
import org.onosproject.net.flow.criteria.EthTypeCriterion;
import org.onosproject.net.flow.criteria.MplsCriterion;
import org.onosproject.net.flow.criteria.VlanIdCriterion;
import org.onosproject.net.flow.instructions.Instruction;
import org.onosproject.net.flow.instructions.L2ModificationInstruction;
import org.onosproject.net.intent.IntentCompilationException;
import org.onosproject.net.intent.PathIntent;
import org.onosproject.net.intent.constraint.EncapsulationConstraint;
import org.onosproject.net.resource.ResourceService;
import org.onosproject.net.resource.impl.LabelAllocator;
import org.slf4j.Logger;

import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

import static org.onosproject.net.LinkKey.linkKey;

/**
 * Shared APIs and implementations for path compilers.
 */

public class PathCompiler<T> {

    private static final String ERROR_VLAN = "No VLAN Ids available for ";
    private static final String ERROR_MPLS = "No available MPLS labels for ";

    static LabelAllocator labelAllocator;

    /**
     * Defines methods used to create objects representing flows.
     */
    public interface PathCompilerCreateFlow<T> {

        void createFlow(TrafficSelector originalSelector,
                        TrafficTreatment originalTreatment,
                        ConnectPoint ingress, ConnectPoint egress,
                        int priority,
                        boolean applyTreatment,
                        List<T> flows,
                        List<DeviceId> devices);

        Logger log();

        ResourceService resourceService();
    }

    private boolean isLast(List<Link> links, int i) {
        return i == links.size() - 2;
    }

    /**
     * Returns the ethertype match needed. If the selector provides
     * an ethertype, it will be used. IPv4 will be used otherwise.
     *
     * @param selector the traffic selector.
     * @return the ethertype we should match against
     */
    private EthType getEthType(TrafficSelector selector) {
        Criterion c = selector.getCriterion(Criterion.Type.ETH_TYPE);
        if (c != null && c instanceof EthTypeCriterion) {
            EthTypeCriterion ethertype = (EthTypeCriterion) c;
            return ethertype.ethType();
        } else {
            return EthType.EtherType.IPV4.ethType();
        }
    }

    /**
     * Creates the flow rules for the path intent using VLAN
     * encapsulation.
     *
     * @param creator the flowrules creator
     * @param flows the list of flows to fill
     * @param devices the devices on the path
     * @param intent the PathIntent to compile
     */
    private void manageVlanEncap(PathCompilerCreateFlow<T> creator, List<T> flows,
                                 List<DeviceId> devices,
                                 PathIntent intent) {

        Set<Link> linksSet = Sets.newConcurrentHashSet();
        for (int i = 1; i <= intent.path().links().size() - 2; i++) {
            linksSet.add(intent.path().links().get(i));
        }

        Map<LinkKey, Identifier<?>> vlanIds = labelAllocator.assignLabelToLinks(
                linksSet,
                intent.id(),
                EncapsulationType.VLAN
        );

        Iterator<Link> links = intent.path().links().iterator();
        Link srcLink = links.next();

        Link link = links.next();

        // Ingress traffic
        VlanId vlanId = (VlanId) vlanIds.get(linkKey(link));
        if (vlanId == null) {
            throw new IntentCompilationException(ERROR_VLAN + link);
        }
        VlanId prevVlanId = vlanId;

        Optional<VlanIdCriterion> vlanCriterion = intent.selector().criteria()
                .stream().filter(criterion -> criterion.type() == Criterion.Type.VLAN_VID)
                .map(criterion -> (VlanIdCriterion) criterion)
                .findAny();

        //Push VLAN if selector does not include VLAN
        TrafficTreatment.Builder treatBuilder = DefaultTrafficTreatment.builder();
        if (!vlanCriterion.isPresent()) {
            treatBuilder.pushVlan();
        }
        //Tag the traffic with the new encapsulation VLAN
        treatBuilder.setVlanId(vlanId);
        creator.createFlow(intent.selector(), treatBuilder.build(),
                           srcLink.dst(), link.src(), intent.priority(), true,
                           flows, devices);

        ConnectPoint prev = link.dst();

        while (links.hasNext()) {

            link = links.next();

            if (links.hasNext()) {
                // Transit traffic
                VlanId egressVlanId = (VlanId) vlanIds.get(linkKey(link));
                if (egressVlanId == null) {
                    throw new IntentCompilationException(ERROR_VLAN + link);
                }

                TrafficSelector transitSelector = DefaultTrafficSelector.builder()
                        .matchInPort(prev.port())
                        .matchVlanId(prevVlanId).build();

                TrafficTreatment.Builder transitTreat = DefaultTrafficTreatment.builder();

                // Set the new vlanId only if the previous one is different
                if (!prevVlanId.equals(egressVlanId)) {
                    transitTreat.setVlanId(egressVlanId);
                }
                creator.createFlow(transitSelector,
                                   transitTreat.build(), prev, link.src(),
                                   intent.priority(), true, flows, devices);
                /* For the next hop we have to remember
                 * the previous egress VLAN id and the egress
                 * node
                 */
                prevVlanId = egressVlanId;
                prev = link.dst();
            } else {
                // Egress traffic
                TrafficSelector egressSelector = DefaultTrafficSelector.builder()
                        .matchInPort(prev.port())
                        .matchVlanId(prevVlanId).build();
                TrafficTreatment.Builder egressTreat = DefaultTrafficTreatment.builder(intent.treatment());

                Optional<L2ModificationInstruction.ModVlanIdInstruction> modVlanIdInstruction = intent.treatment()
                        .allInstructions().stream().filter(
                                instruction -> instruction instanceof L2ModificationInstruction.ModVlanIdInstruction)
                        .map(x -> (L2ModificationInstruction.ModVlanIdInstruction) x).findAny();

                Optional<L2ModificationInstruction.ModVlanHeaderInstruction> popVlanInstruction = intent.treatment()
                        .allInstructions().stream().filter(
                                instruction -> instruction instanceof
                                        L2ModificationInstruction.ModVlanHeaderInstruction)
                        .map(x -> (L2ModificationInstruction.ModVlanHeaderInstruction) x).findAny();

                if (!modVlanIdInstruction.isPresent() && !popVlanInstruction.isPresent()) {
                    if (vlanCriterion.isPresent()) {
                        egressTreat.setVlanId(vlanCriterion.get().vlanId());
                    } else {
                        egressTreat.popVlan();
                    }
                }

                creator.createFlow(egressSelector,
                                   egressTreat.build(), prev, link.src(),
                                   intent.priority(), true, flows, devices);
            }
        }
    }

    /**
     * Creates the flow rules for the path intent using MPLS
     * encapsulation.
     *
     * @param creator the flowrules creator
     * @param flows the list of flows to fill
     * @param devices the devices on the path
     * @param intent the PathIntent to compile
     */
    private void manageMplsEncap(PathCompilerCreateFlow<T> creator, List<T> flows,
                                           List<DeviceId> devices,
                                           PathIntent intent) {

        Set<Link> linksSet = Sets.newConcurrentHashSet();
        for (int i = 1; i <= intent.path().links().size() - 2; i++) {
            linksSet.add(intent.path().links().get(i));
        }

        Map<LinkKey, Identifier<?>> mplsLabels = labelAllocator.assignLabelToLinks(
                linksSet,
                intent.id(),
                EncapsulationType.MPLS
        );
        Iterator<Link> links = intent.path().links().iterator();
        Link srcLink = links.next();

        Link link = links.next();
        // List of flow rules to be installed

        // Ingress traffic
        MplsLabel mplsLabel = (MplsLabel) mplsLabels.get(linkKey(link));
        if (mplsLabel == null) {
            throw new IntentCompilationException(ERROR_MPLS + link);
        }
        MplsLabel prevMplsLabel = mplsLabel;

        Optional<MplsCriterion> mplsCriterion = intent.selector().criteria()
                .stream().filter(criterion -> criterion.type() == Criterion.Type.MPLS_LABEL)
                .map(criterion -> (MplsCriterion) criterion)
                .findAny();

        //Push MPLS if selector does not include MPLS
        TrafficTreatment.Builder treatBuilder = DefaultTrafficTreatment.builder();
        if (!mplsCriterion.isPresent()) {
            treatBuilder.pushMpls();
        }
        //Tag the traffic with the new encapsulation MPLS label
        treatBuilder.setMpls(mplsLabel);
        creator.createFlow(intent.selector(), treatBuilder.build(),
                srcLink.dst(), link.src(), intent.priority(), true, flows, devices);

        ConnectPoint prev = link.dst();

        while (links.hasNext()) {

            link = links.next();

            if (links.hasNext()) {
                // Transit traffic
                MplsLabel transitMplsLabel = (MplsLabel) mplsLabels.get(linkKey(link));
                if (transitMplsLabel == null) {
                    throw new IntentCompilationException(ERROR_MPLS + link);
                }
                TrafficSelector transitSelector = DefaultTrafficSelector.builder()
                        .matchInPort(prev.port())
                        .matchEthType(Ethernet.MPLS_UNICAST)
                        .matchMplsLabel(prevMplsLabel).build();

                TrafficTreatment.Builder transitTreat = DefaultTrafficTreatment.builder();

                // Set the new MPLS label only if the previous one is different
                if (!prevMplsLabel.equals(transitMplsLabel)) {
                    transitTreat.setMpls(transitMplsLabel);
                }
                creator.createFlow(transitSelector,
                        transitTreat.build(), prev, link.src(), intent.priority(), true, flows, devices);
                prevMplsLabel = transitMplsLabel;
                prev = link.dst();
            } else {
                TrafficSelector.Builder egressSelector = DefaultTrafficSelector.builder()
                        .matchInPort(prev.port())
                        .matchEthType(Ethernet.MPLS_UNICAST)
                        .matchMplsLabel(prevMplsLabel);
                TrafficTreatment.Builder egressTreat = DefaultTrafficTreatment.builder(intent.treatment());

                // Egress traffic
                // check if the treatement is popVlan or setVlan (rewrite),
                // than selector needs to match any VlanId
                for (Instruction instruct : intent.treatment().allInstructions()) {
                    if (instruct instanceof L2ModificationInstruction) {
                        L2ModificationInstruction l2Mod = (L2ModificationInstruction) instruct;
                        if (l2Mod.subtype() == L2ModificationInstruction.L2SubType.VLAN_PUSH) {
                            break;
                        }
                        if (l2Mod.subtype() == L2ModificationInstruction.L2SubType.VLAN_POP ||
                                l2Mod.subtype() == L2ModificationInstruction.L2SubType.VLAN_ID) {
                            egressSelector.matchVlanId(VlanId.ANY);
                        }
                    }
                }

                if (mplsCriterion.isPresent()) {
                    egressTreat.setMpls(mplsCriterion.get().label());
                } else {
                    egressTreat.popMpls(getEthType(intent.selector()));
                }

                creator.createFlow(egressSelector.build(),
                        egressTreat.build(), prev, link.src(), intent.priority(), true, flows, devices);
            }

        }

    }

    /**
     * Compiles an intent down to flows.
     *
     * @param creator how to create the flows
     * @param intent intent to process
     * @param flows list of generated flows
     * @param devices list of devices that correspond to the flows
     */
    public void compile(PathCompilerCreateFlow<T> creator,
                        PathIntent intent,
                        List<T> flows,
                        List<DeviceId> devices) {
        // Note: right now recompile is not considered
        // TODO: implement recompile behavior

        List<Link> links = intent.path().links();

        Optional<EncapsulationConstraint> encapConstraint = intent.constraints().stream()
                .filter(constraint -> constraint instanceof EncapsulationConstraint)
                .map(x -> (EncapsulationConstraint) x).findAny();
        //if no encapsulation or is involved only a single switch use the default behaviour
        if (!encapConstraint.isPresent() || links.size() == 2) {
            for (int i = 0; i < links.size() - 1; i++) {
                ConnectPoint ingress = links.get(i).dst();
                ConnectPoint egress = links.get(i + 1).src();
                creator.createFlow(intent.selector(), intent.treatment(),
                                   ingress, egress, intent.priority(),
                                   isLast(links, i), flows, devices);
            }
            return;
        }

        encapConstraint.map(EncapsulationConstraint::encapType)
                .map(type -> {
                    switch (type) {
                        case VLAN:
                            manageVlanEncap(creator, flows, devices, intent);
                            break;
                        case MPLS:
                             manageMplsEncap(creator, flows, devices, intent);
                            break;
                        default:
                            // Nothing to do
                    }
                    return 0;
                });
    }

}


text:
using System;
using System.Collections.Generic;
using System.Linq.Expressions;

namespace GenericApi
{
    public interface IGenericRepository<T, TContext>
    {
        T Add(T entity);
        T AddOrUpdate(T entity);
        void Delete(dynamic id);
        T FindById(dynamic id, params Expression<Func<T, object>>[] includeProperties);
        List<T> GetAll(params Expression<Func<T, object>>[] includeProperties);
        List<T> Query(Func<T, bool> predicate, params Expression<Func<T, object>>[] includeProperties);
        T FindById(dynamic id, string[] includeProperties);
        List<T> GetAll(string[] includeProperties);
        List<T> Query(Func<T, bool> predicate, string[] includeProperties);
        T Update(T entity);
    }
}


text:
from django.db.models import IntegerField, CharField

from zoogle.core.models import BaseModel

class Zdoc(BaseModel):
    size = IntegerField()
    owner = CharField(max_length=100)