text:
__author__ = 'JGJeffryes'

from pymongo import MongoClient
import sys
import platform
import time
from optparse import OptionParser
from minedatabase.queries import quick_search
from minedatabase.databases import establish_db_client

"""
def write_pathway_html(db, path, outfile):
    html_printer = Printer(db)
    for _id in path:
        if _id[0] == 'C':
            html_printer.print_compound_html(_id, outfile, labels=['Names', 'KEGG_code'])

        if _id[0] == 'R':
            html_printer.print_rxn_html(_id, outfile)"""

class PathwaySearch():
    def __init__(self, options):
        client = establish_db_client()
        self.paths = []
        self.db = client[options.db]
        self.options = options
        self.native_comps = set()
        self.discovered = set()

    def __compound_tests(self, compound):
        if not compound:
                if self.options.verbose:
                    print "Could not find " + compound['_id']
                return False
        try:
            if float(compound['NP_likeness']) <= self.options.np_min:
                if self.options.verbose:
                    print "%s excluded for insufficient natural product likeness" % compound['_id']
                return False
        except KeyError:
            pass
        return True

    def __reaction_checks(self, reaction):
        if not reaction:
            if options.verbose:
                print "Could not find " + reaction['_id']
            return False
        try:
            if float(reaction['Energy']) >= self.options.gibbs_cap:
                if options.verbose:
                    print "%s excluded for thermodynamics" % reaction['_id']
                return False
        except (KeyError, ValueError):
            pass
        return True

    def dfs(self):
        self._DFS_ittr(self.options.start_comp, [])
        return sorted(self.paths, key=lambda x: len(x))

    def _DFS_ittr(self, comp_id, path):
        if comp_id == self.options.end_comp:
            self.paths.append(path+[comp_id])
            if not self.options.all_paths:
                return self.paths
        elif len(path)/2 < self.options.len_limit:
            compound = self.db.compounds.find_one({"_id": comp_id}, {'NP_likeness': 1, 'Reactant_in': 1})
            if self.__compound_tests(compound):
                try:
                    for rxn_id in compound["Reactant_in"]:
                        reaction = self.db.reactions.find_one({"_id": rxn_id}, {'Products': 1, 'Energy': 1})
                        if self.__reaction_checks(reaction):
                            for prod_tuple in reaction['Products']:
                                if prod_tuple[1][0] == "C":
                                    self._DFS_ittr(prod_tuple[1], path + [comp_id, rxn_id])
                except KeyError:
                    if self.options.verbose:
                        print "Dead End"
        else:
            if self.options.verbose:
                print "Max Depth"

    def bfs(self):
        self.__load_queue(self.options.start_comp)
        while len(self.queue) > 0:
            #remove a compound from the queue and add add to discovered. if the path length is at the limit we break
            compound = self.queue.pop(0)
            #the depth is half the path length because we only count reactions
            depth = len(compound['Path']) / 2
            if self.options.verbose:
                print "Depth: %s\tQueue:%s" % (depth, len(self.queue))
            if compound['_id'] == self.options.end_comp:
                #save the path
                self.paths.append(compound['Path'] + [compound['_id']])
                #continue or exit depending on the multi flag
                if not self.options.all_paths:
                    return self.paths
            if depth < self.options.len_limit:
                #look at the products for every reaction in which the start compound is a reactant
                try:
                    for rxn_id in compound["Reactant_in"]:
                        for product_id in self.__get_products(rxn_id):
                            product = self.db.compounds.find_one({"_id": product_id}, {'NP_likeness': 1, 'Reactant_in': 1})
                            #if the product passes, record the path taken to reach the new node and enqueue
                            if self.__compound_tests(product):
                                product['Path'] = compound['Path'] + [compound['_id'], rxn_id]
                                self.queue.append(product)
                except KeyError:
                    if self.options.verbose:
                        print "Dead End"

        return self.paths

    def __load_queue(self, start_id):
        self.queue = []
        #look at the products for every reaction in which the start compound is a reactant and add all the non-cofactors
        # to the queue.
        self.discovered.add(start_id)
        start_dict = self.db.compounds.find_one({"_id": start_id})
        for rxn_id in start_dict["Reactant_in"]:
            reaction = self.db.reactions.find_one({"_id": rxn_id})
            for comp_tuple in reaction["Products"]:
                #check if compound is a cofactor by changing the first part of its _id
                if comp_tuple[1][0] == "C":
                    #the second part of the tuple has the id, thus the 1 subscript
                    compound = self.db.compounds.find_one({"_id": str(comp_tuple[1])})
                    #Start a path field to keep track of all the steps that brought us to the compound
                    compound['Path'] = [start_id, rxn_id]
                    self.discovered.add(compound['_id'])
                    self.queue.append(compound)

    def __get_products(self, rxn_id):
        products = []
        reaction = self.db.reactions.find_one({"_id": rxn_id}, {'Products': 1, 'Energy': 1})
        #reaction passes tests, return all products that haven't been discovered and are not cofactors
        if self.__reaction_checks(reaction):
            for comp_tuple in reaction["Products"]:
                if (comp_tuple[1] not in self.discovered) and (comp_tuple[1][0] == "C"):
                    self.discovered.add(comp_tuple[1])
                    products.append(comp_tuple[1])
        return products

if __name__ == '__main__':
    start_time = time.time()
    #This block handles user flags and arguments. For more information see the optparse API documentation

    usage = "usage: %prog [options] run_name"
    parser = OptionParser(usage)
    parser.add_option("-d", "--database", dest="db", default="1GenEcoCyc",
                      help="The name of the database to search")
    parser.add_option("-s", "--start", dest="start_comp", default="Cb5b3273ab083d77ed29fbef8f7e464929af29c13",
                      help="Specify the starting compound for a search, defaults to glucose")
    parser.add_option("-e", "--end", dest="end_comp", default="C89b394fd02e5e5e60ae1e167780ea7ab3276288e",
                      help="Specify the ending compound for a search, defaults to ethanol")
    parser.add_option("-l", "--length", dest="len_limit", type="int", default=3,
                      help="Maximum length of pathways to return, defaults to 3 reactions")
    parser.add_option("-n", "--NP", dest="np_min", type="float", default=-3,
                      help="Set a floor on the minimum natural product likeness of compounds in a pathway")
    parser.add_option("-g", "--GibbsCap", dest="gibbs_cap", type="float", default=1000,
                      help="Set a cap on the gibbs free energy of a reaction in a pathway")
    parser.add_option("-a", "--all_paths", dest="all_paths", action="store_true", default=False,
                      help="Find all valid paths shorter than length cap")
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False,
                      help="Print verbose output")
    (options, args) = parser.parse_args()

    client = establish_db_client()
    db = client[options.db]

    options.start_comp = quick_search(db, options.start_comp)[0]['_id']
    options.end_comp = quick_search(db, options.end_comp)[0]['_id']

    search = PathwaySearch(options)
    if options.all_paths:
        paths = search.dfs()
    else:
        paths = search.bfs()

    for i, path in enumerate(paths):
        """if args:
            with open(args[0]+'%s.html' % i, 'w') as outfile:
                write_pathway_html(db, path, outfile)
        else:"""
        print(', '.join(str(x) for x in path))

    print 'Found %s paths shorter than %s reactions' % (len(paths), options.len_limit)
    print 'Script complete, execution took %s sec' %(time.time() - start_time)


text:
#region Copyright 2018 D-Haven.org
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#endregion

using System.Threading.Tasks;

namespace DHaven.Faux.Test.ReturnTypes
{
    public class Value
    {
        public string Content { get; set; }
        public bool IsValid { get; set; }
    }

    [FauxClient("return")]
    public interface IReturnService
    {
        [HttpPut]
        [return:ResponseHeader("Location")]
        Task<string> PutAsync();

        [HttpPut]
        [return:ResponseHeader("Location")]
        string Put();

        [HttpPost("echo")]
        [return: Body(Format = Format.Json)]
        Task<Value> EchoAsync([Body] Value val);

        [HttpPost("echo")]
        [return: Body(Format = Format.Json)]
        Value Echo([Body] Value val);

        [HttpGet("{id}")]
        [return:Body]
        Value Get([PathValue] int id,
            [ResponseHeader("Location")] out string location,
            [ResponseHeader("Content-Type")] out string mimeType);
    }
}

redpajama_set_name: RedPajamaGithub


text:
<Addin namespace="FSpot"
    category="Services"
    author="Yavor Georgiev"
    name="UPnP Photo Sharing"
    defaultEnabled="true">
        <Dependencies>
            <Addin id="Core" version="0.7"/>
        </Dependencies>
        <Extension path = "/FSpot/Services">
            <Service class="Mono.Upnp.Dcp.MediaServer1.FSpot.FSpotUpnpService"/>
        </Extension>
        <Extension path = "/FSpot/Sidebar">
            <SidebarPage sidebar_page_type="Mono.Upnp.Dcp.MediaServer1.FSpot.FSpotUpnpSidebarPage"/>
        </Extension>
        <Extension path = "/FSpot/Menus/Tools">
            <Command id = "UpnpSharingConfig" _label = "Configure UPnP Sharing" command_type = "Mono.Upnp.Dcp.MediaServer1.FSpot.SharingConfig" />
        </Extension>
</Addin>


text:
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<!--This file represents the results of running a test suite-->
<test-results name="C:\Program Files\NUnit 2.6\bin\tests\mock-assembly.dll" total="21" errors="1" failures="1" not-run="7" inconclusive="1" ignored="4" skipped="0" invalid="3" date="2012-02-04" time="11:46:05">
  <environment nunit-version="2.6.0.12035" clr-version="2.0.50727.4963" os-version="Microsoft Windows NT 6.1.7600.0" platform="Win32NT" cwd="C:\Program Files\NUnit 2.6\bin" machine-name="CHARLIE-LAPTOP" user="charlie" user-domain="charlie-laptop" />
  <culture-info current-culture="en-US" current-uiculture="en-US" />
  <test-suite type="Assembly" name="C:\Program Files\NUnit 2.6\bin\tests\mock-assembly.dll" executed="True" result="Failure" success="False" time="0.094" asserts="0">
    <results>
      <test-suite type="Namespace" name="NUnit" executed="True" result="Failure" success="False" time="0.078" asserts="0">
        <results>
          <test-suite type="Namespace" name="Tests" executed="True" result="Failure" success="False" time="0.078" asserts="0">
            <results>
              <test-suite type="Namespace" name="Assemblies" executed="True" result="Failure" success="False" time="0.031" asserts="0">
                <results>
                  <test-suite type="TestFixture" name="MockTestFixture" description="Fake Test Fixture" executed="True" result="Failure" success="False" time="0.031" asserts="0">
                    <categories>
                      <category name="FixtureCategory" />
                    </categories>
                    <results>
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.FailingTest" executed="True" result="Failure" success="False" time="0.016" asserts="0">
                        <failure>
                          <message><![CDATA[Intentional failure]]></message>
                          <stack-trace><![CDATA[at NUnit.Tests.Assemblies.MockTestFixture.FailingTest()
]]></stack-trace>
                        </failure>
                      </test-case>
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.InconclusiveTest" executed="True" result="Inconclusive" success="False" time="0.000" asserts="0">
                        <reason>
                          <message><![CDATA[No valid data]]></message>
                        </reason>
                      </test-case>
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.MockTest1" description="Mock Test #1" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.MockTest2" description="This is a really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really, really long description" executed="True" result="Success" success="True" time="0.000" asserts="0">
                        <categories>
                          <category name="MockCategory" />
                        </categories>
                        <properties>
                          <property name="Severity" value="Critical" />
                        </properties>
                      </test-case>
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.MockTest3" executed="True" result="Success" success="True" time="0.016" asserts="0">
                        <categories>
                          <category name="AnotherCategory" />
                          <category name="MockCategory" />
                        </categories>
                        <reason>
                          <message><![CDATA[Succeeded!]]></message>
                        </reason>
                      </test-case>
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.MockTest4" executed="False" result="Ignored">
                        <categories>
                          <category name="Foo" />
                        </categories>
                        <reason>
                          <message><![CDATA[ignoring this test method for now]]></message>
                        </reason>
                      </test-case>
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.MockTest5" executed="False" result="NotRunnable">
                        <reason>
                          <message><![CDATA[Method is not public]]></message>
                        </reason>
                      </test-case>
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.NotRunnableTest" executed="False" result="NotRunnable">
                        <reason>
                          <message><![CDATA[No arguments were provided]]></message>
                        </reason>
                      </test-case>
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.TestWithException" executed="True" result="Error" success="False" time="0.000" asserts="0">
                        <failure>
                          <message><![CDATA[System.ApplicationException : Intentional Exception]]></message>
                          <stack-trace><![CDATA[at NUnit.Tests.Assemblies.MockTestFixture.MethodThrowsException()
at NUnit.Tests.Assemblies.MockTestFixture.TestWithException()
]]></stack-trace>
                        </failure>
                      </test-case>
                      <test-case name="NUnit.Tests.Assemblies.MockTestFixture.TestWithManyProperties" executed="True" result="Success" success="True" time="0.000" asserts="0">
                        <properties>
                          <property name="Size" value="5" />
                          <property name="TargetMethod" value="SomeClassName" />
                        </properties>
                      </test-case>
                    </results>
                  </test-suite>
                </results>
              </test-suite>
              <test-suite type="TestFixture" name="BadFixture" executed="False" result="NotRunnable">
                <reason>
                  <message><![CDATA[No suitable constructor was found]]></message>
                </reason>
                <results>
                  <test-case name="NUnit.Tests.BadFixture.SomeTest" executed="False" result="NotRunnable">
                    <reason>
                      <message><![CDATA[No suitable constructor was found]]></message>
                    </reason>
                  </test-case>
                </results>
              </test-suite>
              <test-suite type="TestFixture" name="FixtureWithTestCases" executed="True" result="Success" success="True" time="0.000" asserts="0">
                <results>
                  <test-suite type="ParameterizedTest" name="GenericMethod" executed="True" result="Success" success="True" time="0.000" asserts="0">
                    <results>
                      <test-case name="NUnit.Tests.FixtureWithTestCases.GenericMethod<Int32>(2,4)" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                      <test-case name="NUnit.Tests.FixtureWithTestCases.GenericMethod<Double>(9.2d,11.7d)" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                    </results>
                  </test-suite>
                  <test-suite type="ParameterizedTest" name="MethodWithParameters" executed="True" result="Success" success="True" time="0.000" asserts="0">
                    <results>
                      <test-case name="NUnit.Tests.FixtureWithTestCases.MethodWithParameters(2,2)" executed="True" result="Success" success="True" time="0.000" asserts="1" />
                      <test-case name="NUnit.Tests.FixtureWithTestCases.MethodWithParameters(9,11)" executed="True" result="Success" success="True" time="0.000" asserts="1" />
                    </results>
                  </test-suite>
                </results>
              </test-suite>
              <test-suite type="GenericFixture" name="GenericFixture<T>" executed="True" result="Success" success="True" time="0.000" asserts="0">
                <results>
                  <test-suite type="TestFixture" name="GenericFixture<Double>(11.5d)" executed="True" result="Success" success="True" time="0.000" asserts="0">
                    <results>
                      <test-case name="NUnit.Tests.GenericFixture<Double>(11.5d).Test1" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                      <test-case name="NUnit.Tests.GenericFixture<Double>(11.5d).Test2" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                    </results>
                  </test-suite>
                  <test-suite type="TestFixture" name="GenericFixture<Int32>(5)" executed="True" result="Success" success="True" time="0.000" asserts="0">
                    <results>
                      <test-case name="NUnit.Tests.GenericFixture<Int32>(5).Test1" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                      <test-case name="NUnit.Tests.GenericFixture<Int32>(5).Test2" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                    </results>
                  </test-suite>
                </results>
              </test-suite>
              <test-suite type="TestFixture" name="IgnoredFixture" executed="False" result="Ignored">
                <reason>
                  <message><![CDATA[]]></message>
                </reason>
                <results>
                  <test-case name="NUnit.Tests.IgnoredFixture.Test1" executed="False" result="Ignored">
                    <reason>
                      <message><![CDATA[]]></message>
                    </reason>
                  </test-case>
                  <test-case name="NUnit.Tests.IgnoredFixture.Test2" executed="False" result="Ignored">
                    <reason>
                      <message><![CDATA[]]></message>
                    </reason>
                  </test-case>
                  <test-case name="NUnit.Tests.IgnoredFixture.Test3" executed="False" result="Ignored">
                    <reason>
                      <message><![CDATA[]]></message>
                    </reason>
                  </test-case>
                </results>
              </test-suite>
              <test-suite type="ParameterizedFixture" name="ParameterizedFixture" executed="True" result="Success" success="True" time="0.000" asserts="0">
                <results>
                  <test-suite type="TestFixture" name="ParameterizedFixture(42)" executed="True" result="Success" success="True" time="0.000" asserts="0">
                    <results>
                      <test-case name="NUnit.Tests.ParameterizedFixture(42).Test1" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                      <test-case name="NUnit.Tests.ParameterizedFixture(42).Test2" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                    </results>
                  </test-suite>
                  <test-suite type="TestFixture" name="ParameterizedFixture(5)" executed="True" result="Success" success="True" time="0.000" asserts="0">
                    <results>
                      <test-case name="NUnit.Tests.ParameterizedFixture(5).Test1" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                      <test-case name="NUnit.Tests.ParameterizedFixture(5).Test2" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                    </results>
                  </test-suite>
                </results>
              </test-suite>
              <test-suite type="Namespace" name="Singletons" executed="True" result="Success" success="True" time="0.016" asserts="0">
                <results>
                  <test-suite type="TestFixture" name="OneTestCase" executed="True" result="Success" success="True" time="0.016" asserts="0">
                    <results>
                      <test-case name="NUnit.Tests.Singletons.OneTestCase.TestCase" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                    </results>
                  </test-suite>
                </results>
              </test-suite>
              <test-suite type="Namespace" name="TestAssembly" executed="True" result="Success" success="True" time="0.000" asserts="0">
                <results>
                  <test-suite type="TestFixture" name="MockTestFixture" executed="True" result="Success" success="True" time="0.000" asserts="0">
                    <results>
                      <test-case name="NUnit.Tests.TestAssembly.MockTestFixture.MyTest" executed="True" result="Success" success="True" time="0.000" asserts="0" />
                    </results>
                  </test-suite>
                </results>
              </test-suite>
            </results>
          </test-suite>
        </results>
      </test-suite>
    </results>
  </test-suite>
</test-results>


text:
<!DOCTYPE html>

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">

    <meta name="description" content="">
    <meta name="author" content="">
    <link rel="icon" href="../../favicon.ico">

    <title>Sora | Champion</title>

    <link href="css/bootstrap.min.css" rel="stylesheet">

    <link href="css/ie10-viewport-bug-workaround.css" rel="stylesheet">

    <link href="css/pages/page.css" rel="stylesheet">

          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
            Toggle navigation

          </button>
          Champion

Home

Rules

Challenge

Join

Leaders and E4

Frontier

Champions

A-Team

Badges

Formats

Hall of Fame

Champion

Currently serving Champion(s).

					<text class="text-muted">Art: Metatyph</text>

Sora Revan

					Rule

					Gen 7 OU

					"The world could always use more heroes!"

					Ace: All


text:
package org.netbeans.cubeon.local.query;

import java.util.Collection;
import org.netbeans.cubeon.tasks.spi.Notifier;
import org.netbeans.cubeon.tasks.spi.query.TaskQueryEventAdapter;
import org.openide.util.Lookup;
import org.openide.util.lookup.Lookups;

/**
 *
 * @author Anuradha
 */
public class QueryExtension extends  Notifier<TaskQueryEventAdapter> {

    private LocalQuery query;

    public QueryExtension(LocalQuery query) {
        this.query = query;

    }

    public Lookup getLookup() {
        return Lookups.singleton(this);
    }

    //events---------------------------
    void fireAttributesUpdated() {
        Collection<? extends TaskQueryEventAdapter> adapters = getAll();
        for (TaskQueryEventAdapter adapter : adapters) {
            adapter.atributesupdated();
        }

    }

    void fireSynchronized() {
        Collection<? extends TaskQueryEventAdapter> adapters = getAll();
        for (TaskQueryEventAdapter adapter : adapters) {
            adapter.querySynchronized();
        }

    }
    void fireRemoved() {
        Collection<? extends TaskQueryEventAdapter> adapters = getAll();
        for (TaskQueryEventAdapter adapter : adapters) {
            adapter.removed();
        }

    }
}


text:
import { PropTypes } from "react";

export const ChapterShape = PropTypes.shape({
  id: PropTypes.number.isRequired,
  name: PropTypes.string.isRequired,
  description: PropTypes.string,
  publicKey: PropTypes.string,
  membershipFee: PropTypes.number,
  currency: PropTypes.string,
  rides: PropTypes.arrayOf(PropTypes.number)
});


text:
package openmods.api;

public interface IAddAwareTile {
	public void onAdded();
}


text:
package cruise.umple.implementation.py;

import org.junit.*;
import cruise.umple.implementation.*;

public class PythonNToNTest extends NToNTest
{

  @Before
  public void setUp()
  {
    super.setUp();
    language = "Python";
    languagePath = "py";
  }
}


text:
package utils;

import com.bc.zarr.JZarrException;

import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Collections;

public class OutputHelper {
    public static final String ANSI_RESET = "\u001B[0m";
    public static final String ANSI_BLACK = "\u001B[30m";
    public static final String ANSI_RED = "\u001B[31m";
    public static final String ANSI_GREEN = "\u001B[32m";
    public static final String ANSI_YELLOW = "\u001B[33m";
    public static final String ANSI_BLUE = "\u001B[34m";
    public static final String ANSI_PURPLE = "\u001B[35m";
    public static final String ANSI_CYAN = "\u001B[36m";
    public static final String ANSI_WHITE = "\u001B[37m";

    public static final String ANSI_BLACK_BACKGROUND = "\u001B[40m";
    public static final String ANSI_RED_BACKGROUND = "\u001B[41m";
    public static final String ANSI_GREEN_BACKGROUND = "\u001B[42m";
    public static final String ANSI_YELLOW_BACKGROUND = "\u001B[43m";
    public static final String ANSI_BLUE_BACKGROUND = "\u001B[44m";
    public static final String ANSI_PURPLE_BACKGROUND = "\u001B[45m";
    public static final String ANSI_CYAN_BACKGROUND = "\u001B[46m";
    public static final String ANSI_WHITE_BACKGROUND = "\u001B[47m";

    public static void printHeadline() {
        final StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
        final String methodName = stackTrace[2].getMethodName();
        final String[] strings = splitCamelCase(methodName);
        final String headline = String.join(" ", strings);
        final int n = headline.trim().length();

        System.out.println();
        System.out.print(ANSI_CYAN_BACKGROUND);
        System.out.print(ANSI_BLACK);
        System.out.println(headline);
        System.out.println(String.join("", Collections.nCopies(n, "=")));
        System.out.println(ANSI_RESET); // reset
    }

    public static void printMethodSeparator() {
        System.out.println();
        System.out.println("------------------------------------------------------------------");
        System.out.println();
    }

    private static String[] splitCamelCase(String string) {
        return string.split("(?<!^)(?=[A-Z])");
    }

    public static void createOutput(Writer writer) throws IOException, JZarrException {
        final String fileName = createOutputFilename();
        String methodName = getOutputMethodName();
        final Path workDir = Paths.get(".");
        final Path outDir = workDir.resolve("docs").resolve("examples").resolve("output");
        Files.createDirectories(outDir);
        final Path filePath = outDir.resolve(fileName);
        if (!Files.exists(filePath)) {
            OutputStream stream = Files.newOutputStream(filePath);
            stream.close();
        }
        try (OutputStream outputStream = Files.newOutputStream(filePath, StandardOpenOption.APPEND);
             PrintStream printStream = new PrintStream(outputStream)) {
            printStream.println(methodName + "_output_start");
            writer.write(printStream);
            printStream.println(methodName + "__output_end__");
            printStream.println();
            printStream.println("=====================================================================================");
            printStream.println();
        }
    }

    private static String createOutputFilename() {
        final StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
        final StackTraceElement element = stackTrace[3];
        String className = element.getClassName();
        if (className.contains(".")) {
            className = className.substring(className.lastIndexOf(".") + 1);
        }
        return className + ".txt";
    }

    private static String getOutputMethodName() {
        final StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
        final StackTraceElement element = stackTrace[3];
        return element.getMethodName();
    }

    public interface Writer {
        void write(PrintStream out) throws IOException, JZarrException;
    }
}


text:
layout: post
title: make npm3 install look like npm2
date: '2016-03-08 10:01:58'
---

Got this tip from a friend - [rico sta. cruz](http://ricostacruz.com/):

Just set the config to use `loglevel http` and not show the progress bar `progress false` and you're good to go.

`npm config set loglevel http && npm config set progress false`


text:
[![Build Status](https://travis-ci.org/kmonsoor/str2bool.svg?branch=master)](https://travis-ci.org/kmonsoor/str2bool)
[![License](https://img.shields.io/badge/license-MIT-blue.svg)](http://kmonsoor.mit-license.org/)

# str2bool  :  String to Boolean conversion

a simple multi-lingual utility function to convert a string to boolean i.e. True, False,
by guessing the approximate meaning of the input string.
By default, it will return `False`. Return ``True`` ONLY if input is certainly a 'true'-ish word e.g. 'Positive', 'yeah' etc.

Maybe, someday it will be able to detect "boolean"-lity of large sentences.
But, now it can sense only single AND basic words in many languages.

### Currently supported languages:
 * English
 * Bengali
 * Dutch
 * Polish
 * Arabic
 * Chinese
 * Korean
 * Spanish
 * Ukrainian
   And, some more ...

Please consider adding your language-specific "yes"|"True"-ish words. See **Contribution** section below.

## Installation
    pip install git+git://github.com/kmonsoor/str2bool.git

## Example usage
    >>> import str2bool as s2b
    >>> print(s2b.supported_languages)
    ['EN', 'BN', 'AR', 'NL', 'ES', 'KO', 'JA', 'ZH', 'TL', 'VI']
    >>> s2b.string_to_boolean('True')
    True
    >>> s2b.string_to_boolean('0')
    False
    >>> s2b.string_to_boolean('Tak')  # polish
    True
    >>> s2b.string_to_boolean('না')   # bengali "no"
    False
    >>> s2b.string_to_boolean('joe is a good boy')
    Traceback (most recent call last):
      File "C:\Program Files (x86)\JetBrains\PyCharm 4.0.5\helpers\pycharm\docrunner.py", line 135, in __run
       compileflags, 1), test.globs)
      File "<doctest str2bool[6]>", line 1, in <module>
       s2b.string_to_boolean('joe is a good boy')
      File "C:/Users/k/PycharmProjects/str2bool/str2bool.py", line 51, in string_to_boolean
       raise ValueError('Only single words are supported in this alpha-version utility function.')
    ValueError: Only single words are supported in this alpha-version utility function.

## Contribution

 * To word new list in the "True-ish" checklist, please:

   1. Append the "yes"|"True"-ish word(s) at the end of the ``truish_words`` list,
   2. update ``supported_languages`` list, and
   3. create a pull-request.

 * Please add your suggestion, feature-request, constructive criticism, bug as __issue__ on this GitHub repo
 * Even your generic observation, word-of-wisdom, inspiration, or just "Thanks, buddy" is highly welcome. Please send them to my email: k (at) kmonsoor (dot) com
 * Also, I would love to know any use-case where this tiny library is helping. :)

## License
``str2bool`` is released as open-source software under the MIT License (MIT).


text:
<?php namespace DevelMe\Console;

use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputArgument;

/**
 * View Generator based on Schema
 * @package DevelMe\Console
 * @version 1.0.0
 * @author Verron Knowles <Verron.Knowles@develme.com>
 */
class ViewMakeCommand extends SchemaGeneratorCommand
{
	/**
	 * The console command name.
	 *
	 * @var string
	 */
	protected $name = 'make:schema-view';

	/**
	 * The console command description.
	 *
	 * @var string
	 */
	protected $description = 'Create a new resource view class based on MySQL\'s information_schema';

	/**
	 * The type of class being generated.
	 *
	 * @var string
	 */
	protected $type = 'View';

    /**
     * Ignore certain things
     *
     * @type Array
     */
    private $ignore_items = array('id', 'updated_at', 'created_at', 'deleted_at');

    /**
     * Stubs
     *
     * @type Array
     */
    protected $stub_names = array(
        'index',
        'create',
        'update',
        'read',
    );

	/**
	 * Get the stub file for the generator.
	 *
	 * @return string
	 */
	protected function getStub()
	{
		return $this->getStubDirectory() . 'base.stub';
	}

    /**
     * Get the stub directory for the generator
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return string
     */
    protected function getStubDirectory()
    {
        $theme = $this->option('theme');
        $theme_directory = __DIR__."/../themes/$theme/view/";
        $directory = !is_null($theme) && $this->files->isDirectory($theme_directory) ? $theme_directory : __DIR__.'/../stubs/view/';

        return $directory;
    }

	/**
	 * Parse the name and format according to the root namespace.
	 *
	 * @param  string  $name
	 * @return string
	 */
	protected function parseName($name)
	{
        return strtolower(snake_case(class_basename($this->original_name))) . "/" .strtolower($name);
	}

    /**
     * Override Get App Namespace command
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return string
     */
    protected function getPath($name)
    {
        $path = $this->option('path');
        $view_name = "$name.blade.php";
        $base_path = str_finish($this->laravel->basePath(), "/");
        return is_null($path) ? "${base_path}resources/views/${view_name}" : $base_path . str_finish($path, '/') . $view_name;
    }

	/**
	 * Get the desired class name from the input.
	 *
	 * @return string
	 */
	protected function getNameInput()
	{
		return $this->current_name;
	}

    /**
     * Fire event overload for GeneratorCommand
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return void
     */
    public function fire()
    {
        $this->original_name = $original_name = $this->argument('name');
        $stubs = $this->stub_names;

        foreach ($stubs as $stub_name) {
            $this->current_name = $stub_name;

		    $this->info("Creating $stub_name view..");
            parent::fire();
        }
    }

    /**
     * Get contents for base.stub
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return string
     */
    protected function buildClassContents($name, $stub_data)
    {
        $table_name = $this->getTableName($name);
        $simple_name = str_singular($table_name);
        $proper_name = $this->getProperName($simple_name);
        $model_name = $name;
        $layout_master = "layouts.master";
        $layout_content = "content";

        // Build replacement tags
        $replacements = $this->sqlSafeIndexes(compact('proper_name', 'simple_name', 'model_name', 'table_name', 'layout_master', 'layout_content'));
        $this->otherTags = $this->sqlSafeIndexes(compact('layout_master', 'layout_content'));

        $stub_contents = "";
        $view = $this->current_name;
        $stub_location = $this->getStubDirectory() . "$view.stub";
        if ($this->files->isFile($stub_location)) {
            $view_replacements = $this->generateColumnInfo($view, $stub_data, $replacements);
            $stub_contents .= trim(file_get_contents($stub_location)). "\n";
        }
        //dd($replacements);
        $replacements = array_merge($replacements, $view_replacements);
        $this->replaceTag($stub_contents, $replacements);

        return $stub_contents;
    }

    /**
     * Keep column titles from SQL table undamaged
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return Array
     */
    private function sqlSafeIndexes($private_array)
    {
        $change_keys = array();
        foreach ($private_array as $pkey => $pval)
            $change_keys["&$pkey"] = $pval;

        return $change_keys;
    }

    /**
     * Generate column information
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return string
     */
    protected function generateColumnInfo($stub, $column_info, $additional_info = array())
    {
        // Setup fillable namespace
        $fillable = array();
        $data_heading = array();
        $data_body = array();

        foreach ($column_info as $columnkey => $column) {
            if (empty(strpos($columnkey, '&'))) {
                if (!in_array($column['name'], $this->ignore_items, $column['name'])) {
                    $data_file = $this->getColumnFileData($column);

                    $column = array_merge($column, $additional_info, $this->sqlSafeIndexes(compact('data_file')));
                    $column_name = $column['name'];

                    $data_heading[] = $this->getColumnStubData($stub, 'heading', $column);
                    $data_body[] = $this->getColumnStubData($stub, 'body', $column);
                    $fillable[] = "\"$column_name\"";
                }
            }
        }

        // Build Data_Body combo
        $data_head_body = array();
        foreach ($data_heading as $dkey => $dvalue)
            $data_head_body[] = "$dvalue$data_body[$dkey]";

        $fillable = implode(",", $fillable);
        $data_heading = trim(implode("", $data_heading));
        $data_body = trim(implode("", $data_body));
        $data_head_body = trim(implode("", $data_head_body));

        // Build replacement tags
        return $this->sqlSafeIndexes(compact('fillable', 'data_heading', 'data_body', 'data_head_body'));
    }

    /**
     * getColumnStubData
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return string
     */
    protected function getColumnStubData($stub = '', $section, $column)
    {
        $column_data = $this->getStubSection($stub, $section);
        $this->replaceTag($column_data, $column);

        return $column_data;
    }

    /**
     * Get column data from column's &file attribute
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return string
     */
    protected function getColumnFileData($column)
    {
        $column_data = $this->getStubFile($column['&file']);

        // Fix up required
        $column['required'] = $column['required'] ? "required" : "";

        $this->replaceTag($column_data, $column);

        return $column_data;
    }

    /**
     * Get stub heading data
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return string
     */
    protected function getStubSection($stub, $section)
    {
        $stub_location = $this->getStubDirectory() . "${stub}_data_${section}.stub";
        return file_get_contents($stub_location);
    }

    /**
     * Get stub heading data
     *
     * @author Verron Knowles <Verron.Knowles@develme.com>
     * @return string
     */
    protected function getStubFile($file)
    {
        $stub_location = $this->getStubDirectory() . "${file}";

        return $this->files->isFile($stub_location) ? trim(file_get_contents($stub_location)) : null;
    }

    /**
	 * Get the default namespace for the class.
	 *
	 * @param  string  $rootNamespace
	 * @return string
	 */
	protected function getDefaultNamespace($rootNamespace)
	{
		return 'resource/views';
	}

    /**
	 * Get the console command arguments.
	 *
	 * @return array
	 */
	protected function getOptions()
	{
		return array(
			array('table', null, InputOption::VALUE_REQUIRED, 'The name of the table'),
			array('theme', null, InputOption::VALUE_OPTIONAL, 'The theme for the view [bootstrap|foundation|custom]'),
			array('path', null, InputOption::VALUE_OPTIONAL, 'The path to store the views'),
		);
	}

	/**
	 * Get the console command arguments.
	 *
	 * @return array
	 */
	protected function getArguments()
	{
		return array(
			array('name', InputArgument::REQUIRED, 'The name of the model to use with view.'),
		);
	}

}


text:
package com.cjoop.client.d.service;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.cas.authentication.CasAssertionAuthenticationToken;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
import org.springframework.security.core.userdetails.AuthenticationUserDetailsService;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.core.userdetails.UsernameNotFoundException;

/**
 * Authenticate a user from the database.
 */
public class CustomUserDetailsService implements AuthenticationUserDetailsService<CasAssertionAuthenticationToken> {

	private final Logger log = LoggerFactory.getLogger(CustomUserDetailsService.class);

	private Set<String> admins;

	public CustomUserDetailsService() {
		super();
	}

	/**
	 * @param admins
	 */
	public CustomUserDetailsService(Set<String> admins) {
		super();
		this.admins = admins;
	}

	@Override
	public UserDetails loadUserDetails(CasAssertionAuthenticationToken token) throws UsernameNotFoundException {
		String login = token.getPrincipal().toString();
		String lowercaseLogin = login.toLowerCase();

		log.debug("Authenticating '{}'", login);
		List<GrantedAuthority> grantedAuthorities = new ArrayList<GrantedAuthority>();

		if (admins != null && admins.contains(lowercaseLogin)) {
			grantedAuthorities.add(new SimpleGrantedAuthority(AuthoritiesConstants.ADMIN));
		} else {
			grantedAuthorities.add(new GrantedAuthority() {
				private static final long serialVersionUID = 1L;

				@Override
				public String getAuthority() {
					return AuthoritiesConstants.USER;
				}
			});
		}
		org.springframework.security.core.userdetails.User userdetails = new org.springframework.security.core.userdetails.User(
				lowercaseLogin, "", true, true, true, true, grantedAuthorities);
		return userdetails;

	}
}


text:
title: "Gallery | Exterior Stone Cladding | Niagara Riverhouse"
meta:
  description: "Beautifully illuminated up and down by over fifty spot and flood lights, the Niagra house and the Charcoal Rock Panels sparkle on cold New York nights."
permalink: /gallery/project/modern-stacked-stone-residence/
product: "The Niagara Residence"
pictured: rock-panel-charcoal
gallery:
- alt: Natural Stacked Stone Veneer Example
  src: /assets/images/gallery/Niagara-Residence-(24)_0.jpg
- alt: Natural Stacked Stone Veneer Example
  src: /assets/images/gallery/Niagara-Residence-(12)a.jpg
- alt: Natural Stacked Stone Veneer Example
  src: /assets/images/gallery/Niagara-Residence-(4)-copy.jpg
- alt: Natural Stacked Stone Veneer Example
  src: /assets/images/gallery/Niagara-Residence-(32).jpg
- alt: Natural Stacked Stone Veneer Example
  src: /assets/images/gallery/Niagara-Residence-(3).jpg
- alt: Natural Stacked Stone Veneer Example
  src: /assets/images/gallery/Niagara-Residence-(1).jpg
- alt: Natural Stacked Stone Veneer Example
  src: /assets/images/gallery/Niagara-Residence-(23).jpg
- alt: Natural Stacked Stone Veneer Example
  src: /assets/images/gallery/Niagara-Residence-(14).jpg
- alt: Natural Stacked Stone Veneer Example
  src: /assets/images/gallery/Niagara-Residence.jpg
other_galleries:
- rock-panel-exteriors
- xl-rock-panel-exteriors
- rock-panel-charcoal
- xl-rock-panel-charcoal
rows:
- image:
    alt: Niagara Home
    title: Niagara Home
    src: /assets/images/unsorted/Niagara.png
  heading: Norstone Rock Panels for Modern Facades
  body: The principal feature of the 1.85 acre site fronting the Niagara River just upstream from the Horseshoe Falls is the long unobstructed river views across the full 164 ft width of the property. The 4700 sq.ft of interior living space and 1200 sq.ft of covered terrace were crafted to embrace this feature and find a balance between the relative transparency encouraged by the views and the privacy concerns of the owners.
- image:
    alt: Charcoal Rock Panel Information
    title: Charcoal Rock Panel Information
    src: /assets/images/unsorted/stuff-to-know-2.png
  heading: Featured Products & Design
  body: "The Charcoal Rock Panel, paired with the large grey dimensional stone on the Niagra House, offer up a great contrast in both color and composition to the wood finish used extensively throughout the home's exterior doors, awnings and terrace walls. Beautifully illuminated up and down by over fifty spot and flood lights, the Niagra house and the Charcoal Rock Panels sparkle on cold New York nights.

Visit Product Page"
layout: gallery
redirect_from:
  - /gallery/project/Modern-Stacked-Stone-Residence
---

NIAGARA RIVERHOUSE IS THE NEW MODERN

{% for row in page.rows %}

{{ row.heading }}

{{ row.body }}

{% endfor %}


text:
/*!
 * \file roi_pooling-inl.h
 * \brief roi pooling operator and symbol
 * \author Kye-Hyeon Kim, Jian Guo
 */
#ifndef MXNET_OPERATOR_ROI_POOLING_INL_H_
#define MXNET_OPERATOR_ROI_POOLING_INL_H_

#include <dmlc/logging.h>
#include <dmlc/parameter.h>
#include <mxnet/operator.h>
#include <map>
#include <vector>
#include <string>
#include <utility>
#include "./mshadow_op.h"
#include "./operator_common.h"

namespace mxnet {
namespace op {

// Declare enumeration of input order to make code more intuitive.
// These enums are only visible within this header
namespace roipool {
enum ROIPoolingOpInputs { kData, kBox };
enum ROIPoolingOpOutputs { kOut, kMaxIdx };
}  // namespace roipool

struct ROIPoolingParam : public dmlc::Parameter<ROIPoolingParam> {
  mxnet::TShape pooled_size;
  float spatial_scale;
  DMLC_DECLARE_PARAMETER(ROIPoolingParam) {
    DMLC_DECLARE_FIELD(pooled_size)
        .set_expect_ndim(2)
        .enforce_nonzero()
        .describe("ROI pooling output shape (h,w) ");
    DMLC_DECLARE_FIELD(spatial_scale)
        .set_range(0.0, 1.0)
        .describe(
            "Ratio of input feature map height (or w) to raw image height (or w). "
            "Equals the reciprocal of total stride in convolutional layers");
  }
};

template <typename xpu, typename DType>
class ROIPoolingOp : public Operator {
 public:
  explicit ROIPoolingOp(ROIPoolingParam p) {
    this->param_ = p;
  }

  virtual void Forward(const OpContext& ctx,
                       const std::vector<TBlob>& in_data,
                       const std::vector<OpReqType>& req,
                       const std::vector<TBlob>& out_data,
                       const std::vector<TBlob>& aux_args) {
    using namespace mshadow;
    size_t expected = 2;
    CHECK_EQ(in_data.size(), expected);
    CHECK_EQ(out_data.size(), expected);
    CHECK_EQ(out_data[roipool::kOut].shape_[0], in_data[roipool::kBox].shape_[0]);
    CHECK_EQ(out_data[roipool::kMaxIdx].shape_[0], in_data[roipool::kBox].shape_[0]);
    Stream<xpu>* s = ctx.get_stream<xpu>();

    Tensor<xpu, 4, DType> data      = in_data[roipool::kData].get<xpu, 4, DType>(s);
    Tensor<xpu, 2, DType> bbox      = in_data[roipool::kBox].get<xpu, 2, DType>(s);
    Tensor<xpu, 4, DType> out       = out_data[roipool::kOut].get<xpu, 4, DType>(s);
    Tensor<xpu, 4, index_t> max_idx = out_data[roipool::kMaxIdx].get<xpu, 4, index_t>(s);
    CHECK_EQ(data.CheckContiguous(), true);
    CHECK_EQ(bbox.CheckContiguous(), true);
    CHECK_EQ(out.CheckContiguous(), true);
    CHECK_EQ(max_idx.CheckContiguous(), true);
    out     = -FLT_MAX;
    max_idx = -1.0f;
    ROIPoolForward(out, data, bbox, max_idx, param_.spatial_scale);
  }

  virtual void Backward(const OpContext& ctx,
                        const std::vector<TBlob>& out_grad,
                        const std::vector<TBlob>& in_data,
                        const std::vector<TBlob>& out_data,
                        const std::vector<OpReqType>& req,
                        const std::vector<TBlob>& in_grad,
                        const std::vector<TBlob>& aux_args) {
    using namespace mshadow;
    size_t expected = 2;
    CHECK_EQ(in_data.size(), expected);
    CHECK_EQ(out_data.size(), expected);
    CHECK_EQ(out_grad[roipool::kOut].shape_[0], in_data[roipool::kBox].shape_[0]);
    CHECK_EQ(out_data[roipool::kMaxIdx].shape_[0], in_data[roipool::kBox].shape_[0]);
    CHECK_NE(req[roipool::kData], kWriteInplace)
        << "ROIPooling: Backward doesn't support kWriteInplace.";
    CHECK_NE(req[roipool::kBox], kWriteInplace)
        << "ROIPooling: Backward doesn't support kWriteInplace.";
    Stream<xpu>* s = ctx.get_stream<xpu>();

    Tensor<xpu, 4, DType> grad_out  = out_grad[roipool::kOut].get<xpu, 4, DType>(s);
    Tensor<xpu, 2, DType> bbox      = in_data[roipool::kBox].get<xpu, 2, DType>(s);
    Tensor<xpu, 4, index_t> max_idx = out_data[roipool::kMaxIdx].get<xpu, 4, index_t>(s);
    Tensor<xpu, 4, DType> grad_in   = in_grad[roipool::kData].get<xpu, 4, DType>(s);
    Tensor<xpu, 2, DType> grad_roi  = in_grad[roipool::kBox].get<xpu, 2, DType>(s);
    CHECK_EQ(grad_out.CheckContiguous(), true);
    CHECK_EQ(bbox.CheckContiguous(), true);
    CHECK_EQ(max_idx.CheckContiguous(), true);
    CHECK_EQ(grad_in.CheckContiguous(), true);
    if (kAddTo == req[roipool::kData] || kWriteTo == req[roipool::kData]) {
      if (kWriteTo == req[roipool::kData]) {
        grad_in = 0.0f;
      }
      ROIPoolBackwardAcc(grad_in, grad_out, bbox, max_idx, param_.spatial_scale);
    }
    if (kWriteTo == req[roipool::kBox]) {
      grad_roi = 0.0f;
    }
  }

 private:
  ROIPoolingParam param_;
};  // class ROIPoolingOp

// Decalre Factory function, used for dispatch specialization
template <typename xpu>
Operator* CreateOp(ROIPoolingParam param, int dtype);

#if DMLC_USE_CXX11
class ROIPoolingProp : public OperatorProperty {
 public:
  std::vector<std::string> ListArguments() const override {
    return {"data", "rois"};
  }

  std::vector<std::string> ListOutputs() const override {
    return {"output", "maxidx"};
  }

  int NumOutputs() const override {
    return 2;
  }

  int NumVisibleOutputs() const override {
    return 1;
  }

  void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) override {
    param_.Init(kwargs);
  }

  std::map<std::string, std::string> GetParams() const override {
    return param_.__DICT__();
  }

  bool InferShape(mxnet::ShapeVector* in_shape,
                  mxnet::ShapeVector* out_shape,
                  mxnet::ShapeVector* aux_shape) const override {
    using namespace mshadow;
    CHECK_EQ(in_shape->size(), 2U) << "Input:[data, rois]";

    // data: [batch_size, c, h, w]
    mxnet::TShape dshape = in_shape->at(roipool::kData);
    CHECK_EQ(dshape.ndim(), 4U) << "data should be a 4D tensor";

    for (int i = 0; i < dshape.ndim(); i++) {
      CHECK_LT(dshape[i], INT32_MAX) << "ROI Pooling does not support large"
                                     << "dimensions (>= 2^31).";
    }

    // bbox: [num_rois, 5]
    mxnet::TShape bshape = in_shape->at(roipool::kBox);
    CHECK_EQ(bshape.ndim(), 2U) << "bbox should be a 2D tensor of shape [batch, 5]";
    CHECK_EQ(bshape[1], 5U) << "bbox should be a 2D tensor of shape [batch, 5]";

    // out: [num_rois, c, pooled_h, pooled_w]
    // max_idx: [num_rois, c, pooled_h, pooled_w]
    out_shape->clear();
    out_shape->push_back(
        Shape4(bshape[0], dshape[1], param_.pooled_size[0], param_.pooled_size[1]));
    out_shape->push_back(
        Shape4(bshape[0], dshape[1], param_.pooled_size[0], param_.pooled_size[1]));
    return true;
  }

  bool InferType(std::vector<int>* in_type,
                 std::vector<int>* out_type,
                 std::vector<int>* aux_type) const override {
    using namespace mshadow;
    CHECK_EQ(in_type->size(), 2U);
    int dtype = (*in_type)[0];
    CHECK_EQ(dtype, (*in_type)[1]);
    CHECK_NE(dtype, -1) << "Input must have specified type";

    out_type->clear();
    out_type->push_back(dtype);
#if MXNET_USE_INT64_TENSOR_SIZE == 1
    out_type->push_back(kInt64);
#else
    out_type->push_back(kInt32);
#endif
    return true;
  }

  OperatorProperty* Copy() const override {
    ROIPoolingProp* roi_pooling_sym = new ROIPoolingProp();
    roi_pooling_sym->param_         = this->param_;
    return roi_pooling_sym;
  }

  std::string TypeString() const override {
    return "ROIPooling";
  }

  // decalre dependency and inplace optimization options
  std::vector<int> DeclareBackwardDependency(const std::vector<int>& out_grad,
                                             const std::vector<int>& in_data,
                                             const std::vector<int>& out_data) const override {
    return {out_grad[roipool::kOut], in_data[roipool::kBox], out_data[roipool::kMaxIdx]};
  }

  Operator* CreateOperator(Context ctx) const override {
    LOG(FATAL) << "Not Implemented.";
    return nullptr;
  }

  Operator* CreateOperatorEx(Context ctx,
                             mxnet::ShapeVector* in_shape,
                             std::vector<int>* in_type) const override;

 private:
  ROIPoolingParam param_;
};  // class ROIPoolingProp
#endif
}  // namespace op
}  // namespace mxnet
#endif  // MXNET_OPERATOR_ROI_POOLING_INL_H_


text:
 .. Licensed to the Apache Software Foundation (ASF) under one
    or more contributor license agreements.  See the NOTICE file
    distributed with this work for additional information
    regarding copyright ownership.  The ASF licenses this file
    to you under the Apache License, Version 2.0 (the
    "License"); you may not use this file except in compliance
    with the License.  You may obtain a copy of the License at

 ..   http://www.apache.org/licenses/LICENSE-2.0

 .. Unless required by applicable law or agreed to in writing,
    software distributed under the License is distributed on an
    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    KIND, either express or implied.  See the License for the
    specific language governing permissions and limitations
    under the License.

.. _howto/connection:aws:

Amazon Web Services Connection
==============================

The Amazon Web Services connection type enables the :ref:`AWS Integrations
<AWS>`.

Authenticating to AWS
---------------------

Authentication may be performed using any of the `boto3 options <https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#configuring-credentials>`_. Alternatively, one can pass credentials in as a Connection initialisation parameter.

To use IAM instance profile, create an "empty" connection (i.e. one with no AWS Access Key ID or AWS Secret Access Key
specified, or ``aws://``).

Default Connection IDs
-----------------------

The default connection ID is ``aws_default``. If the environment/machine where you are running Airflow has the
file credentials in ``/home/.aws/``, and the default connection has user and pass fields empty, it will take
automatically the credentials from there.

.. note:: Previously, the ``aws_default`` connection had the "extras" field set to ``{"region_name": "us-east-1"}``
    on install. This means that by default the ``aws_default`` connection used the ``us-east-1`` region.
    This is no longer the case and the region needs to be set manually, either in the connection screens in Airflow,
    or via the ``AWS_DEFAULT_REGION`` environment variable.

.. _howto/connection:aws:configuring-the-connection:

Configuring the Connection
--------------------------

AWS Access Key ID (optional)
    Specify the AWS access key ID used for the initial connection.
    If you do an `assume role <https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html>`__
    by specifying a ``role_arn`` in the **Extra** field,
    then temporary credentials will be used for subsequent calls to AWS.

AWS Secret Access Key (optional)
    Specify the AWS secret access key used for the initial connection.
    If you do an `assume role <https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html>`__
    by specifying a ``role_arn`` in the **Extra** field,
    then temporary credentials will be used for subsequent calls to AWS.

Extra (optional)
    Specify the extra parameters (as json dictionary) that can be used in AWS
    connection. All parameters are optional.

    The following extra parameters used to create an initial
    `boto3.session.Session <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html>`__:

    * ``aws_access_key_id``: AWS access key ID used for the initial connection.
    * ``aws_secret_access_key``: AWS secret access key used for the initial connection
    * ``aws_session_token``: AWS session token used for the initial connection if you use external credentials.
      You are responsible for renewing these.
    * ``region_name``: AWS Region for the connection.
    * ``profile_name``: The name of a profile to use listed in
      `configuration and credential file settings <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-settings>`__.

    The following extra parameters used for `assume role <https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html>`__:

    * ``role_arn``: If specified, then assume this role, obtaining a set of temporary security credentials using the ``assume_role_method``.
    * ``assume_role_method``: AWS STS client method, one of
      `assume_role <https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html>`__,
      `assume_role_with_saml <https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithSAML.html>`__ or
      `assume_role_with_web_identity <https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html>`__
      if not specified then **assume_role** is used.
    * ``assume_role_kwargs``: Additional **kwargs** passed to ``assume_role_method``.

    The following extra parameters used for
    `boto3.client <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#boto3.session.Session.client>`__
    and `boto3.resource <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#boto3.session.Session.resource>`__:

    * ``config_kwargs``: Additional **kwargs** used to construct a
      `botocore.config.Config <https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html>`__.
    * ``endpoint_url``: Endpoint URL for the connection.
    * ``verify``: Whether or not to verify SSL certificates.

.. warning:: Extra parameters below are deprecated and will be removed in a future version of this provider.

    * ``aws_account_id``: Used to construct ``role_arn`` if it was not specified.
    * ``aws_iam_role``: Used to construct ``role_arn`` if it was not specified.
    * ``external_id``: A unique identifier that might be required when you assume a role in another account.
      Used if ``ExternalId`` in ``assume_role_kwargs`` was not specified.
    * ``s3_config_file``: Path to local credentials file.
    * ``s3_config_format``: ``s3_config_file`` format, one of
      `aws <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-settings>`_,
      `boto <http://boto.cloudhackers.com/en/latest/boto_config_tut.html#details>`_ or
      `s3cmd <https://s3tools.org/kb/item14.htm>`_ if not specified then **boto** is used.
    * ``profile``: If you are getting your credentials from the ``s3_config_file``
      you can specify the profile with this parameter.
    * ``host``: Used as connection's URL. Use ``endpoint_url`` instead.
    * ``session_kwargs``: Additional **kwargs** passed to
      `boto3.session.Session <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html>`__.

If you are configuring the connection via a URI, ensure that all components of the URI are URL-encoded.

Examples
--------

Snippet to create Connection and convert to URI
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  .. code-block:: python

    import os
    from airflow.models.connection import Connection

    conn = Connection(
        conn_id="sample_aws_connection",
        conn_type="aws",
        login="AKIAIOSFODNN7EXAMPLE",  # Reference to AWS Access Key ID
        password="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",  # Reference to AWS Secret Access Key
        extra={
            # Specify extra parameters here
            "region_name": "eu-central-1",
        },
    )

    # Generate Environment Variable Name and Connection URI
    env_key = f"AIRFLOW_CONN_{conn.conn_id.upper()}"
    conn_uri = conn.get_uri()
    print(f"{env_key}={conn_uri}")
    # AIRFLOW_CONN_SAMPLE_AWS_CONNECTION=aws://AKIAIOSFODNN7EXAMPLE:wJalrXUtnFEMI%2FK7MDENG%2FbPxRfiCYEXAMPLEKEY@/?region_name=eu-central-1

    # Test connection
    os.environ[env_key] = conn_uri
    print(conn.test_connection())

Using instance profile
^^^^^^^^^^^^^^^^^^^^^^

  This will use boto's default credential look-up chain (the profile named "default" from the ~/.boto/ config files,
  and instance profile when running inside AWS)

  **URI format example**

  .. code-block:: bash

    export AIRFLOW_CONN_AWS_DEFAULT=aws://

  **JSON format example**

  .. code-block:: bash

    export AIRFLOW_CONN_AWS_DEFAULT='{"conn_type": "aws"}'

With a AWS IAM key pair
^^^^^^^^^^^^^^^^^^^^^^^

  **URI format example**

  .. code-block:: bash

    export AIRFLOW_CONN_AWS_DEFAULT=aws://AKIAIOSFODNN7EXAMPLE:wJalrXUtnFEMI%2FK7MDENG%2FbPxRfiCYEXAMPLEKEY@

  Note here, that the secret access key has been URL-encoded (changing ``/`` to ``%2F``), and also the
  trailing ``@`` (without which, it is treated as ``<host>:<port>`` and will not work)

  **JSON format example**

  .. code-block:: bash

    export AIRFLOW_CONN_AWS_DEFAULT='{
      "conn_type": "aws",
      "login": "AKIAIOSFODNN7EXAMPLE",
      "password": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
    }'

Examples for the **Extra** field
--------------------------------

1. Using *~/.aws/credentials* and *~/.aws/config* file, with a profile.

This assumes all other Connection fields eg **AWS Access Key ID** or **AWS Secret Access Key**  are empty.

.. code-block:: json

    {
      "profile_name": "my_profile"
    }

2. Specifying a role_arn to assume and a region_name

.. code-block:: json

    {
      "role_arn": "arn:aws:iam::112223334444:role/my_role",
      "region_name": "ap-southeast-2"
    }

.. seealso::
    https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_assumerole

3. Configuring an outbound HTTP proxy

.. code-block:: json

    {
      "config_kwargs": {
        "proxies": {
          "http": "http://myproxy.mycompany.local:8080",
          "https": "http://myproxy.mycompany.local:8080"
        }
      }
    }

4. Using AssumeRoleWithSAML

.. code-block:: json

    {
      "region_name":"eu-west-1",
      "role_arn":"arn:aws:iam::112223334444:role/my_role",
      "assume_role_method":"assume_role_with_saml",
      "assume_role_with_saml":{
        "principal_arn":"arn:aws:iam::112223334444:saml-provider/my_saml_provider",
        "idp_url":"https://idp.mycompany.local/.../saml/clients/amazon-aws",
        "idp_auth_method":"http_spegno_auth",
        "mutual_authentication":"OPTIONAL",
        "idp_request_kwargs":{
          "headers":{"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
          "verify":false
        },
        "idp_request_retry_kwargs": {
          "total": 10,
          "backoff_factor":1,
          "status":10,
          "status_forcelist": [400, 429, 500, 502, 503, 504]
        },
        "log_idp_response":false,
        "saml_response_xpath":"////INPUT[@NAME='SAMLResponse']/@VALUE",
      },
      "assume_role_kwargs": { "something":"something" }
    }

The following settings may be used within the ``assume_role_with_saml`` container in Extra.

    * ``principal_arn``: The ARN of the SAML provider created in IAM that describes the identity provider.
    * ``idp_url``: The URL to your IDP endpoint, which provides SAML Assertions.
    * ``idp_auth_method``: Specify "http_spegno_auth" to use the Python ``requests_gssapi`` library. This library is more up to date than ``requests_kerberos`` and is backward compatible. See ``requests_gssapi`` documentation on PyPI.
    * ``mutual_authentication``: Can be "REQUIRED", "OPTIONAL" or "DISABLED". See ``requests_gssapi`` documentation on PyPI.
    * ``idp_request_kwargs``: Additional ``kwargs`` passed to ``requests`` when requesting from the IDP (over HTTP/S).
    * ``idp_request_retry_kwargs``: Additional ``kwargs`` to construct a
      `urllib3.util.Retry <https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#urllib3.util.Retry>`_
      used as a retry strategy when requesting from the IDP.
    * ``log_idp_response``: Useful for debugging - if specified, print the IDP response content to the log. Note that a successful response will contain sensitive information!
    * ``saml_response_xpath``: How to query the IDP response using XML / HTML xpath.
    * ``assume_role_kwargs``: Additional ``kwargs`` passed to ``sts_client.assume_role_with_saml``.

.. note:: The ``requests_gssapi`` library is used to obtain a SAML response from your IDP.
    You may need to ``pip uninstall python-gssapi`` and ``pip install gssapi`` instead for this to work.
    The ``python-gssapi`` library is outdated, and conflicts with some versions of ``paramiko`` which Airflow uses elsewhere.

.. seealso::
    :class:`~airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook`
    https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_assumerolewithsaml
    https://pypi.org/project/requests-gssapi/

Avoid Throttling exceptions
---------------------------

Amazon Web Services have quota limits for simultaneous API call as result with frequent calls
``apache-airflow-providers-amazon`` components might fail during execution with a
throttling exception, e.g. *ThrottlingException*, *ProvisionedThroughputExceededException*.

``botocore.config.Config`` supports different exponential backoff modes out of the box:
``legacy``, ``standard``, ``adaptive``

By default, ``botocore.config.Config`` uses ``legacy`` mode with 5 maximum retry attempts,
which may not be enough in some cases.

If you encounter throttling exceptions, you may change the mode to ``standard`` with more retry attempts.

.. seealso::
    https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html#retries

Set in Connection
^^^^^^^^^^^^^^^^^

**Connection extra field**:
  .. code-block:: json

    {
      "config_kwargs": {
        "retries": {
          "mode": "standard",
          "max_attempts": 10
        }
      }
    }

Set in AWS Config File
^^^^^^^^^^^^^^^^^^^^^^

**~/.aws/config**:
  .. code-block:: ini

    [profile awesome_aws_profile]
    retry_mode = standard
    max_attempts = 10

**Connection extra field**:
  .. code-block:: json

    {
      "profile_name": "awesome_aws_profile"
    }

Set by Environment Variables
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  .. note:: This sets the retry mode on all connections,
    unless another retry config is explicitly set on a specific connection.

  .. code-block:: bash

    export AWS_RETRY_MODE=standard
    export AWS_MAX_ATTEMPTS=10

.. _howto/connection:aws:session-factory:

Session Factory
---------------

The default ``BaseSessionFactory`` for the connection can handle most of the authentication methods for AWS.
In the case that you would like to have full control of
`boto3 session <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html>`__ creation or
you are using custom `federation <https://aws.amazon.com/identity/federation/>`__ that requires
`external process to source the credentials <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html>`__,
you can subclass :class:`~airflow.providers.amazon.aws.hooks.base_aws.BaseSessionFactory` and override ``create_session``
and/or ``_create_basic_session`` method depending on your needs.

You will also need to add configuration for ``AwsBaseHook`` to use the custom implementation by their full path.

Example
^^^^^^^

**Configuration**:
  .. code-block:: ini

    [aws]
    session_factory = my_company.aws.MyCustomSessionFactory

**Connection extra field**:
  .. code-block:: json

    {
      "federation": {
        "username": "my_username",
        "password": "my_password"
      }
    }

**Custom Session Factory**:
  .. code-block:: python

    def get_federated_aws_credentials(username: str, password: str):
        """
        Mock interaction with federation endpoint/process and returns AWS credentials.
        """
        return {
            "Version": 1,
            "AccessKeyId": "key",
            "SecretAccessKey": "secret",
            "SessionToken": "token",
            "Expiration": "2050-12-31T00:00:00.000Z",
        }

    class MyCustomSessionFactory(BaseSessionFactory):
        @property
        def federated(self):
            return "federation" in self.extra_config

        def _create_basic_session(self, session_kwargs: dict[str, Any]) -> boto3.session.Session:
            if self.federated:
                return self._create_federated_session(session_kwargs)
            else:
                return super()._create_basic_session(session_kwargs)

        def _create_federated_session(self, session_kwargs: dict[str, Any]) -> boto3.session.Session:
            username = self.extra_config["federation"]["username"]
            region_name = self._get_region_name()
            self.log.debug(
                f"Creating federated session with username={username} region_name={region_name} for "
                f"connection {self.conn.conn_id}"
            )
            credentials = RefreshableCredentials.create_from_metadata(
                metadata=self._refresh_federated_credentials(),
                refresh_using=self._refresh_federated_credentials,
                method="custom-federation",
            )
            session = botocore.session.get_session()
            session._credentials = credentials
            session.set_config_variable("region", region_name)
            return boto3.session.Session(botocore_session=session, **session_kwargs)

        def _refresh_federated_credentials(self) -> dict[str, str]:
            self.log.debug("Refreshing federated AWS credentials")
            credentials = get_federated_aws_credentials(**self.extra_config["federation"])
            access_key_id = credentials["AccessKeyId"]
            expiry_time = credentials["Expiration"]
            self.log.info(
                f"New federated AWS credentials received with aws_access_key_id={access_key_id} and "
                f"expiry_time={expiry_time} for connection {self.conn.conn_id}"
            )
            return {
                "access_key": access_key_id,
                "secret_key": credentials["SecretAccessKey"],
                "token": credentials["SessionToken"],
                "expiry_time": expiry_time,
            }

.. _howto/connection:aws:gcp-federation:

Google Cloud to AWS authentication using Web Identity Federation
----------------------------------------------------------------

Thanks to `Web Identity Federation <https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html>`__, you can use the credentials from the Google Cloud platform to authorize
access in the Amazon Web Service platform. If you additionally use authorizations with access token obtained
from `metadata server <https://cloud.google.com/compute/docs/storing-retrieving-metadata>`__ or
`Workload Identity <https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#gke_mds>`__,
you can improve the security of your environment by eliminating long-lived credentials.

The Google Cloud credentials is exchanged for the Amazon Web Service
`temporary credentials <https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html>`__
by `AWS Security Token Service <https://docs.aws.amazon.com/STS/latest/APIReference/welcome.html>`__.

The following diagram illustrates a typical communication flow used to obtain the AWS credentials.

.. figure::  /img/aws-web-identity-federation-gcp.png

    Communication Flow Diagram

Role setup
^^^^^^^^^^

In order for a Google identity to be recognized by AWS, you must configure roles in AWS.

You can do it by using the role wizard or by using `the Terraform <https://www.terraform.io/>`__.

Role wizard
"""""""""""

To create an IAM role for web identity federation:

1. Sign in to the AWS Management Console and open the IAM console at https://console.aws.amazon.com/iam/.
2. In the navigation pane, choose **Roles** and then choose **Create role**.
3. Choose the **Web identity** role type.
4. For Identity provider, choose the **Google**.
5. Type the service account email address (in the form ``<NAME>@<PROJECT_ID>.iam.gserviceaccount.com``) into the **Audience** box.
6. Review your web identity information and then choose **Next: Permissions**.
7. Select the policy to use for the permissions policy or choose **Create policy** to open a new browser tab and create a new policy from scratch. For more information, see `Creating IAM Policy <https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create-console.html#access_policies_create-start>`__.
8. Choose **Next: Tags**.
9. (Optional) Add metadata to the role by attaching tags as key–value pairs. For more information about using tags in IAM, see `Tagging IAM users and roles <https://docs.aws.amazon.com/IAM/latest/UserGuide/id_tags.html>`__.
10. Choose **Next: Review**.
11. For **Role name**, type a role name. Role names must be unique within your AWS account.
12. (Optional) For **Role description**, type a description for the new role.
13. Review the role and then choose **Create role**.

For more information, see: `Creating a role for web identity or OpenID connect federation (console) <https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_oidc.html>`__

Finally, you should get a role that has a similar policy to the one below:

.. code-block:: json

    {
      "Version": "2012-10-17",
      "Statement": [
        {
          "Effect": "Allow",
          "Principal": {
            "Federated": "accounts.google.com"
          },
          "Action": "sts:AssumeRoleWithWebIdentity",
          "Condition": {
            "StringEquals": {
              "accounts.google.com:aud": "<NAME>@<PROJECT_ID>.iam.gserviceaccount.com"
            }
          }
        }
      ]
    }

In order to protect against the misuse of the Google OpenID token, you can also limit the scope of use by configuring
restrictions per audience. You will need to configure the same value for the connection, and then this value also included in the ID Token. AWS will test if this value matches.
For that, you can add a new condition to the policy.

.. code-block:: json

    {
      "Condition": {
        "StringEquals": {
          "accounts.google.com:aud": "<NAME>@<PROJECT_ID>.iam.gserviceaccount.com",
          "accounts.google.com:oaud": "service-amp.my-company.com"
        }
      }
    }

After creating the role, you should configure the connection in Airflow.

Terraform
"""""""""

In order to quickly configure a new role, you can use the following Terraform script, which configures
AWS roles along with the assigned policy.
Before using it, you need correct the variables in the ``locals`` section to suit your environment:

* ``google_service_account`` - The email address of the service account that will have permission to use
  this role
* ``google_openid_audience`` - Constant value that is configured in the Airflow role and connection.
  It prevents misuse of the Google ID token.
* ``aws_role_name`` - The name of the new AWS role.
* ``aws_policy_name`` - The name of the new AWS policy.

For more information on using Terraform scripts, see:
`Terraform docs - Get started - AWS <https://learn.hashicorp.com/collections/terraform/aws-get-started>`__

After executing the plan, you should configure the connection in Airflow.

.. code-block: terraform

    locals {
      google_service_account = "<NAME>@<PROJECT>.iam.gserviceaccount.com"
      google_openid_audience = "<SERVICE_NAME>.<DOMAIN>"
      aws_role_name          = "WebIdentity-Role"
      aws_policy_name        = "WebIdentity-Role"
    }

    terraform {
      required_providers {
        aws = {
          source  = "hashicorp/aws"
          version = "~> 3.0"
        }
      }
    }

    provider "aws" {
      region = "us-east-1"
    }

    data "aws_iam_policy_document" "assume_role_policy" {
      statement {
        actions = [
          "sts:AssumeRoleWithWebIdentity"
        ]
        effect = "Allow"

        condition {
          test = "StringEquals"
          variable = "accounts.google.com:aud"
          values = [local.google_service_account]
        }

        condition {
          test = "StringEquals"
          variable = "accounts.google.com:oaud"
          values = [local.google_openid_audience]
        }

        principals {
          identifiers = ["accounts.google.com"]
          type = "Federated"
        }
      }
    }

    resource "aws_iam_role" "role_web_identity" {
      name               = local.aws_role_name
      description        = "Terraform managed policy"
      path               = "/"
      assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
    }
    # terraform import aws_iam_role.role_web_identity "WebIdentity-Role"

    data "aws_iam_policy_document" "web_identity_bucket_policy_document" {
      statement {
        effect = "Allow"
        actions = [
          "s3:ListAllMyBuckets"
        ]
        resources = ["*"]
      }
    }

    resource "aws_iam_policy" "web_identity_bucket_policy" {
      name = local.aws_policy_name
      path = "/"
      description = "Terraform managed policy"
      policy = data.aws_iam_policy_document.web_identity_bucket_policy_document.json
    }
    # terraform import aws_iam_policy.web_identity_bucket_policy arn:aws:iam::240057002457:policy/WebIdentity-S3-Policy

    resource "aws_iam_role_policy_attachment" "policy-attach" {
      role       = aws_iam_role.role_web_identity.name
      policy_arn = aws_iam_policy.web_identity_bucket_policy.arn
    }
    # terraform import aws_iam_role_policy_attachment.policy-attach WebIdentity-Role/arn:aws:iam::240057002457:policy/WebIdentity-S3-Policy

Connection setup
^^^^^^^^^^^^^^^^

In order to use a Google identity, field ``"assume_role_method"`` must be ``"assume_role_with_web_identity"`` and
field ``"assume_role_with_web_identity_federation"`` must be ``"google"`` in the extra section
of the connection setup. It also requires that you set up roles in the ``"role_arn"`` field.
Optionally, you can limit the use of the Google Open ID token by configuring the
``"assume_role_with_web_identity_federation_audience"`` field. The value of these fields must match the value configured in the role.

Airflow will establish Google's credentials based on `the Application Default Credentials <https://cloud.google.com/docs/authentication/production>`__.

Below is an example connection configuration.

.. code-block:: json

  {
    "role_arn": "arn:aws:iam::240057002457:role/WebIdentity-Role",
    "assume_role_method": "assume_role_with_web_identity",
    "assume_role_with_web_identity_federation": "google",
    "assume_role_with_web_identity_federation_audience": "service_a.apache.com"
  }

You can configure connection, also using environmental variable :envvar:`AIRFLOW_CONN_{CONN_ID}`.

.. code-block:: bash

    export AIRFLOW_CONN_AWS_DEFAULT="aws://\
    ?role_arn=arn%3Aaws%3Aiam%3A%3A240057002457%3Arole%2FWebIdentity-Role&\
    assume_role_method=assume_role_with_web_identity&\
    assume_role_with_web_identity_federation=google&\
    assume_role_with_web_identity_federation_audience=aaa.polidea.com"

Using IAM Roles for Service Accounts (IRSA) on EKS
----------------------------------------------------------------

If you are running Airflow on `Amazon EKS <https://aws.amazon.com/eks/>`_, you can grant AWS related permission (such as S3 Read/Write for remote logging) to the Airflow service by granting the IAM role to it's service account. IRSA provides fine-grained permission management for apps(e.g., pods) that run on EKS and use other AWS services. These could be apps that use S3, any other AWS services like Secrets Manager, CloudWatch, DynamoDB etc.

To activate this, the following steps must be followed:

1. Create an IAM OIDC Provider on EKS cluster.
2. Create an IAM Role and Policy to attach to the Airflow service account with web identity provider created at 1.
3. Add the corresponding IAM Role to the Airflow service account as an annotation.

.. seealso::
    https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html

Then you can find ``AWS_ROLE_ARN`` and ``AWS_WEB_IDENTITY_TOKEN_FILE`` in environment variables of appropriate pods that `Amazon EKS Pod Identity Web Hook <https://github.com/aws/amazon-eks-pod-identity-webhook>`__ added. Then `boto3 <https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials>`__ will configure credentials using those variables.
In order to use IRSA in Airflow, you have to create an aws connection with all fields empty. If a field such as ``role-arn`` is set, Airflow does not follow the boto3 default flow because it manually create a session using connection fields. If you did not change the default connection ID, an empty AWS connection named ``aws_default`` would be enough.

Create IAM Role for Service Account(IRSA) using eksctl
------------------------------------------------------
`eksctl <https://eksctl.io/>`_ is a simple CLI tool for creating and managing clusters on EKS. Follow the steps to create IRSA for Airflow.

1. `Install eksctl <https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html>`_ in your local machine.
2. Setup AWS credentials in your terminal to run ``eksctl`` commands.
3. The IAM OIDC Provider is not enabled by default, you can use the following command to enable.

.. code-block:: bash

    eksctl utils associate-iam-oidc-provider --cluster="<EKS_CLUSTER_ID>" --approve

4. Replace ``EKS_CLUSTER_ID``, ``SERVICE_ACCOUNT_NAME`` and ``NAMESPACE`` and execute the the following command. This command will use an existing EKS Cluster ID and create an IAM role, service account and namespace.

.. code-block:: bash

    eksctl create iamserviceaccount --cluster="<EKS_CLUSTER_ID>" --name="<SERVICE_ACCOUNT_NAME>" --namespace="<NAMESPACE>" --attach-policy-arn="<IAM_POLICY_ARN>" --approve``

This is an example command with values. This example is using managed policy with full S3 permissions attached to the IAM role. We highly recommend you to create a restricted IAM policy with necessary permissions to S3, Secrets Manager, CloudWatch etc. and use it with ``--attach-policy-arn``.

.. code-block:: bash

    eksctl create iamserviceaccount --cluster=airflow-eks-cluster --name=airflow-sa --namespace=airflow --attach-policy-arn=arn:aws:iam::aws:policy/AmazonS3FullAccess --approve

5. Use the service account name in Airflow Helm chart deployment or with Kubernetes Pod Operator.

Create IAM Role for Service Account(IRSA) using Terraform
---------------------------------------------------------

For Terraform users, IRSA roles can be created using `Amazon EKS Blueprints for Terraform <https://github.com/aws-ia/terraform-aws-eks-blueprints>`_ module.

This module creates a new IAM Role, service account and namespace. This will associate IAM role with the service account and adds the annotation to the service account.
You need to create an IAM policy with the required permissions that you would like the containers in your pods to have. Replace ``IAM_POLICY_ARN`` with your IAM policy ARN, other required inputs as shown below and run ``terraform apply``.

.. code-block:: terraform

    module "airflow_irsa" {
      source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/irsa"

      eks_cluster_id             = "<EKS_CLUSTER_ID>"
      eks_oidc_provider_arn      = "<EKS_CLUSTER_OIDC_PROVIDER_ARN>"
      irsa_iam_policies          = ["<IAM_POLICY_ARN>"]
      kubernetes_namespace       = "<NAMESPACE>"
      kubernetes_service_account = "<SERVICE_ACCOUNT_NAME>"
    }

Once the Terraform module is applied then you can use the service account in your Airflow deployments or with Kubernetes Pod Operator.


text:
module FC
  class Policy < DbBase
    set_table :policies, 'name, create_storages, copies, delete_deferred_time'

    class << self
      attr_accessor :storages_cache_time, :get_create_storages_mutex
    end
    @storages_cache_time = 20 # ttl for storages cache
    @get_create_storages_mutex = Mutex.new

    def self.filter_by_host(host = nil)
      host = FC::Storage.curr_host unless host
      self.where.select do |policy|
        policy.get_create_storages.detect{|storage| storage.host == host}
      end
    end

    def get_create_storages
      self.class.get_create_storages_mutex.synchronize do
        unless @create_storages_cache && Time.new.to_i - @get_create_storages_time.to_i < self.class.storages_cache_time
          @get_create_storages_time = Time.new.to_i
          names = create_storages.to_s.split(',').map{|s| "'#{s}'"}.join(',')
          @create_storages_cache = names.empty? ? [] : FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
        end
      end
      @create_storages_cache
    end

    # get available storage for create by size and local item path
    def get_proper_storage_for_create(size, local_path = nil)
      FC::Storage.select_proper_storage_for_create(get_create_storages, size) do |storages|
        local_storages = storages.select{|storage| FC::Storage.curr_host == storage.host}
        # find same storage device as local_path device
        if local_path && !local_storages.empty?
          dev = File.stat(local_path).dev
          dev_storage = local_storages.select{|storage| dev == File.stat(storage.path).dev}.first
          local_storages = [dev_storage] if dev_storage
        end
        # if no local storages - use all storages
        local_storages = storages if local_storages.empty?
        local_storages
      end
    end
  end
end


text:
package handler

import (
	"context"
	"math"
	"net/http"
	"sync"
	"time"

	"go.uber.org/atomic"
	"go.uber.org/zap"
	"k8s.io/apimachinery/pkg/types"
	netstats "knative.dev/networking/pkg/http/stats"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/logging/logkey"
	pkgmetrics "knative.dev/pkg/metrics"
	"knative.dev/serving/pkg/activator"
	"knative.dev/serving/pkg/apis/serving"
	asmetrics "knative.dev/serving/pkg/autoscaler/metrics"
	revisioninformer "knative.dev/serving/pkg/client/injection/informers/serving/v1/revision"
	servinglisters "knative.dev/serving/pkg/client/listers/serving/v1"
	"knative.dev/serving/pkg/metrics"
)

const reportInterval = time.Second

// revisionStats is a type that wraps information needed to calculate stats per revision.
//
// stats is thread-safe in itself and thus needs no extra synchronization.
// firstRequest is only read/mutated in `report` which is guaranteed to be single-threaded
// as it is driven by the report channel.
type revisionStats struct {
	stats        *netstats.RequestStats
	firstRequest float64
	refs         atomic.Int64
}

// ConcurrencyReporter reports stats based on incoming requests and ticks.
type ConcurrencyReporter struct {
	logger  *zap.SugaredLogger
	podName string

	// Stat reporting channel
	statCh chan []asmetrics.StatMessage

	rl servinglisters.RevisionLister

	mux sync.RWMutex
	// This map holds the concurrency and request count accounting across revisions.
	stats map[types.NamespacedName]*revisionStats
}

// NewConcurrencyReporter creates a ConcurrencyReporter which listens to incoming
// ReqEvents on reqCh and ticks on reportCh and reports stats on statCh.
func NewConcurrencyReporter(ctx context.Context, podName string, statCh chan []asmetrics.StatMessage) *ConcurrencyReporter {
	return &ConcurrencyReporter{
		logger:  logging.FromContext(ctx),
		podName: podName,
		statCh:  statCh,
		rl:      revisioninformer.Get(ctx).Lister(),

		stats: make(map[types.NamespacedName]*revisionStats),
	}
}

// handleRequestIn handles an event of a request coming into the system. Returns the stats
// the outgoing event should be recorded to.
func (cr *ConcurrencyReporter) handleRequestIn(event netstats.ReqEvent) *revisionStats {
	stat, msg := cr.getOrCreateStat(event)
	if msg != nil {
		cr.statCh <- []asmetrics.StatMessage{*msg}
	}
	stat.stats.HandleEvent(event)
	return stat
}

// handleRequestOut handles an event of a request being done. Takes the stats returned by
// the handleRequestIn call.
func (cr *ConcurrencyReporter) handleRequestOut(stat *revisionStats, event netstats.ReqEvent) {
	stat.stats.HandleEvent(event)
	stat.refs.Dec()
}

// getOrCreateStat gets a stat from the state if present.
// If absent it creates a new one and returns it, potentially returning a StatMessage too
// to trigger an immediate scale-from-0.
func (cr *ConcurrencyReporter) getOrCreateStat(event netstats.ReqEvent) (*revisionStats, *asmetrics.StatMessage) {
	cr.mux.RLock()
	stat := cr.stats[event.Key]
	if stat != nil {
		// Since this is incremented under the lock, it's guaranteed to be observed by
		// the deletion routine.
		stat.refs.Inc()
		cr.mux.RUnlock()
		return stat, nil
	}
	cr.mux.RUnlock()

	// Doubly checked locking.
	cr.mux.Lock()
	defer cr.mux.Unlock()

	stat = cr.stats[event.Key]
	if stat != nil {
		// Since this is incremented under the lock, it's guaranteed to be observed by
		// the deletion routine.
		stat.refs.Inc()
		return stat, nil
	}

	stat = &revisionStats{
		stats:        netstats.NewRequestStats(event.Time),
		firstRequest: 1,
	}
	stat.refs.Inc()
	cr.stats[event.Key] = stat

	return stat, &asmetrics.StatMessage{
		Key: event.Key,
		Stat: asmetrics.Stat{
			PodName:                   cr.podName,
			AverageConcurrentRequests: 1,
			// The way the checks are written, this cannot ever be
			// anything else but 1. The stats map key is only deleted
			// after a reporting period, so we see this code path at most
			// once per period.
			RequestCount: 1,
		},
	}
}

// report cuts a report from all collected statistics and sends the respective messages
// via the statsCh and reports the concurrency metrics to prometheus.
func (cr *ConcurrencyReporter) report(now time.Time) []asmetrics.StatMessage {
	msgs, toDelete := cr.computeReport(now)

	if len(toDelete) > 0 {
		cr.mux.Lock()
		defer cr.mux.Unlock()
		for _, key := range toDelete {
			// Avoid deleting the stat if a request raced fetching it while we've been
			// busy reporting.
			if cr.stats[key].refs.Load() == 0 {
				delete(cr.stats, key)
			}
		}
	}

	return msgs
}

func (cr *ConcurrencyReporter) computeReport(now time.Time) (msgs []asmetrics.StatMessage, toDelete []types.NamespacedName) {
	cr.mux.RLock()
	defer cr.mux.RUnlock()
	msgs = make([]asmetrics.StatMessage, 0, len(cr.stats))
	for key, stat := range cr.stats {
		report := stat.stats.Report(now)

		firstAdj := stat.firstRequest
		stat.firstRequest = 0.

		// This is only 0 if we have seen no activity for the entire reporting
		// period at all.
		if report.AverageConcurrency == 0 {
			toDelete = append(toDelete, key)
		}

		// Subtract the request we already reported when first seeing the
		// revision. We report a min of 0 here because the initial report is
		// always a concurrency of 1 and the actual concurrency reported over
		// the reporting period might be < 1.
		adjustedConcurrency := math.Max(report.AverageConcurrency-firstAdj, 0)
		adjustedCount := report.RequestCount - firstAdj
		msgs = append(msgs, asmetrics.StatMessage{
			Key: key,
			Stat: asmetrics.Stat{
				PodName:                   cr.podName,
				AverageConcurrentRequests: adjustedConcurrency,
				RequestCount:              adjustedCount,
			},
		})
	}

	return msgs, toDelete
}

func (cr *ConcurrencyReporter) reportToMetricsBackend(key types.NamespacedName, concurrency float64) {
	ns := key.Namespace
	revName := key.Name
	revision, err := cr.rl.Revisions(ns).Get(revName)
	if err != nil {
		cr.logger.Errorw("Error while getting revision", zap.String(logkey.Key, key.String()), zap.Error(err))
		return
	}
	configurationName := revision.Labels[serving.ConfigurationLabelKey]
	serviceName := revision.Labels[serving.ServiceLabelKey]

	reporterCtx, _ := metrics.PodRevisionContext(cr.podName, activator.Name, ns, serviceName, configurationName, revName)
	pkgmetrics.Record(reporterCtx, requestConcurrencyM.M(concurrency))
}

// Run runs until stopCh is closed and processes events on all incoming channels.
func (cr *ConcurrencyReporter) Run(stopCh <-chan struct{}) {
	ticker := time.NewTicker(reportInterval)
	defer ticker.Stop()
	cr.run(stopCh, ticker.C)
}

func (cr *ConcurrencyReporter) run(stopCh <-chan struct{}, reportCh <-chan time.Time) {
	for {
		select {
		case now := <-reportCh:
			msgs := cr.report(now)
			for _, msg := range msgs {
				cr.reportToMetricsBackend(msg.Key, msg.Stat.AverageConcurrentRequests)
			}
			if len(msgs) > 0 {
				cr.statCh <- msgs
			}
		case <-stopCh:
			return
		}
	}
}

// Handler returns a handler that records requests coming in/being finished in the stats
// machinery.
func (cr *ConcurrencyReporter) Handler(next http.Handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		revisionKey := RevIDFrom(r.Context())

		stat := cr.handleRequestIn(netstats.ReqEvent{Key: revisionKey, Type: netstats.ReqIn, Time: time.Now()})
		defer func() {
			cr.handleRequestOut(stat, netstats.ReqEvent{Key: revisionKey, Type: netstats.ReqOut, Time: time.Now()})
		}()

		next.ServeHTTP(w, r)
	}
}


text:
AppMarket 2.0 Beta (1.9.80)
===========================

February 3rd, 2014. The OpenNebula team is pleased to announce a beta release of AppMarket 2.0. This release includes important features that are a direct result of feedback from production deployments.

This release extends the AppMarket functionality by adding a new set of features that enables the management and processing of OVA files. A new component *AppMarket Worker* is introduced, which handles the OVA package treatment (download, unpack, OVF parsing) and image format conversion. This release also features a new API and a new AppMarket interface via Sunstone.

What's New in AppMarket 2.0 Beta
--------------------------------

In the following list you can check the highlights of AppMarket 2.0 Beta:

### AppMarket

* **Multi-image Appliances**: AppMarket Appliances can now have multiple disks, which creates new possibilities to upload and register more complex and feature complete Appliances.
* **Sunstone Import**: users will be able to import registered Appliances to OpenNebula via Sunstone by using pre-filled Image and Template creation forms.
* **Sunstone Views**: two new Sunstone views: an AppMarket *admin* view, that allows full control of the AppMarket, and a *user* view, that allows end users to import registered appliances.

These new features will be only available interacting with AppMarket from an OpenNebula 4.6 deployment.

### AppMarket Worker

A new horizontally-scalable component that subscribes to the AppMarket and executes jobs. It enhances the AppMarket functionality by adding:

* **OVA processing**: If a URL to an OVA package is supplied, the AppMarket will be able completely process it and integrate it to its repository. This involves: downloading and unpacking of the OVA package, parsing of the OVF file and creating a corresponding OpenNebula template.
* **Format conversion**: appliances registered in a specific format, can be converted to a new format.
* **Extensibility**: All the operations described above are implemented within the framework of an extensible driver engine, which allows further customization and integration by the administrators.

Acknowledgements
----------------

The new features introduced in the AppMarket 2.0 were funded by [Produban](http://www.produban.com/) in the context of the Fund a Feature Program.