text:
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using SpatialEnrichment;
using SpatialEnrichment.Helpers;
using SpatialEnrichmentWrapper;

namespace UnitTests
{
    [TestClass]
    public class TesselationTests
    {
        [TestMethod]
        public void VolatileGradientDescent()
        {
            var Config = new ConfigParams("");
            Program.Config = Config;
            var res = Program.RandomizeCoordinatesAndSave(20, true);

            var T = new Tesselation(res.Item1.Select(v => (Coordinate)v).ToList(), res.Item2, null, Config);
            var cell = T.ComputeCellFromCoordinateVolatile(new Coordinate(0.0, 0.0));

            Console.WriteLine(cell.CenterOfMass);
        }

        [TestMethod]
        public void SampleProblems()
        {
            Program.Config = new ConfigParams("");
            for (var i = 0; i < 12; i++)
            {
                StaticConfigParams.filenamesuffix = i.ToString();
                var res = Program.RandomizeCoordinatesAndSave(20, true);
            }
        }

        [TestMethod]
        public void SamplingVsGridExperiment()
        {
            Program.Config = new ConfigParams("");
            var vN = new int[] {40, 60, 100};
            var vNu = new int[] {10, 20, 30};
            var res = new Dictionary<Tuple<int, int>, List<double>>();
            foreach (var N in vN)
            {
                foreach (var nu in vNu)
                {
                    res.Add(new Tuple<int, int>(N, nu),  Experiments.CompareExahustiveWithSubsamplingInput(N, nu, 50));
                }
            }

        }

        [TestMethod]
        public void GridExperiment()
        {
            Program.Config = new ConfigParams("");
            StaticConfigParams.filenamesuffix = "0";
            var instance = Program.RandomizeCoordinatesAndSave(50, true);
            var instanceData = instance.Item1.Zip(instance.Item2, (a, b) => new {Coord = a, Label = b}).ToList();
            var uniformGrid = new Gridding();
            uniformGrid.GeneratePivotGrid(1000);
            var grid = uniformGrid.GetPivots().ToList();
            int numcovered = 0;
            double bestP = 1.0;
            Parallel.ForEach(grid, pivot =>
            {
                var boolVec = instanceData.OrderBy(coord=>coord.Coord.EuclideanDistance(pivot)).Select(c=>c.Label).ToArray();
                var res = mHGJumper.minimumHypergeometric(boolVec);
                if (res.Item1 < bestP) bestP = res.Item1;
               // var currid=Interlocked.Increment(ref numcovered);
               // if (currid % 100 == 0)
               //     Console.Write($"\r\r\r\r\r\r\r\r\r\r{currid / grid.Count:P}");
            });
         Console.WriteLine(bestP);
        }

        [TestMethod]
        public void Empirical3DSamplingTest()
        {
            var filename = @"C:\Users\shaybe\Dropbox\Thesis - PHd\SpatialEnrichment\Caulobacter\transferases\phosphoribosyltransferase_3d.csv";
            var data = File.ReadAllLines(filename).Select(l => l.Split(',')).Select(sl =>
                new Tuple<ICoordinate, bool>(
                    new Coordinate3D(double.Parse(sl[0]), double.Parse(sl[1]), double.Parse(sl[2])),
                    bool.Parse(sl[3]))).ToList();
            var empiricalGrid = new Gridding();
            empiricalGrid.GenerateEmpricialDensityGrid(10000, data);
            var grid = empiricalGrid.GetPivots().ToList();
        }

        [TestMethod]
        public void SamplingGridSearchExperiment()
        {
            Program.Config = new ConfigParams("");
            StaticConfigParams.filenamesuffix = "0";
            var instance = Program.RandomizeCoordinatesAndSave(50, true);
            var instanceData = instance.Item1.Zip(instance.Item2, (a, b) => new { Coord = a, Label = b }).ToList();
            var uniformGrid = new Gridding();
            uniformGrid.GeneratePivotGrid(1000);
            var grid = uniformGrid.GetPivots().ToList();
            int numcovered = 0;
            double bestP = 1.0;
            Parallel.ForEach(grid, pivot =>
            {
                var boolVec = instanceData.OrderBy(coord => coord.Coord.EuclideanDistance(pivot)).Select(c => c.Label).ToArray();
                var res = mHGJumper.minimumHypergeometric(boolVec);
                if (res.Item1 < bestP) bestP = res.Item1;
                // var currid=Interlocked.Increment(ref numcovered);
                // if (currid % 100 == 0)
                //     Console.Write($"\r\r\r\r\r\r\r\r\r\r{currid / grid.Count:P}");
            });
            Console.WriteLine(bestP);
        }
    }
}

redpajama_set_name: RedPajamaGithub


text:
package com.puntodeventa.global.Util;

import java.io.IOException;
import java.util.Properties;

/**
 * @author Jorge Hernandez
 */
public class TagHelper {

    private static final LogHelper objLog = new LogHelper("TagHelper");
    private static final Properties prop = new Properties();

    public static String getTag(String tagName) {

        String returnValue = "";

        try {
            prop.load(TagHelper.class.getResourceAsStream("/tags.properties"));
            returnValue =  prop.getProperty(tagName).toString();

        } catch (IOException ex) {
            objLog.Log("Possible cause: Tag " + tagName + " not found."  + ex.getMessage());
        }

        return returnValue;
    }
}


text:
using namespace llvm;

#define DEBUG_TYPE "wasm-instr-info"

#define GET_INSTRINFO_CTOR_DTOR
#include "WebAssemblyGenInstrInfo.inc"

WebAssemblyInstrInfo::WebAssemblyInstrInfo(const WebAssemblySubtarget &STI)
    : RI(STI.getTargetTriple()) {}

void WebAssemblyInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                       MachineBasicBlock::iterator I,
                                       DebugLoc DL, unsigned DestReg,
                                       unsigned SrcReg, bool KillSrc) const {
  BuildMI(MBB, I, DL, get(WebAssembly::COPY), DestReg)
      .addReg(SrcReg, KillSrc ? RegState::Kill : 0);
}

// Branch analysis.
bool WebAssemblyInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
                                         MachineBasicBlock *&TBB,
                                         MachineBasicBlock *&FBB,
                                         SmallVectorImpl<MachineOperand> &Cond,
                                         bool AllowModify) const {
  bool HaveCond = false;
  for (MachineInstr &MI : iterator_range<MachineBasicBlock::instr_iterator>(
           MBB.getFirstInstrTerminator(), MBB.instr_end())) {
    switch (MI.getOpcode()) {
    default:
      // Unhandled instruction; bail out.
      return true;
    case WebAssembly::BRIF:
      if (HaveCond)
        return true;
      Cond.push_back(MI.getOperand(1));
      TBB = MI.getOperand(0).getMBB();
      HaveCond = true;
      break;
    case WebAssembly::BR:
      if (!HaveCond)
        TBB = MI.getOperand(0).getMBB();
      else
        FBB = MI.getOperand(0).getMBB();
      break;
    }
    if (MI.isBarrier())
      break;
  }

  return false;
}

unsigned WebAssemblyInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
  MachineBasicBlock::instr_iterator I = MBB.instr_end();
  unsigned Count = 0;

  while (I != MBB.instr_begin()) {
    --I;
    if (I->isDebugValue())
      continue;
    if (!I->isTerminator())
      break;
    // Remove the branch.
    I->eraseFromParent();
    I = MBB.instr_end();
    ++Count;
  }

  return Count;
}

unsigned WebAssemblyInstrInfo::InsertBranch(
    MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
    ArrayRef<MachineOperand> Cond, DebugLoc DL) const {
  assert(Cond.size() <= 1);

  if (Cond.empty()) {
    if (!TBB)
      return 0;

    BuildMI(&MBB, DL, get(WebAssembly::BR)).addMBB(TBB);
    return 1;
  }

  BuildMI(&MBB, DL, get(WebAssembly::BRIF))
      .addMBB(TBB)
      .addOperand(Cond[0]);
  if (!FBB)
    return 1;

  BuildMI(&MBB, DL, get(WebAssembly::BR)).addMBB(FBB);
  return 2;
}

bool WebAssemblyInstrInfo::ReverseBranchCondition(
    SmallVectorImpl<MachineOperand> &Cond) const {
  assert(Cond.size() == 1);

  // TODO: Add branch reversal here... And re-enable MachineBlockPlacementID
  // when we do.

  return true;
}


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<title>Assimp: ProgressHandler.hpp File Reference</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>

<link href="doxygen.css" rel="stylesheet" type="text/css" />

Assimp
    v3.0 (July 2012)

Main Page

Related Pages

Namespaces

Classes

Files

File List

File Members

include

assimp

Classes |
Namespaces

ProgressHandler.hpp File Reference

Classes

class  	Assimp::ProgressHandler

 	CPP-API: Abstract interface for custom progress report receivers.  More...

Namespaces

namespace  	Assimp

 	Assimp's CPP-API and all internal APIs.

<address class="footer">
Generated on Tue Jul 10 2012 17:55:53 for Assimp by

 1.8.1.1
</address>


text:
<?php

namespace Magento\Theme\Model\ResourceModel\Design\Config\Scope;

use Magento\Framework\App\Config\ScopeConfigInterface;
use Magento\Framework\App\ScopeTreeProviderInterface;
use Magento\Framework\Data\Collection\EntityFactoryInterface;
use Magento\Theme\Model\Design\Config\MetadataProviderInterface;
use Magento\Theme\Model\Design\Config\ValueProcessor;

/**
 * Data collection
 *
 */
class Collection extends \Magento\Framework\Data\Collection
{
    /**
     * @var ScopeTreeProviderInterface
     */
    protected $scopeTree;

    /**
     * @var MetadataProviderInterface
     */
    protected $metadataProvider;

    /**
     * @var ScopeConfigInterface
     */
    protected $appConfig;

    /**
     * @var ValueProcessor
     */
    protected $valueProcessor;

    /**
     * Collection constructor
     *
     * @param EntityFactoryInterface $entityFactory
     * @param ScopeTreeProviderInterface $scopeTree
     * @param MetadataProviderInterface $metadataProvider
     * @param ScopeConfigInterface $appConfig
     * @param ValueProcessor $valueProcessor
     */
    public function __construct(
        EntityFactoryInterface $entityFactory,
        ScopeTreeProviderInterface $scopeTree,
        MetadataProviderInterface $metadataProvider,
        ScopeConfigInterface $appConfig,
        ValueProcessor $valueProcessor
    ) {
        parent::__construct($entityFactory);
        $this->scopeTree = $scopeTree;
        $this->metadataProvider = $metadataProvider;
        $this->appConfig = $appConfig;
        $this->valueProcessor = $valueProcessor;
    }

    /**
     * Load data
     *
     * @param bool $printQuery
     * @param bool $logQuery
     * @return \Magento\Theme\Model\ResourceModel\Design\Config\Scope\Collection
     *
     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
     */
    public function loadData($printQuery = false, $logQuery = false)
    {
        if (!$this->isLoaded()) {
            $default = $this->scopeTree->get();
            $this->prepareItemData();
            foreach ($default['scopes'] as $website) {
                $this->prepareItemData($website);
                foreach ($website['scopes'] as $group) {
                    foreach ($group['scopes'] as $store) {
                        $this->prepareItemData($website, $group, $store);
                    }
                }
            }
            $this->_setIsLoaded(true);
        }
        return $this;
    }

    /**
     * Retrieve fields metadata
     *
     * @param string $scope
     * @param int $scopeId
     * @return array
     */
    protected function getMetadataValues($scope = ScopeConfigInterface::SCOPE_TYPE_DEFAULT, $scopeId = null)
    {
        $result = [];
        foreach ($this->metadataProvider->get() as $itemName => $itemData) {
            if (isset($itemData['use_in_grid']) && (boolean)$itemData['use_in_grid']) {
                $result[$itemName] = $this->valueProcessor->process(
                    $this->appConfig->getValue($itemData['path'], $scope, $scopeId),
                    $scope,
                    $scopeId,
                    $itemData
                );
            }
        }

        return $result;
    }

    /**
     * Prepare item data depend on scope
     *
     * @param array $websiteScope
     * @param array $groupScope
     * @param array $storeScope
     *
     * @return void
     */
    protected function prepareItemData(array $websiteScope = [], array $groupScope = [], array $storeScope = [])
    {
        $result = [
            'store_website_id' => isset($websiteScope['scope_id']) ? $websiteScope['scope_id'] : null,
            'store_group_id' => isset($groupScope['scope_id']) ? $groupScope['scope_id'] : null,
            'store_id' => isset($storeScope['scope_id']) ? $storeScope['scope_id'] : null,
        ];

        $result = array_merge($result, $this->getScopeData($websiteScope, $groupScope, $storeScope));

        $this->_addItem(new \Magento\Framework\DataObject($result));
    }

    /**
     * Retrieve scope data
     *
     * @param array $websiteScope
     * @param array $groupScope
     * @param array $storeScope
     * @return array
     */
    protected function getScopeData(array $websiteScope, array $groupScope, array $storeScope)
    {
        if (isset($storeScope['scope'])) {
            $data = $this->getMetadataValues($storeScope['scope'], $storeScope['scope_id']);
        } elseif (isset($groupScope['scope'])) {
            $data = $this->getMetadataValues($groupScope['scope'], $groupScope['scope_id']);
        } elseif (isset($websiteScope['scope'])) {
            $data = $this->getMetadataValues($websiteScope['scope'], $websiteScope['scope_id']);
        } else {
            $data = $this->getMetadataValues();
        }

        return $data;
    }
}


text:
package sbt
package inc

import java.io.{File, IOException}
import Stamp.getStamp

trait ReadStamps
{
	/** The Stamp for the given product at the time represented by this Stamps instance.*/
	def product(prod: File): Stamp
	/** The Stamp for the given source file at the time represented by this Stamps instance.*/
	def internalSource(src: File): Stamp
	/** The Stamp for the given binary dependency at the time represented by this Stamps instance.*/
	def binary(bin: File): Stamp
}

/** Provides information about files as they were at a specific time.*/
trait Stamps extends ReadStamps
{
	def allInternalSources: collection.Set[File]
	def allBinaries: collection.Set[File]
	def allProducts: collection.Set[File]

	def sources: Map[File, Stamp]
	def binaries: Map[File, Stamp]
	def products: Map[File, Stamp]
	def classNames: Map[File, String]

	def className(bin: File): Option[String]

	def markInternalSource(src: File, s: Stamp): Stamps
	def markBinary(bin: File, className: String, s: Stamp): Stamps
	def markProduct(prod: File, s: Stamp): Stamps

	def filter(prod: File => Boolean, removeSources: Iterable[File], bin: File => Boolean): Stamps

	def ++ (o: Stamps): Stamps
	def groupBy[K](prod: Map[K, File => Boolean], sourcesGrouping: File => K, bin: Map[K, File => Boolean]): Map[K, Stamps]
}

sealed trait Stamp
final class Hash(val value: Array[Byte]) extends Stamp
final class LastModified(val value: Long) extends Stamp
final class Exists(val value: Boolean) extends Stamp

object Stamp
{
	implicit val equivStamp: Equiv[Stamp] = new Equiv[Stamp] {
		def equiv(a: Stamp, b: Stamp) = (a,b) match {
			case (h1: Hash, h2: Hash) => h1.value sameElements h2.value
			case (e1: Exists, e2: Exists) => e1.value == e2.value
			case (lm1: LastModified, lm2: LastModified) => lm1.value == lm2.value
			case _ => false
		}
	}
	def show(s: Stamp): String = s match {
		case h: Hash => "hash(" + Hash.toHex(h.value) + ")"
		case e: Exists => if(e.value) "exists" else "does not exist"
		case lm: LastModified => "last modified(" + lm.value + ")"
	}

	val hash = (f: File) => tryStamp(new Hash(Hash(f)))
	val lastModified = (f: File) => tryStamp(new LastModified(f.lastModified))
	val exists = (f: File) => tryStamp(if(f.exists) present else notPresent)

	def tryStamp(g: => Stamp): Stamp = try { g } catch { case i: IOException => notPresent }

	val notPresent = new Exists(false)
	val present = new Exists(true)

	def getStamp(map: Map[File, Stamp], src: File): Stamp = map.getOrElse(src, notPresent)
}

object Stamps
{
	/** Creates a ReadStamps instance that will calculate and cache the stamp for sources and binaries
	* on the first request according to the provided `srcStamp` and `binStamp` functions.  Each
	* stamp is calculated separately on demand.
	* The stamp for a product is always recalculated. */
	def initial(prodStamp: File => Stamp, srcStamp: File => Stamp, binStamp: File => Stamp): ReadStamps = new InitialStamps(prodStamp, srcStamp, binStamp)

	def empty: Stamps =
	{
		val eSt = Map.empty[File, Stamp]
		apply(eSt, eSt, eSt, Map.empty[File, String])
	}
	def apply(products: Map[File, Stamp], sources: Map[File, Stamp], binaries: Map[File, Stamp], binaryClassNames: Map[File, String]): Stamps =
		new MStamps(products, sources, binaries, binaryClassNames)
}

private class MStamps(val products: Map[File, Stamp], val sources: Map[File, Stamp], val binaries: Map[File, Stamp], val classNames: Map[File, String]) extends Stamps
{
	def allInternalSources: collection.Set[File] = sources.keySet
	def allBinaries: collection.Set[File] = binaries.keySet
	def allProducts: collection.Set[File] = products.keySet

	def ++ (o: Stamps): Stamps =
		new MStamps(products ++ o.products, sources ++ o.sources, binaries ++ o.binaries, classNames ++ o.classNames)

	def markInternalSource(src: File, s: Stamp): Stamps =
		new MStamps(products, sources.updated(src, s), binaries, classNames)

	def markBinary(bin: File, className: String, s: Stamp): Stamps =
		new MStamps(products, sources, binaries.updated(bin, s), classNames.updated(bin, className))

	def markProduct(prod: File, s: Stamp): Stamps =
		new MStamps(products.updated(prod, s), sources, binaries, classNames)

	def filter(prod: File => Boolean, removeSources: Iterable[File], bin: File => Boolean): Stamps =
		new MStamps(products.filterKeys(prod), sources -- removeSources, binaries.filterKeys(bin), classNames.filterKeys(bin))

	def groupBy[K](prod: Map[K, File => Boolean], sourcesGrouping: File => K, bin: Map[K, File => Boolean]): Map[K, Stamps] =
	{
		val sourcesMap: Map[K, Map[File, Stamp]] = sources.groupBy(item => sourcesGrouping(item._1))

		val constFalse = (f: File) => false
		def kStamps(k: K): Stamps = new MStamps(
			products.filterKeys(prod.getOrElse(k, constFalse)),
			sourcesMap.getOrElse(k, Map.empty[File,Stamp]),
			binaries.filterKeys(bin.getOrElse(k, constFalse)),
			classNames.filterKeys(bin.getOrElse(k, constFalse))
		)

		val keys = (prod.keySet ++ sourcesMap.keySet ++ bin.keySet).toList
		Map( keys.map( (k: K) => (k, kStamps(k)) ) : _*)
	}

	def product(prod: File) = getStamp(products, prod)
	def internalSource(src: File) = getStamp(sources, src)
	def binary(bin: File) = getStamp(binaries, bin)
	def className(bin: File) = classNames get bin

}

private class InitialStamps(prodStamp: File => Stamp, srcStamp: File => Stamp, binStamp: File => Stamp) extends ReadStamps
{
	import collection.mutable.{HashMap, Map}
	// cached stamps for files that do not change during compilation
	private val sources: Map[File, Stamp] = new HashMap
	private val binaries: Map[File, Stamp] = new HashMap

	def product(prod: File): Stamp = prodStamp(prod)
	def internalSource(src: File): Stamp = synchronized { sources.getOrElseUpdate(src, srcStamp(src)) }
	def binary(bin: File): Stamp = synchronized { binaries.getOrElseUpdate(bin, binStamp(bin)) }
}


text:
<!DOCTYPE html>

    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">

    <title>{% if page.title %}{{ page.title }} - {% endif %}RockStack</title>
    <link href="/static/css/bootstrap.min.css?rev=1" rel="stylesheet">
    <link href="/static/pygments.css?rev=1" rel="stylesheet">
    <link href="/static/rockstack.css?rev=2" rel="stylesheet" type="text/css" >
    <link href="//fonts.googleapis.com/css?family=Lato:400,700" rel="stylesheet" type="text/css">
    <link href="/favicon.ico" type="image/x-icon" rel="icon">

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-ex1-collapse">
            Toggle navigation

          </button>
          RockStack

News

Documentation

Rock Better Runtimes

      {{ content }}


text:
package net.tokensmith.jwt.jwe.factory;

import helper.entity.Factory;
import org.junit.Before;
import org.junit.Test;
import net.tokensmith.jwt.jwe.Transformation;
import net.tokensmith.jwt.jwk.KeyAlgorithm;
import net.tokensmith.jwt.jwk.generator.jdk.SecretKeyGenerator;

import javax.crypto.Cipher;
import javax.crypto.SecretKey;

import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.MatcherAssert.assertThat;

public class CipherSymmetricFactoryTest {

    private CipherSymmetricFactory subject;

    @Before
    public void setUp() {
        subject = new CipherSymmetricFactory();
    }

    @Test
    public void forEncryptWhenSecretKey() throws Exception {
        SecretKeyGenerator secretKeyGenerator = new SecretKeyGenerator();
        SecretKey secretKey = secretKeyGenerator.makeKey(KeyAlgorithm.AES);
        byte[] aad = Factory.aad();

        Cipher actual = subject.forEncrypt(Transformation.AES_GCM_NO_PADDING, secretKey, aad);

        assertThat(actual, is(notNullValue()));
        assertThat(actual.getAlgorithm(), is(Transformation.AES_GCM_NO_PADDING.getValue()));
        assertThat(actual.getIV(), is(notNullValue()));
    }

    @Test
    public void forDecryptWhenSecretKey() throws Exception {
        SecretKeyGenerator secretKeyGenerator = new SecretKeyGenerator();
        SecretKey secretKey = secretKeyGenerator.makeKey(KeyAlgorithm.AES);
        byte[] aad = Factory.aad();

        byte[] initVector = subject.makeInitVector();
        Cipher actual = subject.forDecrypt(Transformation.AES_GCM_NO_PADDING, secretKey, initVector, aad);

        assertThat(actual, is(notNullValue()));
        assertThat(actual.getAlgorithm(), is(Transformation.AES_GCM_NO_PADDING.getValue()));
        assertThat(actual.getIV(), is(notNullValue()));
        assertThat(actual.getIV(), is(initVector));
    }
}


text:
ACCEPTED

#### According to
Index Fungorum

#### Published in
null

#### Original name
Gloeosporium saponariae Siemaszko

### Remarks
null


text:
import logging

from django.core.management.base import BaseCommand

from corehq.apps.domain.models import Domain
from corehq.apps.app_manager.dbaccessors import get_apps_in_domain
from corehq.apps.app_manager.commcare_settings import get_custom_commcare_settings

logger = logging.getLogger('audit_app_profile_properties')
logger.setLevel('DEBUG')

class Command(BaseCommand):
    help = '''
        Determine how many apps have changed their profile properties from the default values.
    '''

    def handle(self, *args, **options):
        domains = [row['key'] for row in Domain.get_all(include_docs=False)]
        settings = {s['id']: s['default'] for s in get_custom_commcare_settings() if 'default' in s}
        deviant_counts = {id: 0 for id in settings.keys()}
        app_count = 0
        for domain in domains:
            for app in get_apps_in_domain(domain, include_remote=False):
                #logger.info("looking at app {}".format(app.id))
                if ('properties' in app.profile):
                    app_count = app_count + 1
                    for id, default in settings.iteritems():
                        if (id not in app.profile['properties']):
                            #logger.info("{}: not found".format(id))
                            pass
                        elif (app.profile['properties'][id] != default):
                            #logger.info("{}: {} != {}".format(id, app.profile['properties'][id], default))
                            deviant_counts[id] = deviant_counts[id] + 1
                        else:
                            #logger.info("{}: {} == {}".format(id, app.profile['properties'][id], default))
                            pass
        for id, count in deviant_counts.iteritems():
            logger.info("{}\t{}".format(count, id))
        logger.info('done with audit_app_profile_properties, examined {} apps in {} domains'.format(app_count, len(domains)))


text:
module Hotdog
  module Expression
    class ExpressionNode
      def evaluate(environment, options={})
        raise(NotImplementedError.new("must be overridden"))
      end

      def optimize(options={})
        self.dup
      end

      def compact(options={})
        self
      end

      def dump(options={})
        {}
      end

      def ==(other)
        self.dump == other.dump
      end
    end

    class UnaryExpressionNode < ExpressionNode
      attr_reader :op, :expression

      def initialize(op, expression, options={})
        case (op || "not").to_s
        when "NOOP", "noop"
          @op = :NOOP
        when "!", "~", "NOT", "not"
          @op = :NOT
        else
          raise(SyntaxError.new("unknown unary operator: #{op.inspect}"))
        end
        @expression = expression
        @options = {}
      end

      def evaluate(environment, options={})
        case @op
        when :NOOP
          @expression.evaluate(environment, options)
        when :NOT
          values = @expression.evaluate(environment, options).tap do |values|
            environment.logger.debug("expr: #{values.length} value(s)")
          end
          if values.empty?
            EverythingNode.new().evaluate(environment, options).tap do |values|
              environment.logger.debug("NOT expr: #{values.length} value(s)")
            end
          else
            # workaround for "too many terms in compound SELECT"
            min, max = environment.execute("SELECT MIN(id), MAX(id) FROM hosts LIMIT 1;").first.to_a
            sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
            (min / (sqlite_limit_compound_select - 2)).upto(max / (sqlite_limit_compound_select - 2)).flat_map { |i|
              range = ((sqlite_limit_compound_select - 2) * i)...((sqlite_limit_compound_select - 2) * (i + 1))
              selected = values.select { |n| range === n }
              if 0 < selected.length
                q = "SELECT id FROM hosts " \
                      "WHERE ? <= id AND id < ? AND id NOT IN (%s);"
                environment.execute(q % selected.map { "?" }.join(", "), [range.first, range.last] + selected).map { |row| row.first }
              else
                []
              end
            }.tap do |values|
              environment.logger.debug("NOT expr: #{values.length} value(s)")
            end
          end
        else
          []
        end
      end

      def optimize(options={})
        o_self = compact(options)
        if UnaryExpressionNode === o_self
          case o_self.op
          when :NOT
            case o_self.expression
            when EverythingNode
              NothingNode.new(options)
            when NothingNode
              EverythingNode.new(options)
            else
              o_self.optimize1(options)
            end
          else
            o_self.optimize1(options)
          end
        else
          o_self.optimize(options)
        end
      end

      def compact(options={})
        case op
        when :NOOP
          expression.compact(options)
        else
          UnaryExpressionNode.new(
            op,
            expression.compact(options),
          )
        end
      end

      def ==(other)
        self.class === other and @op == other.op and @expression == other.expression
      end

      def dump(options={})
        {unary_op: @op.to_s, expression: @expression.dump(options)}
      end

      protected
      def optimize1(options={})
        case op
        when :NOOP
          expression.optimize(options)
        when :NOT
          if UnaryExpressionNode === expression
            case expression.op
            when :NOOP
              expression.optimize(options)
            when :NOT
              expression.expression.optimize(options)
            else
              self.dup
            end
          else
            optimize2(options)
          end
        else
          self.dup
        end
      end

      def optimize2(options={})
        sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
        case expression
        when QueryExpressionNode
          q = expression.query
          v = expression.values
          if q and v.length <= sqlite_limit_compound_select
            QueryExpressionNode.new("SELECT id AS host_id FROM hosts EXCEPT #{q.sub(/\s*;\s*\z/, "")};", v)
          else
            self.dup
          end
        when TagExpressionNode
          q = expression.maybe_query(options)
          v = expression.condition_values(options)
          if q and v.length <= sqlite_limit_compound_select
            QueryExpressionNode.new("SELECT id AS host_id FROM hosts EXCEPT #{q.sub(/\s*;\s*\z/, "")};", v)
          else
            self.dup
          end
        else
          self.dup
        end
      end
    end

    class BinaryExpressionNode < ExpressionNode
      attr_reader :op, :left, :right

      def initialize(op, left, right, options={})
        case (op || "or").to_s
        when "&&", "&", "AND", "and"
          @op = :AND
        when ",", "||", "|", "OR", "or"
          @op = :OR
        when "^", "XOR", "xor"
          @op = :XOR
        else
          raise(SyntaxError.new("unknown binary operator: #{op.inspect}"))
        end
        @left = left
        @right = right
        @options = {}
      end

      def evaluate(environment, options={})
        case @op
        when :AND
          left_values = @left.evaluate(environment, options).tap do |values|
            environment.logger.debug("lhs(#{values.length})")
          end
          if left_values.empty?
            []
          else
            right_values = @right.evaluate(environment, options).tap do |values|
              environment.logger.debug("rhs(#{values.length})")
            end
            if right_values.empty?
              []
            else
              # workaround for "too many terms in compound SELECT"
              min, max = environment.execute("SELECT MIN(id), MAX(id) FROM hosts LIMIT 1;").first.to_a
              sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
              (min / ((sqlite_limit_compound_select - 2) / 2)).upto(max / ((sqlite_limit_compound_select - 2) / 2)).flat_map { |i|
                range = (((sqlite_limit_compound_select - 2) / 2) * i)...(((sqlite_limit_compound_select - 2) / 2) * (i + 1))
                left_selected = left_values.select { |n| range === n }
                right_selected = right_values.select { |n| range === n }
                if 0 < left_selected.length and 0 < right_selected.length
                  q = "SELECT id FROM hosts " \
                        "WHERE ? <= id AND id < ? AND ( id IN (%s) AND id IN (%s) );"
                  environment.execute(q % [left_selected.map { "?" }.join(", "), right_selected.map { "?" }.join(", ")], [range.first, range.last] + left_selected + right_selected).map { |row| row.first }
                else
                  []
                end
              }.tap do |values|
                environment.logger.debug("lhs(#{left_values.length}) AND rhs(#{right_values.length}) => #{values.length}")
              end
            end
          end
        when :OR
          left_values = @left.evaluate(environment, options).tap do |values|
            environment.logger.debug("lhs(#{values.length})")
          end
          right_values = @right.evaluate(environment, options).tap do |values|
            environment.logger.debug("rhs(#{values.length})")
          end
          if left_values.empty?
            right_values
          else
            if right_values.empty?
              []
            else
              # workaround for "too many terms in compound SELECT"
              min, max = environment.execute("SELECT MIN(id), MAX(id) FROM hosts LIMIT 1;").first.to_a
              sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
              (min / ((sqlite_limit_compound_select - 2) / 2)).upto(max / ((sqlite_limit_compound_select - 2) / 2)).flat_map { |i|
                range = (((sqlite_limit_compound_select - 2) / 2) * i)...(((sqlite_limit_compound_select - 2) / 2) * (i + 1))
                left_selected = left_values.select { |n| range === n }
                right_selected = right_values.select { |n| range === n }
                if 0 < left_selected.length or 0 < right_selected.length
                  q = "SELECT id FROM hosts " \
                        "WHERE ? <= id AND id < ? AND ( id IN (%s) OR id IN (%s) );"
                  environment.execute(q % [left_selected.map { "?" }.join(", "), right_selected.map { "?" }.join(", ")], [range.first, range.last] + left_selected + right_selected).map { |row| row.first }
                else
                  []
                end
              }.tap do |values|
                environment.logger.debug("lhs(#{left_values.length}) OR rhs(#{right_values.length}) => #{values.length}")
              end
            end
          end
        when :XOR
          left_values = @left.evaluate(environment, options).tap do |values|
            environment.logger.debug("lhs(#{values.length})")
          end
          right_values = @right.evaluate(environment, options).tap do |values|
            environment.logger.debug("rhs(#{values.length})")
          end
          if left_values.empty?
            right_values
          else
            if right_values.empty?
              []
            else
              # workaround for "too many terms in compound SELECT"
              min, max = environment.execute("SELECT MIN(id), MAX(id) FROM hosts LIMIT 1;").first.to_a
              sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
              (min / ((sqlite_limit_compound_select - 2) / 4)).upto(max / ((sqlite_limit_compound_select - 2) / 4)).flat_map { |i|
                range = (((sqlite_limit_compound_select - 2) / 4) * i)...(((sqlite_limit_compound_select - 2) / 4) * (i + 1))
                left_selected = left_values.select { |n| range === n }
                right_selected = right_values.select { |n| range === n }
                if 0 < left_selected.length or 0 < right_selected.length
                  q = "SELECT id FROM hosts " \
                        "WHERE ? <= id AND id < ? AND NOT (id IN (%s) AND id IN (%s)) AND ( id IN (%s) OR id IN (%s) );"
                  lq = left_selected.map { "?" }.join(", ")
                  rq = right_selected.map { "?" }.join(", ")
                  environment.execute(q % [lq, rq, lq, rq], [range.first, range.last] + left_selected + right_selected + left_selected + right_selected).map { |row| row.first }
                else
                  []
                end
              }.tap do |values|
                environment.logger.debug("lhs(#{left_values.length}) XOR rhs(#{right_values.length}) => #{values.length}")
              end
            end
          end
        else
          []
        end
      end

      def optimize(options={})
        o_left = @left.optimize(options)
        o_right = @right.optimize(options)
        case op
        when :AND
          case o_left
          when EverythingNode
            o_right
          when NothingNode
            o_left
          else
            if o_left == o_right
              o_left
            else
              BinaryExpressionNode.new(
                op,
                o_left,
                o_right,
              ).optimize1(options)
            end
          end
        when :OR
          case o_left
          when EverythingNode
            o_left
          when NothingNode
            o_right
          else
            if o_left == o_right
              o_left
            else
              if MultinaryExpressionNode === o_left
                if o_left.op == op
                  o_left.merge(o_right, fallback: self)
                else
                  BinaryExpressionNode.new(
                    op,
                    o_left,
                    o_right,
                  ).optimize1(options)
                end
              else
                if MultinaryExpressionNode === o_right
                  if o_right.op == op
                    o_right.merge(o_left, fallback: self)
                  else
                    BinaryExpressionNode.new(
                      op,
                      o_left,
                      o_right,
                    ).optimize1(options)
                  end
                else
                  MultinaryExpressionNode.new(op, [o_left, o_right], fallback: self)
                end
              end
            end
          end
        when :XOR
          if o_left == o_right
            NothingNode.new(options)
          else
            BinaryExpressionNode.new(
              op,
              o_left,
              o_right,
            ).optimize1(options)
          end
        else
          self.dup
        end
      end

      def ==(other)
        self.class === other and @op == other.op and @left == other.left and @right == other.right
      end

      def dump(options={})
        {left: @left.dump(options), binary_op: @op.to_s, right: @right.dump(options)}
      end

      protected
      def optimize1(options)
        if TagExpressionNode === left and TagExpressionNode === right
          lq = left.maybe_query(options)
          lv = left.condition_values(options)
          rq = right.maybe_query(options)
          rv = right.condition_values(options)
          sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
          if lq and rq and lv.length + rv.length <= sqlite_limit_compound_select
            case op
            when :AND
              q = "#{lq.sub(/\s*;\s*\z/, "")} INTERSECT #{rq.sub(/\s*;\s*\z/, "")};"
              QueryExpressionNode.new(q, lv + rv, fallback: self)
            when :OR
              q = "#{lq.sub(/\s*;\s*\z/, "")} UNION #{rq.sub(/\s*;\s*\z/, "")};"
              QueryExpressionNode.new(q, lv + rv, fallback: self)
            when :XOR
              q = "#{lq.sub(/\s*;\s*\z/, "")} UNION #{rq.sub(/\s*;\s*\z/, "")} " \
                    "EXCEPT #{lq.sub(/\s*;\s*\z/, "")} " \
                      "INTERSECT #{rq.sub(/\s*;\s*\z/, "")};"
              QueryExpressionNode.new(q, lv + rv, fallback: self)
            else
              self.dup
            end
          else
            self.dup
          end
        else
          self.dup
        end
      end
    end

    class MultinaryExpressionNode < ExpressionNode
      attr_reader :op, :expressions

      def initialize(op, expressions, options={})
        case (op || "or").to_s
        when ",", "||", "|", "OR", "or"
          @op = :OR
        else
          raise(SyntaxError.new("unknown multinary operator: #{op.inspect}"))
        end
        sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
        if sqlite_limit_compound_select < expressions.length
          raise(ArgumentError.new("expressions limit exceeded: #{expressions.length} for #{sqlite_limit_compound_select}"))
        end
        @expressions = expressions
        @options = options
      end

      def merge(other, options={})
        if MultinaryExpressionNode === other and op == other.op
          MultinaryExpressionNode.new(op, expressions + other.expressions, options)
        else
          MultinaryExpressionNode.new(op, expressions + [other], options)
        end
      end

      def evaluate(environment, options={})
        case @op
        when :OR
          if expressions.all? { |expression| TagExpressionNode === expression }
            values = expressions.group_by { |expression| expression.class }.values.flat_map { |expressions|
              query_without_condition = expressions.first.maybe_query_without_condition(options)
              if query_without_condition
                condition_length = expressions.map { |expression| expression.condition_values(options).length }.max
                sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
                expressions.each_slice(sqlite_limit_compound_select / condition_length).flat_map { |expressions|
                  q = query_without_condition.sub(/\s*;\s*\z/, " WHERE #{expressions.map { |expression| "( %s )" % expression.condition(options) }.join(" OR ")};")
                  environment.execute(q, expressions.flat_map { |expression| expression.condition_values(options) }).map { |row| row.first }
                }
              else
                []
              end
            }
          else
            values = []
          end
        else
          values = []
        end
        if values.empty?
          if @options[:fallback]
            @options[:fallback].evaluate(environment, options={})
          else
            []
          end
        else
          values
        end
      end

      def dump(options={})
        {multinary_op: @op.to_s, expressions: expressions.map { |expression| expression.dump(options) }}
      end
    end

    class QueryExpressionNode < ExpressionNode
      def initialize(query, values=[], options={})
        @query = query
        @values = values
        @options = options
      end
      attr_reader :query
      attr_reader :values

      def evaluate(environment, options={})
        values = environment.execute(@query, @values).map { |row| row.first }
        if values.empty? and @options[:fallback]
          @options[:fallback].evaluate(environment, options)
        else
          values
        end
      end

      def dump(options={})
        data = {query: @query, values: @values}
        data[:fallback] = @options[:fallback].dump(options) if @options[:fallback]
        data
      end
    end

    class FuncallNode < ExpressionNode
      attr_reader :function, :args

      def initialize(function, args, options={})
        # FIXME: smart argument handling (e.g. arity & type checking)
        case function.to_s
        when "HEAD", "head"
          @function = :HEAD
        when "GROUP_BY", "group_by"
          @function = :GROUP_BY
        when "LIMIT", "limit"
          @function = :HEAD
        when "ORDER_BY", "order_by"
          @function = :ORDER_BY
        when "REVERSE", "reverse"
          @function = :REVERSE
        when "SAMPLE", "sample"
          @function = :HEAD
          args[0] = FuncallNode.new("SHUFFLE", [args[0]])
        when "SHUFFLE", "shuffle"
          @function = :SHUFFLE
        when "SLICE", "slice"
          @function = :SLICE
        when "SORT", "sort"
          @function = :ORDER_BY
        when "TAIL", "tail"
          @function = :TAIL
        else
          raise(SyntaxError.new("unknown function call: #{function}"))
        end
        @args = args
        @options = options
      end

      def dump(options={})
        args = @args.map { |arg|
          if ExpressionNode === arg
            arg.dump(options)
          else
            arg
          end
        }
        {funcall: @function.to_s, args: args}
      end

      def optimize(options={})
        case function
        when :GROUP_BY
          o_args = [@args[0].optimize(options)]
          if TagExpressionNode === args[1]
            # workaround for expressions like `ORDER_BY((environment:development),role)`
            o_args << @args[1].tagname
          else
            o_args << @args[1]
          end
        when :ORDER_BY
          o_args = [@args[0].optimize(options)]
          if @args[1]
            if TagExpressionNode === @args[1]
              # workaround for expressions like `ORDER_BY((environment:development),role)`
              o_args << @args[1].tagname
            else
              o_args << @args[1]
            end
          end
        else
          o_args = @args.map { |arg|
            if ExpressionNode === arg
              arg.optimize(options)
            else
              arg
            end
          }
        end
        FuncallNode.new(
          @function,
          o_args,
        )
      end

      def evaluate(environment, options={})
        case function
        when :HEAD
          args[0].evaluate(environment, options).take(args[1] || 1)
        when :GROUP_BY
          intermediate = args[0].evaluate(environment, options)
          q = "SELECT DISTINCT hosts_tags.host_id FROM hosts_tags " \
                "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
                "WHERE tags.name = ? AND hosts_tags.host_id IN (%s) " \
                "GROUP BY tags.value;" % intermediate.map { "?" }.join(", ")
          QueryExpressionNode.new(q, [args[1]] + intermediate).evaluate(environment, options)
        when :ORDER_BY
          intermediate = args[0].evaluate(environment, options)
          if args[1]
            q = "SELECT DISTINCT hosts_tags.host_id FROM hosts_tags " \
                  "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
                  "WHERE tags.name = ? AND hosts_tags.host_id IN (%s) " \
                  "ORDER BY tags.value;" % intermediate.map { "?" }.join(", ")
            QueryExpressionNode.new(q, [args[1]] + intermediate).evaluate(environment, options)
          else
            q = "SELECT DISTINCT hosts_tags.host_id FROM hosts_tags " \
                  "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
                  "WHERE hosts_tags.host_id IN (%s) " \
                  "ORDER BY hosts_tags.host_id;" % intermediate.map { "?" }.join(", ")
            QueryExpressionNode.new(q, intermediate).evaluate(environment, options)
          end
        when :REVERSE
          args[0].evaluate(environment, options).reverse()
        when :SHUFFLE
          args[0].evaluate(environment, options).shuffle()
        when :SLICE
          args[0].evaluate(environment, options).slice(args[1], args[2] || 1)
        when :TAIL
          args[0].evaluate(environment, options).last(args[1] || 1)
        else
          []
        end
      end
    end

    class EverythingNode < QueryExpressionNode
      def initialize(options={})
        super("SELECT id AS host_id FROM hosts;", [], options)
      end
    end

    class NothingNode < QueryExpressionNode
      def initialize(options={})
        super("SELECT NULL AS host_id WHERE host_id NOT NULL;", [], options)
      end
    end

    class TagExpressionNode < ExpressionNode
      def initialize(tagname, tagvalue, separator=nil, options={})
        @tagname = tagname
        @tagvalue = tagvalue
        @separator = separator
        @options = options
      end
      attr_reader :tagname
      attr_reader :tagvalue
      attr_reader :separator

      def tagname?
        !(tagname.nil? or tagname.to_s.empty?)
      end

      def tagvalue?
        !(tagvalue.nil? or tagvalue.to_s.empty?)
      end

      def separator?
        !(separator.nil? or separator.to_s.empty?)
      end

      def maybe_query(options={})
        query_without_condition = maybe_query_without_condition(options)
        if query_without_condition
          query_without_condition.sub(/\s*;\s*\z/, " WHERE #{condition(options)};")
        else
          nil
        end
      end

      def maybe_query_without_condition(options={})
        tables = condition_tables(options)
        if tables.empty?
          nil
        else
          case tables
          when [:hosts]
            "SELECT hosts.id AS host_id FROM hosts;"
          when [:hosts, :tags]
            "SELECT DISTINCT hosts_tags.host_id FROM hosts_tags INNER JOIN hosts ON hosts_tags.host_id = hosts.id INNER JOIN tags ON hosts_tags.tag_id = tags.id;"
          when [:tags]
            "SELECT DISTINCT hosts_tags.host_id FROM hosts_tags INNER JOIN tags ON hosts_tags.tag_id = tags.id;"
          else
            raise(NotImplementedError.new("unknown tables: #{tables.join(", ")}"))
          end
        end
      end

      def condition(options={})
        raise(NotImplementedError.new("must be overridden"))
      end

      def condition_tables(options={})
        raise(NotImplementedError.new("must be overridden"))
      end

      def condition_values(options={})
        raise(NotImplementedError.new("must be overridden"))
      end

      def evaluate(environment, options={})
        q = maybe_query(options)
        if q
          values = environment.execute(q, condition_values(options)).map { |row| row.first }
          if values.empty?
            if options[:did_fallback]
              []
            else
              if not environment.fixed_string? and @options[:fallback]
                # avoid optimizing @options[:fallback] to prevent infinite recursion
                @options[:fallback].evaluate(environment, options.merge(did_fallback: true))
              else
                []
              end
            end
          else
            values
          end
        else
          []
        end
      end

      def ==(other)
        self.class == other.class and @tagname == other.tagname and @tagvalue == other.tagvalue
      end

      def optimize(options={})
        # fallback to glob expression
        self.dup.tap do |o_self|
          o_self.instance_eval {
            @options[:fallback] ||= maybe_fallback(options)
          }
        end
      end

      def to_glob(s)
        (s.start_with?("*") ? "" : "*") + s.gsub(/[-.\/_]/, "?") + (s.end_with?("*") ? "" : "*")
      end

      def maybe_glob(s)
        s ? to_glob(s.to_s) : nil
      end

      def dump(options={})
        data = {}
        data[:tagname] = tagname.to_s if tagname
        data[:separator] = separator.to_s if separator
        data[:tagvalue] = tagvalue.to_s if tagvalue
        data[:fallback] = @options[:fallback].dump(options) if @options[:fallback]
        data
      end

      def maybe_fallback(options={})
        nil
      end
    end

    class AnyHostNode < TagExpressionNode
      def initialize(separator=nil, options={})
        super("@host", nil, separator, options)
      end

      def condition(options={})
        "1"
      end

      def condition_tables(options={})
        [:hosts]
      end

      def condition_values(options={})
        []
      end
    end

    class StringExpressionNode < TagExpressionNode
    end

    class StringHostNode < StringExpressionNode
      def initialize(tagvalue, separator=nil, options={})
        super("@host", tagvalue.to_s, separator, options)
      end

      def condition(options={})
        "hosts.name = ?"
      end

      def condition_tables(options={})
        [:hosts]
      end

      def condition_values(options={})
        [tagvalue]
      end

      def maybe_fallback(options={})
        fallback = GlobHostNode.new(to_glob(tagvalue), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class StringTagNode < StringExpressionNode
      def initialize(tagname, tagvalue, separator=nil, options={})
        super(tagname.to_s, tagvalue.to_s, separator, options)
      end

      def condition(options={})
        "tags.name = ? AND tags.value = ?"
      end

      def condition_tables(options={})
        [:tags]
      end

      def condition_values(options={})
        [tagname, tagvalue]
      end

      def maybe_fallback(options={})
        fallback = GlobTagNode.new(to_glob(tagname), to_glob(tagvalue), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class StringTagnameNode < StringExpressionNode
      def initialize(tagname, separator=nil, options={})
        super(tagname.to_s, nil, separator, options)
      end

      def condition(options={})
        "tags.name = ?"
      end

      def condition_tables(options={})
        [:tags]
      end

      def condition_values(options={})
        [tagname]
      end

      def maybe_fallback(options={})
        fallback = GlobTagnameNode.new(to_glob(tagname), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class StringTagvalueNode < StringExpressionNode
      def initialize(tagvalue, separator=nil, options={})
        super(nil, tagvalue.to_s, separator, options)
      end

      def condition(options={})
        "hosts.name = ? OR tags.value = ?"
      end

      def condition_tables(options={})
        [:hosts, :tags]
      end

      def condition_values(options={})
        [tagvalue, tagvalue]
      end

      def maybe_fallback(options={})
        fallback = GlobTagvalueNode.new(to_glob(tagvalue), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class StringHostOrTagNode < StringExpressionNode
      def initialize(tagname, separator=nil, options={})
        super(tagname.to_s, nil, separator, options)
      end

      def condition(options={})
        "hosts.name = ? OR tags.name = ? OR tags.value = ?"
      end

      def condition_tables(options={})
        [:hosts, :tags]
      end

      def condition_values(options={})
        [tagname, tagname, tagname]
      end

      def maybe_fallback(options={})
        fallback = GlobHostOrTagNode.new(to_glob(tagname), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class GlobExpressionNode < TagExpressionNode
      def dump(options={})
        data = {}
        data[:tagname_glob] = tagname.to_s if tagname
        data[:separator] = separator.to_s if separator
        data[:tagvalue_glob] = tagvalue.to_s if tagvalue
        data[:fallback] = @options[:fallback].dump(options) if @options[:fallback]
        data
      end
    end

    class GlobHostNode < GlobExpressionNode
      def initialize(tagvalue, separator=nil, options={})
        super("@host", tagvalue.to_s, separator, options)
      end

      def condition(options={})
        "LOWER(hosts.name) GLOB LOWER(?)"
      end

      def condition_tables(options={})
        [:hosts]
      end

      def condition_values(options={})
        [tagvalue]
      end

      def maybe_fallback(options={})
        fallback = GlobHostNode.new(to_glob(tagvalue), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class GlobTagNode < GlobExpressionNode
      def initialize(tagname, tagvalue, separator=nil, options={})
        super(tagname.to_s, tagvalue.to_s, separator, options)
      end

      def condition(options={})
        "LOWER(tags.name) GLOB LOWER(?) AND LOWER(tags.value) GLOB LOWER(?)"
      end

      def condition_tables(options={})
        [:tags]
      end

      def condition_values(options={})
        [tagname, tagvalue]
      end

      def maybe_fallback(options={})
        fallback = GlobTagNode.new(to_glob(tagname), to_glob(tagvalue), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class GlobTagnameNode < GlobExpressionNode
      def initialize(tagname, separator=nil, options={})
        super(tagname.to_s, nil, separator, options)
      end

      def condition(options={})
        "LOWER(tags.name) GLOB LOWER(?)"
      end

      def condition_tables(options={})
        [:tags]
      end

      def condition_values(options={})
        [tagname]
      end

      def maybe_fallback(options={})
        fallback = GlobTagnameNode.new(to_glob(tagname), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class GlobTagvalueNode < GlobExpressionNode
      def initialize(tagvalue, separator=nil, options={})
        super(nil, tagvalue.to_s, separator, options)
      end

      def condition(options={})
        "LOWER(hosts.name) GLOB LOWER(?) OR LOWER(tags.value) GLOB LOWER(?)"
      end

      def condition_tables(options={})
        [:hosts, :tags]
      end

      def condition_values(options={})
        [tagvalue, tagvalue]
      end

      def maybe_fallback(options={})
        fallback = GlobTagvalueNode.new(to_glob(tagvalue), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class GlobHostOrTagNode < GlobExpressionNode
      def initialize(tagname, separator=nil, options={})
        super(tagname.to_s, nil, separator, options)
      end

      def condition(options={})
        "LOWER(hosts.name) GLOB LOWER(?) OR LOWER(tags.name) GLOB LOWER(?) OR LOWER(tags.value) GLOB LOWER(?)"
      end

      def condition_tables(options={})
        [:hosts, :tags]
      end

      def condition_values(options={})
        [tagname, tagname, tagname]
      end

      def maybe_fallback(options={})
        fallback = GlobHostOrTagNode.new(to_glob(tagname), separator)
        query = fallback.maybe_query(options)
        if query
          QueryExpressionNode.new(query, fallback.condition_values(options))
        else
          nil
        end
      end
    end

    class RegexpExpressionNode < TagExpressionNode
      def dump(options={})
        data = {}
        data[:tagname_regexp] = tagname.to_s if tagname
        data[:separator] = separator.to_s if separator
        data[:tagvalue_regexp] = tagvalue.to_s if tagvalue
        data[:fallback] = @options[:fallback].dump(options) if @options[:fallback]
        data
      end
    end

    class RegexpHostNode < RegexpExpressionNode
      def initialize(tagvalue, separator=nil, options={})
        case tagvalue
        when /\A\/(.*)\/\z/
          tagvalue = $1
        end
        super("@host", tagvalue, separator, options)
      end

      def condition(options={})
        "hosts.name REGEXP ?"
      end

      def condition_tables(options={})
        [:hosts]
      end

      def condition_values(options={})
        [tagvalue]
      end
    end

    class RegexpTagNode < RegexpExpressionNode
      def initialize(tagname, tagvalue, separator=nil, options={})
        case tagname
        when /\A\/(.*)\/\z/
          tagname = $1
        end
        case tagvalue
        when /\A\/(.*)\/\z/
          tagvalue = $1
        end
        super(tagname, tagvalue, separator, options)
      end

      def condition(options={})
        "tags.name REGEXP ? AND tags.value REGEXP ?"
      end

      def condition_tables(options={})
        [:tags]
      end

      def condition_values(options={})
        [tagname, tagvalue]
      end
    end

    class RegexpTagnameNode < RegexpExpressionNode
      def initialize(tagname, separator=nil, options={})
        case tagname
        when /\A\/(.*)\/\z/
          tagname = $1
        end
        super(tagname.to_s, nil, separator, options)
      end

      def condition(options={})
        "tags.name REGEXP ?"
      end

      def condition_tables(options={})
        [:tags]
      end

      def condition_values(options={})
        [tagname]
      end
    end

    class RegexpTagvalueNode < RegexpExpressionNode
      def initialize(tagvalue, separator=nil, options={})
        case tagvalue
        when /\A\/(.*)\/\z/
          tagvalue = $1
        end
        super(nil, tagvalue.to_s, separator, options)
      end

      def condition(options={})
        "hosts.name REGEXP ? OR tags.value REGEXP ?"
      end

      def condition_tables(options={})
        [:hosts, :tags]
      end

      def condition_values(options={})
        [tagvalue, tagvalue]
      end
    end

    class RegexpHostOrTagNode < RegexpExpressionNode
      def initialize(tagname, separator=nil, options={})
        super(tagname, nil, separator, options)
      end

      def condition(options={})
        "hosts.name REGEXP ? OR tags.name REGEXP ? OR tags.value REGEXP ?"
      end

      def condition_tables(options={})
        [:hosts, :tags]
      end

      def condition_values(options={})
        [tagname, tagname, tagname]
      end
    end
  end
end

# vim:set ft=ruby :


text:
Let's Encrypt has two methods of verifying ownership of domains. The first is through the addition of a custom DNS record (say acme-12321313.subdomain.domain.com). This is what https://github.com/janeczku/rancher-letsencrypt does. That service creates Let's Encrypt challenges via DNS resolution. The other way of proving ownership of domains is through a webserver webroot over HTTP.

*Update:* The [janeczku/rancher-letsencrypt](https://github.com/janeczku/rancher-letsencrypt) project [now supports HTTP webroot verification](https://github.com/janeczku/rancher-letsencrypt/commit/2777fcd8eb15fed992a01d41387d2904e010e501). The Tozny project was created many months before this feature was added.

## Our Service

With our environment, we wanted to do webroot verification for Let's Encrypt and Rancher. We wanted a service that would manage TLS certificates automatically, and renew them as needed. We also wanted this tightly integrated with Rancher for complete automation. This way load balancers (and other services) could automatically pick up certs through the Rancher API. Also, when we update a cert in Rancher, the load balancers will receive the updated cert with zero downtime. We also did not want to give keys for updating DNS records for our entire domain to every rancher environment for security purposes (isolation is best!)

Tozny has been using this service in production for over a year now, and has been battle tested. We renew over 40 subdomains regularly without issue.

## How it Works

The service launches two containers:
- `letsencrypt-nginx`
- `letsencrypt-python`

The `letsencrypt-nginx` container is stock nginx, but shares the webroot with the `letsencrypt-python` service container. This way the `letsencrypt-python` container can add ACME challenges to the `<host>/.well-known/acme-challenge/` directory on the webserver for verification. The python container is a sidekick of the nginx container. The containers are launched as a Rancher Service Account, so special environment variables containing the Rancher server API url, and access keys are passed into the container at runtime.

#### Example Rancher Load Balancer (HAProxy) GUI Config
(Based on Rancher GUI v1.3.3)

1. Use the "Add Service" dropdown to select "Add Load Balancer" or edit an existing Load Balancer
2. If empty, fill in the **Name**
3. Enter the following into the **Port Rules** section for each server for which you are requesting a certificate:

| Access | Protocol | Request Host          | Port | Path          | Target            | Port |
|--------|----------|-----------------------|------|---------------|-------------------|------|
| Public | HTTP     | *yourserver.name.com* | 80   | /.well-known/ | letsencrypt-nginx | 80   |
Example "Target" is based on the default container name `letsencrypt-nginx` used by this project

*Note: If you are using custom haproxy.cfg settings to redirect http traffic to https (or wish to do so now), make sure to exclude the `/.well-known/` directory using `!{ url_dir /.well-known/ }` as in:*

```
frontend 80
  redirect scheme https code 301 if !{ url_dir /.well-known/ } !{ ssl_fc }
```
This example custom haproxy.cfg will merge the redirect setting with the default Rancher haproxy.cfg frontend definition and set up permanent ("301") redirects to HTTPS for *all* other HTTP traffic.

## Requirements

- DNS control of domain names (ability to create host.subdomain.domain.com records to point to Rancher IP)
- Front-end load balancer exposing a privileged port (less than 1024) to the internet for Let's Encrypt verification
- This Rancher service
- Rancher Cattle as Container Scheduler/Orchestrator
- Rancher v1.1.4 - v1.4.2 (versions tested with this service)

## How to use

Create a front end load balancer (or use the one in `traffic-manager` directory). If you are making one, you need to make sure it is a L7 HTTP load balancer on your chosen privileged port. This way the load balancer can redirect /.well-known/\* traffic to the `letsencrypt-nginx` container for verification. You can then route all other traffic to your normal HTTP services. This way only during verification does traffic get directed to the `letsencrypt-nginx` container.

#### Rancher Compose

Use `rancher-compose up` to launch the stack in rancher. **In order to get a Let's Encrypt Production certificate, you must set the environment variable STAGING=False**. This will then tell the service to use the production Let's Encrypt api instead of the staging api.
To use the environment file, you need to pass the path using the `--env-file` or `-e` option.

#### Rancher Catalog (UI)

Add this repository as a catalog to your rancher instance:
1. Open Rancher
2. Select *Admin* in the navigation
3. Select *Settings*

In the *Catalog* section you can add this catalog by entering a name (e.g. `rancher-lets-encrypt`), the URL to this repository and a branch.

Afterwards you will be able to select the new catalog from the `Catalog` menu item in the navigation. There you will find the `Rancher Let's Encrypt Service`. By clicking **View Details** you can configure the service to your needs and then launch it.

# Certificate Workflows

"staging" refers to Let's Encrypt staging API.
"production" refers to Let's Encrypt production API.

This flowchart/execution diagram shows all the cases the service deals with, and how it responds to different stages.

- get certs from rancher API
    - local copy of cert
        - cert in rancher
            - upgrade staging cert to production
                - create cert
                - push to rancher
            - upgrade self signed cert to production
                - create cert
                - push to rancher
            - rancher cert expired
                - local cert expired
                    - create cert (renew)
                    - push to rancher
                - local cert not expired
                    - push to rancher
        - cert not in rancher
            - local cert expired
                - create cert
                - push to rancher
            - local cert not expired
                - push to rancher
    - no local copy of cert
        - create cert
        - push to rancher


text:
package views

import (
	"encoding/json"
	"github.com/lastbackend/lastbackend/internal/util/resource"

	"github.com/lastbackend/lastbackend/internal/pkg/models"
)

type ClusterView struct{}

func (cv *ClusterView) New(obj *models.Cluster) *Cluster {
	c := Cluster{}
	c.Status = cv.ToClusterStatus(obj.Status)
	return &c
}

func (cl *Cluster) ToJson() ([]byte, error) {
	return json.Marshal(cl)
}

func (cv *ClusterView) NewList(obj map[string]*models.Cluster) *ClusterList {
	if obj == nil {
		return nil
	}

	c := make(ClusterList, 0)
	for _, v := range obj {
		c = append(c, cv.New(v))
	}
	return &c
}

func (cl *ClusterList) ToJson() ([]byte, error) {
	if cl == nil {
		cl = &ClusterList{}
	}
	return json.Marshal(cl)
}

func (cv *ClusterView) ClusterStatus() *ClusterStatus {
	return new(ClusterStatus)
}

func (cv *ClusterView) ToClusterStatus(status models.ClusterStatus) ClusterStatus {
	return ClusterStatus{
		Nodes: status.Nodes,
		Capacity: ClusterResources{
			Containers: status.Capacity.Containers,
			Pods:       status.Capacity.Pods,
			RAM:        resource.EncodeMemoryResource(status.Capacity.RAM),
			Cpu:        resource.EncodeCpuResource(status.Capacity.CPU),
			Storage:    resource.EncodeMemoryResource(status.Capacity.Storage),
		},
		Allocated: ClusterResources{
			Containers: status.Allocated.Containers,
			Pods:       status.Allocated.Pods,
			RAM:        resource.EncodeMemoryResource(status.Allocated.RAM),
			Cpu:        resource.EncodeCpuResource(status.Allocated.CPU),
			Storage:    resource.EncodeMemoryResource(status.Allocated.Storage),
		},
	}
}


text:
#include BOSS_OPENSSL_V_openssl__rand_h //original-code:<openssl/rand.h>
#include "../ssl_locl.h"
#include "statem_locl.h"

/*
 * This file implements the SSL/TLS/DTLS state machines.
 *
 * There are two primary state machines:
 *
 * 1) Message flow state machine
 * 2) Handshake state machine
 *
 * The Message flow state machine controls the reading and sending of messages
 * including handling of non-blocking IO events, flushing of the underlying
 * write BIO, handling unexpected messages, etc. It is itself broken into two
 * separate sub-state machines which control reading and writing respectively.
 *
 * The Handshake state machine keeps track of the current SSL/TLS handshake
 * state. Transitions of the handshake state are the result of events that
 * occur within the Message flow state machine.
 *
 * Overall it looks like this:
 *
 * ---------------------------------------------            -------------------
 * |                                           |            |                 |
 * | Message flow state machine                |            |                 |
 * |                                           |            |                 |
 * | -------------------- -------------------- | Transition | Handshake state |
 * | | MSG_FLOW_READING | | MSG_FLOW_WRITING | | Event      | machine         |
 * | | sub-state        | | sub-state        | |----------->|                 |
 * | | machine for      | | machine for      | |            |                 |
 * | | reading messages | | writing messages | |            |                 |
 * | -------------------- -------------------- |            |                 |
 * |                                           |            |                 |
 * ---------------------------------------------            -------------------
 *
 */

/* Sub state machine return values */
typedef enum {
    /* Something bad happened or NBIO */
    SUB_STATE_ERROR,
    /* Sub state finished go to the next sub state */
    SUB_STATE_FINISHED,
    /* Sub state finished and handshake was completed */
    SUB_STATE_END_HANDSHAKE
} SUB_STATE_RETURN;

static int state_machine(SSL *s, int server);
static void init_read_state_machine(SSL *s);
static SUB_STATE_RETURN read_state_machine(SSL *s);
static void init_write_state_machine(SSL *s);
static SUB_STATE_RETURN write_state_machine(SSL *s);

OSSL_HANDSHAKE_STATE SSL_get_state(const SSL *ssl)
{
    return ssl->statem.hand_state;
}

int SSL_in_init(SSL *s)
{
    return s->statem.in_init;
}

int SSL_is_init_finished(SSL *s)
{
    return !(s->statem.in_init) && (s->statem.hand_state == TLS_ST_OK);
}

int SSL_in_before(SSL *s)
{
    /*
     * Historically being "in before" meant before anything had happened. In the
     * current code though we remain in the "before" state for a while after we
     * have started the handshake process (e.g. as a server waiting for the
     * first message to arrive). There "in before" is taken to mean "in before"
     * and not started any handshake process yet.
     */
    return (s->statem.hand_state == TLS_ST_BEFORE)
        && (s->statem.state == MSG_FLOW_UNINITED);
}

/*
 * Clear the state machine state and reset back to MSG_FLOW_UNINITED
 */
void ossl_statem_clear(SSL *s)
{
    s->statem.state = MSG_FLOW_UNINITED;
    s->statem.hand_state = TLS_ST_BEFORE;
    s->statem.in_init = 1;
    s->statem.no_cert_verify = 0;
}

/*
 * Set the state machine up ready for a renegotiation handshake
 */
void ossl_statem_set_renegotiate(SSL *s)
{
    s->statem.state = MSG_FLOW_RENEGOTIATE;
    s->statem.in_init = 1;
}

/*
 * Put the state machine into an error state. This is a permanent error for
 * the current connection.
 */
void ossl_statem_set_error(SSL *s)
{
    s->statem.state = MSG_FLOW_ERROR;
}

/*
 * Discover whether the current connection is in the error state.
 *
 * Valid return values are:
 *   1: Yes
 *   0: No
 */
int ossl_statem_in_error(const SSL *s)
{
    if (s->statem.state == MSG_FLOW_ERROR)
        return 1;

    return 0;
}

void ossl_statem_set_in_init(SSL *s, int init)
{
    s->statem.in_init = init;
}

int ossl_statem_get_in_handshake(SSL *s)
{
    return s->statem.in_handshake;
}

void ossl_statem_set_in_handshake(SSL *s, int inhand)
{
    if (inhand)
        s->statem.in_handshake++;
    else
        s->statem.in_handshake--;
}

void ossl_statem_set_hello_verify_done(SSL *s)
{
    s->statem.state = MSG_FLOW_UNINITED;
    s->statem.in_init = 1;
    /*
     * This will get reset (briefly) back to TLS_ST_BEFORE when we enter
     * state_machine() because |state| is MSG_FLOW_UNINITED, but until then any
     * calls to SSL_in_before() will return false. Also calls to
     * SSL_state_string() and SSL_state_string_long() will return something
     * sensible.
     */
    s->statem.hand_state = TLS_ST_SR_CLNT_HELLO;
}

int ossl_statem_connect(SSL *s)
{
    return state_machine(s, 0);
}

int ossl_statem_accept(SSL *s)
{
    return state_machine(s, 1);
}

typedef void (*info_cb) (const SSL *, int, int);

static info_cb get_callback(SSL *s)
{
    if (s->info_callback != NULL)
        return s->info_callback;
    else if (s->ctx->info_callback != NULL)
        return s->ctx->info_callback;

    return NULL;
}

/*
 * The main message flow state machine. We start in the MSG_FLOW_UNINITED or
 * MSG_FLOW_RENEGOTIATE state and finish in MSG_FLOW_FINISHED. Valid states and
 * transitions are as follows:
 *
 * MSG_FLOW_UNINITED     MSG_FLOW_RENEGOTIATE
 *        |                       |
 *        +-----------------------+
 *        v
 * MSG_FLOW_WRITING <---> MSG_FLOW_READING
 *        |
 *        V
 * MSG_FLOW_FINISHED
 *        |
 *        V
 *    [SUCCESS]
 *
 * We may exit at any point due to an error or NBIO event. If an NBIO event
 * occurs then we restart at the point we left off when we are recalled.
 * MSG_FLOW_WRITING and MSG_FLOW_READING have sub-state machines associated with them.
 *
 * In addition to the above there is also the MSG_FLOW_ERROR state. We can move
 * into that state at any point in the event that an irrecoverable error occurs.
 *
 * Valid return values are:
 *   1: Success
 * <=0: NBIO or error
 */
static int state_machine(SSL *s, int server)
{
    BUF_MEM *buf = NULL;
    unsigned long Time = (unsigned long)time(NULL);
    void (*cb) (const SSL *ssl, int type, int val) = NULL;
    OSSL_STATEM *st = &s->statem;
    int ret = -1;
    int ssret;

    if (st->state == MSG_FLOW_ERROR) {
        /* Shouldn't have been called if we're already in the error state */
        return -1;
    }

    RAND_add(&Time, sizeof(Time), 0);
    ERR_clear_error();
    clear_sys_error();

    cb = get_callback(s);

    st->in_handshake++;
    if (!SSL_in_init(s) || SSL_in_before(s)) {
        if (!SSL_clear(s))
            return -1;
    }
#ifndef OPENSSL_NO_SCTP
    if (SSL_IS_DTLS(s)) {
        /*
         * Notify SCTP BIO socket to enter handshake mode and prevent stream
         * identifier other than 0. Will be ignored if no SCTP is used.
         */
        BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE,
                 st->in_handshake, NULL);
    }
#endif

#ifndef OPENSSL_NO_HEARTBEATS
    /*
     * If we're awaiting a HeartbeatResponse, pretend we already got and
     * don't await it anymore, because Heartbeats don't make sense during
     * handshakes anyway.
     */
    if (s->tlsext_hb_pending) {
        if (SSL_IS_DTLS(s))
            dtls1_stop_timer(s);
        s->tlsext_hb_pending = 0;
        s->tlsext_hb_seq++;
    }
#endif

    /* Initialise state machine */

    if (st->state == MSG_FLOW_RENEGOTIATE) {
        s->renegotiate = 1;
        if (!server)
            s->ctx->stats.sess_connect_renegotiate++;
    }

    if (st->state == MSG_FLOW_UNINITED || st->state == MSG_FLOW_RENEGOTIATE) {
        if (st->state == MSG_FLOW_UNINITED) {
            st->hand_state = TLS_ST_BEFORE;
        }

        s->server = server;
        if (cb != NULL)
            cb(s, SSL_CB_HANDSHAKE_START, 1);

        if (SSL_IS_DTLS(s)) {
            if ((s->version & 0xff00) != (DTLS1_VERSION & 0xff00) &&
                (server || (s->version & 0xff00) != (DTLS1_BAD_VER & 0xff00))) {
                SSLerr(SSL_F_STATE_MACHINE, ERR_R_INTERNAL_ERROR);
                goto end;
            }
        } else {
            if ((s->version >> 8) != SSL3_VERSION_MAJOR) {
                SSLerr(SSL_F_STATE_MACHINE, ERR_R_INTERNAL_ERROR);
                goto end;
            }
        }

        if (!ssl_security(s, SSL_SECOP_VERSION, 0, s->version, NULL)) {
            SSLerr(SSL_F_STATE_MACHINE, SSL_R_VERSION_TOO_LOW);
            goto end;
        }

        if (s->init_buf == NULL) {
            if ((buf = BUF_MEM_new()) == NULL) {
                goto end;
            }
            if (!BUF_MEM_grow(buf, SSL3_RT_MAX_PLAIN_LENGTH)) {
                goto end;
            }
            s->init_buf = buf;
            buf = NULL;
        }

        if (!ssl3_setup_buffers(s)) {
            goto end;
        }
        s->init_num = 0;

        /*
         * Should have been reset by tls_process_finished, too.
         */
        s->s3->change_cipher_spec = 0;

        /*
         * Ok, we now need to push on a buffering BIO ...but not with
         * SCTP
         */
#ifndef OPENSSL_NO_SCTP
        if (!SSL_IS_DTLS(s) || !BIO_dgram_is_sctp(SSL_get_wbio(s)))
#endif
            if (!ssl_init_wbio_buffer(s)) {
                goto end;
            }

        if (!server || st->state != MSG_FLOW_RENEGOTIATE) {
            if (!ssl3_init_finished_mac(s)) {
                ossl_statem_set_error(s);
                goto end;
            }
        }

        if (server) {
            if (st->state != MSG_FLOW_RENEGOTIATE) {
                s->ctx->stats.sess_accept++;
            } else if (!s->s3->send_connection_binding &&
                       !(s->options &
                         SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION)) {
                /*
                 * Server attempting to renegotiate with client that doesn't
                 * support secure renegotiation.
                 */
                SSLerr(SSL_F_STATE_MACHINE,
                       SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED);
                ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_HANDSHAKE_FAILURE);
                ossl_statem_set_error(s);
                goto end;
            } else {
                /*
                 * st->state == MSG_FLOW_RENEGOTIATE, we will just send a
                 * HelloRequest
                 */
                s->ctx->stats.sess_accept_renegotiate++;
            }
        } else {
            s->ctx->stats.sess_connect++;

            /* mark client_random uninitialized */
            memset(s->s3->client_random, 0, sizeof(s->s3->client_random));
            s->hit = 0;

            s->s3->tmp.cert_request = 0;

            if (SSL_IS_DTLS(s)) {
                st->use_timer = 1;
            }
        }

        st->state = MSG_FLOW_WRITING;
        init_write_state_machine(s);
        st->read_state_first_init = 1;
    }

    while (st->state != MSG_FLOW_FINISHED) {
        if (st->state == MSG_FLOW_READING) {
            ssret = read_state_machine(s);
            if (ssret == SUB_STATE_FINISHED) {
                st->state = MSG_FLOW_WRITING;
                init_write_state_machine(s);
            } else {
                /* NBIO or error */
                goto end;
            }
        } else if (st->state == MSG_FLOW_WRITING) {
            ssret = write_state_machine(s);
            if (ssret == SUB_STATE_FINISHED) {
                st->state = MSG_FLOW_READING;
                init_read_state_machine(s);
            } else if (ssret == SUB_STATE_END_HANDSHAKE) {
                st->state = MSG_FLOW_FINISHED;
            } else {
                /* NBIO or error */
                goto end;
            }
        } else {
            /* Error */
            ossl_statem_set_error(s);
            goto end;
        }
    }

    st->state = MSG_FLOW_UNINITED;
    ret = 1;

 end:
    st->in_handshake--;

#ifndef OPENSSL_NO_SCTP
    if (SSL_IS_DTLS(s)) {
        /*
         * Notify SCTP BIO socket to leave handshake mode and allow stream
         * identifier other than 0. Will be ignored if no SCTP is used.
         */
        BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE,
                 st->in_handshake, NULL);
    }
#endif

    BUF_MEM_free(buf);
    if (cb != NULL) {
        if (server)
            cb(s, SSL_CB_ACCEPT_EXIT, ret);
        else
            cb(s, SSL_CB_CONNECT_EXIT, ret);
    }
    return ret;
}

/*
 * Initialise the MSG_FLOW_READING sub-state machine
 */
static void init_read_state_machine(SSL *s)
{
    OSSL_STATEM *st = &s->statem;

    st->read_state = READ_STATE_HEADER;
}

static int grow_init_buf(SSL *s, size_t size) {

    size_t msg_offset = (char *)s->init_msg - s->init_buf->data;

    if (!BUF_MEM_grow_clean(s->init_buf, (int)size))
        return 0;

    if (size < msg_offset)
        return 0;

    s->init_msg = s->init_buf->data + msg_offset;

    return 1;
}

/*
 * This function implements the sub-state machine when the message flow is in
 * MSG_FLOW_READING. The valid sub-states and transitions are:
 *
 * READ_STATE_HEADER <--+<-------------+
 *        |             |              |
 *        v             |              |
 * READ_STATE_BODY -----+-->READ_STATE_POST_PROCESS
 *        |                            |
 *        +----------------------------+
 *        v
 * [SUB_STATE_FINISHED]
 *
 * READ_STATE_HEADER has the responsibility for reading in the message header
 * and transitioning the state of the handshake state machine.
 *
 * READ_STATE_BODY reads in the rest of the message and then subsequently
 * processes it.
 *
 * READ_STATE_POST_PROCESS is an optional step that may occur if some post
 * processing activity performed on the message may block.
 *
 * Any of the above states could result in an NBIO event occurring in which case
 * control returns to the calling application. When this function is recalled we
 * will resume in the same state where we left off.
 */
static SUB_STATE_RETURN read_state_machine(SSL *s)
{
    OSSL_STATEM *st = &s->statem;
    int ret, mt;
    unsigned long len = 0;
    int (*transition) (SSL *s, int mt);
    PACKET pkt;
    MSG_PROCESS_RETURN(*process_message) (SSL *s, PACKET *pkt);
    WORK_STATE(*post_process_message) (SSL *s, WORK_STATE wst);
    unsigned long (*max_message_size) (SSL *s);
    void (*cb) (const SSL *ssl, int type, int val) = NULL;

    cb = get_callback(s);

    if (s->server) {
        transition = ossl_statem_server_read_transition;
        process_message = ossl_statem_server_process_message;
        max_message_size = ossl_statem_server_max_message_size;
        post_process_message = ossl_statem_server_post_process_message;
    } else {
        transition = ossl_statem_client_read_transition;
        process_message = ossl_statem_client_process_message;
        max_message_size = ossl_statem_client_max_message_size;
        post_process_message = ossl_statem_client_post_process_message;
    }

    if (st->read_state_first_init) {
        s->first_packet = 1;
        st->read_state_first_init = 0;
    }

    while (1) {
        switch (st->read_state) {
        case READ_STATE_HEADER:
            /* Get the state the peer wants to move to */
            if (SSL_IS_DTLS(s)) {
                /*
                 * In DTLS we get the whole message in one go - header and body
                 */
                ret = dtls_get_message(s, &mt, &len);
            } else {
                ret = tls_get_message_header(s, &mt);
            }

            if (ret == 0) {
                /* Could be non-blocking IO */
                return SUB_STATE_ERROR;
            }

            if (cb != NULL) {
                /* Notify callback of an impending state change */
                if (s->server)
                    cb(s, SSL_CB_ACCEPT_LOOP, 1);
                else
                    cb(s, SSL_CB_CONNECT_LOOP, 1);
            }
            /*
             * Validate that we are allowed to move to the new state and move
             * to that state if so
             */
            if (!transition(s, mt)) {
                ossl_statem_set_error(s);
                return SUB_STATE_ERROR;
            }

            if (s->s3->tmp.message_size > max_message_size(s)) {
                ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_ILLEGAL_PARAMETER);
                SSLerr(SSL_F_READ_STATE_MACHINE, SSL_R_EXCESSIVE_MESSAGE_SIZE);
                return SUB_STATE_ERROR;
            }

            /* dtls_get_message already did this */
            if (!SSL_IS_DTLS(s)
                    && s->s3->tmp.message_size > 0
                    && !grow_init_buf(s, s->s3->tmp.message_size
                                         + SSL3_HM_HEADER_LENGTH)) {
                ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_INTERNAL_ERROR);
                SSLerr(SSL_F_READ_STATE_MACHINE, ERR_R_BUF_LIB);
                return SUB_STATE_ERROR;
            }

            st->read_state = READ_STATE_BODY;
            /* Fall through */

        case READ_STATE_BODY:
            if (!SSL_IS_DTLS(s)) {
                /* We already got this above for DTLS */
                ret = tls_get_message_body(s, &len);
                if (ret == 0) {
                    /* Could be non-blocking IO */
                    return SUB_STATE_ERROR;
                }
            }

            s->first_packet = 0;
            if (!PACKET_buf_init(&pkt, s->init_msg, len)) {
                ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_INTERNAL_ERROR);
                SSLerr(SSL_F_READ_STATE_MACHINE, ERR_R_INTERNAL_ERROR);
                return SUB_STATE_ERROR;
            }
            ret = process_message(s, &pkt);

            /* Discard the packet data */
            s->init_num = 0;

            switch (ret) {
            case MSG_PROCESS_ERROR:
                return SUB_STATE_ERROR;

            case MSG_PROCESS_FINISHED_READING:
                if (SSL_IS_DTLS(s)) {
                    dtls1_stop_timer(s);
                }
                return SUB_STATE_FINISHED;

            case MSG_PROCESS_CONTINUE_PROCESSING:
                st->read_state = READ_STATE_POST_PROCESS;
                st->read_state_work = WORK_MORE_A;
                break;

            default:
                st->read_state = READ_STATE_HEADER;
                break;
            }
            break;

        case READ_STATE_POST_PROCESS:
            st->read_state_work = post_process_message(s, st->read_state_work);
            switch (st->read_state_work) {
            default:
                return SUB_STATE_ERROR;

            case WORK_FINISHED_CONTINUE:
                st->read_state = READ_STATE_HEADER;
                break;

            case WORK_FINISHED_STOP:
                if (SSL_IS_DTLS(s)) {
                    dtls1_stop_timer(s);
                }
                return SUB_STATE_FINISHED;
            }
            break;

        default:
            /* Shouldn't happen */
            ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_INTERNAL_ERROR);
            SSLerr(SSL_F_READ_STATE_MACHINE, ERR_R_INTERNAL_ERROR);
            ossl_statem_set_error(s);
            return SUB_STATE_ERROR;
        }
    }
}

/*
 * Send a previously constructed message to the peer.
 */
static int statem_do_write(SSL *s)
{
    OSSL_STATEM *st = &s->statem;

    if (st->hand_state == TLS_ST_CW_CHANGE
        || st->hand_state == TLS_ST_SW_CHANGE) {
        if (SSL_IS_DTLS(s))
            return dtls1_do_write(s, SSL3_RT_CHANGE_CIPHER_SPEC);
        else
            return ssl3_do_write(s, SSL3_RT_CHANGE_CIPHER_SPEC);
    } else {
        return ssl_do_write(s);
    }
}

/*
 * Initialise the MSG_FLOW_WRITING sub-state machine
 */
static void init_write_state_machine(SSL *s)
{
    OSSL_STATEM *st = &s->statem;

    st->write_state = WRITE_STATE_TRANSITION;
}

/*
 * This function implements the sub-state machine when the message flow is in
 * MSG_FLOW_WRITING. The valid sub-states and transitions are:
 *
 * +-> WRITE_STATE_TRANSITION ------> [SUB_STATE_FINISHED]
 * |             |
 * |             v
 * |      WRITE_STATE_PRE_WORK -----> [SUB_STATE_END_HANDSHAKE]
 * |             |
 * |             v
 * |       WRITE_STATE_SEND
 * |             |
 * |             v
 * |     WRITE_STATE_POST_WORK
 * |             |
 * +-------------+
 *
 * WRITE_STATE_TRANSITION transitions the state of the handshake state machine

 * WRITE_STATE_PRE_WORK performs any work necessary to prepare the later
 * sending of the message. This could result in an NBIO event occurring in
 * which case control returns to the calling application. When this function
 * is recalled we will resume in the same state where we left off.
 *
 * WRITE_STATE_SEND sends the message and performs any work to be done after
 * sending.
 *
 * WRITE_STATE_POST_WORK performs any work necessary after the sending of the
 * message has been completed. As for WRITE_STATE_PRE_WORK this could also
 * result in an NBIO event.
 */
static SUB_STATE_RETURN write_state_machine(SSL *s)
{
    OSSL_STATEM *st = &s->statem;
    int ret;
    WRITE_TRAN(*transition) (SSL *s);
    WORK_STATE(*pre_work) (SSL *s, WORK_STATE wst);
    WORK_STATE(*post_work) (SSL *s, WORK_STATE wst);
    int (*construct_message) (SSL *s);
    void (*cb) (const SSL *ssl, int type, int val) = NULL;

    cb = get_callback(s);

    if (s->server) {
        transition = ossl_statem_server_write_transition;
        pre_work = ossl_statem_server_pre_work;
        post_work = ossl_statem_server_post_work;
        construct_message = ossl_statem_server_construct_message;
    } else {
        transition = ossl_statem_client_write_transition;
        pre_work = ossl_statem_client_pre_work;
        post_work = ossl_statem_client_post_work;
        construct_message = ossl_statem_client_construct_message;
    }

    while (1) {
        switch (st->write_state) {
        case WRITE_STATE_TRANSITION:
            if (cb != NULL) {
                /* Notify callback of an impending state change */
                if (s->server)
                    cb(s, SSL_CB_ACCEPT_LOOP, 1);
                else
                    cb(s, SSL_CB_CONNECT_LOOP, 1);
            }
            switch (transition(s)) {
            case WRITE_TRAN_CONTINUE:
                st->write_state = WRITE_STATE_PRE_WORK;
                st->write_state_work = WORK_MORE_A;
                break;

            case WRITE_TRAN_FINISHED:
                return SUB_STATE_FINISHED;
                break;

            default:
                return SUB_STATE_ERROR;
            }
            break;

        case WRITE_STATE_PRE_WORK:
            switch (st->write_state_work = pre_work(s, st->write_state_work)) {
            default:
                return SUB_STATE_ERROR;

            case WORK_FINISHED_CONTINUE:
                st->write_state = WRITE_STATE_SEND;
                break;

            case WORK_FINISHED_STOP:
                return SUB_STATE_END_HANDSHAKE;
            }
            if (construct_message(s) == 0)
                return SUB_STATE_ERROR;

            /* Fall through */

        case WRITE_STATE_SEND:
            if (SSL_IS_DTLS(s) && st->use_timer) {
                dtls1_start_timer(s);
            }
            ret = statem_do_write(s);
            if (ret <= 0) {
                return SUB_STATE_ERROR;
            }
            st->write_state = WRITE_STATE_POST_WORK;
            st->write_state_work = WORK_MORE_A;
            /* Fall through */

        case WRITE_STATE_POST_WORK:
            switch (st->write_state_work = post_work(s, st->write_state_work)) {
            default:
                return SUB_STATE_ERROR;

            case WORK_FINISHED_CONTINUE:
                st->write_state = WRITE_STATE_TRANSITION;
                break;

            case WORK_FINISHED_STOP:
                return SUB_STATE_END_HANDSHAKE;
            }
            break;

        default:
            return SUB_STATE_ERROR;
        }
    }
}

/*
 * Flush the write BIO
 */
int statem_flush(SSL *s)
{
    s->rwstate = SSL_WRITING;
    if (BIO_flush(s->wbio) <= 0) {
        return 0;
    }
    s->rwstate = SSL_NOTHING;

    return 1;
}

/*
 * Called by the record layer to determine whether application data is
 * allowed to be sent in the current handshake state or not.
 *
 * Return values are:
 *   1: Yes (application data allowed)
 *   0: No (application data not allowed)
 */
int ossl_statem_app_data_allowed(SSL *s)
{
    OSSL_STATEM *st = &s->statem;

    if (st->state == MSG_FLOW_UNINITED || st->state == MSG_FLOW_RENEGOTIATE)
        return 0;

    if (!s->s3->in_read_app_data || (s->s3->total_renegotiations == 0))
        return 0;

    if (s->server) {
        /*
         * If we're a server and we haven't got as far as writing our
         * ServerHello yet then we allow app data
         */
        if (st->hand_state == TLS_ST_BEFORE
            || st->hand_state == TLS_ST_SR_CLNT_HELLO)
            return 1;
    } else {
        /*
         * If we're a client and we haven't read the ServerHello yet then we
         * allow app data
         */
        if (st->hand_state == TLS_ST_CW_CLNT_HELLO)
            return 1;
    }

    return 0;
}

#ifndef OPENSSL_NO_SCTP
/*
 * Set flag used by SCTP to determine whether we are in the read sock state
 */
void ossl_statem_set_sctp_read_sock(SSL *s, int read_sock)
{
    s->statem.in_sctp_read_sock = read_sock;
}

/*
 * Called by the record layer to determine whether we are in the read sock
 * state or not.
 *
 * Return values are:
 *   1: Yes (we are in the read sock state)
 *   0: No (we are not in the read sock state)
 */
int ossl_statem_in_sctp_read_sock(SSL *s)
{
    return s->statem.in_sctp_read_sock;
}
#endif


text:
<!DOCTYPE html>

    The layer trees before and after should be the same, except for the position of the 50x50 sized composited fixed position layer.

    BEFORE:

    AFTER:


text:
package mocks

import (
	"sync"

	"github.com/umputun/feed-master/app/feed"
)

// TwitterNotifMock is a mock implementation of proc.TwitterNotif.
//
// 	func TestSomethingThatUsesTwitterNotif(t *testing.T) {
//
// 		// make and configure a mocked proc.TwitterNotif
// 		mockedTwitterNotif := &TwitterNotifMock{
// 			SendFunc: func(item feed.Item) error {
// 				panic("mock out the Send method")
// 			},
// 		}
//
// 		// use mockedTwitterNotif in code that requires proc.TwitterNotif
// 		// and then make assertions.
//
// 	}
type TwitterNotifMock struct {
	// SendFunc mocks the Send method.
	SendFunc func(item feed.Item) error

	// calls tracks calls to the methods.
	calls struct {
		// Send holds details about calls to the Send method.
		Send []struct {
			// Item is the item argument value.
			Item feed.Item
		}
	}
	lockSend sync.RWMutex
}

// Send calls SendFunc.
func (mock *TwitterNotifMock) Send(item feed.Item) error {
	if mock.SendFunc == nil {
		panic("TwitterNotifMock.SendFunc: method is nil but TwitterNotif.Send was just called")
	}
	callInfo := struct {
		Item feed.Item
	}{
		Item: item,
	}
	mock.lockSend.Lock()
	mock.calls.Send = append(mock.calls.Send, callInfo)
	mock.lockSend.Unlock()
	return mock.SendFunc(item)
}

// SendCalls gets all the calls that were made to Send.
// Check the length with:
//     len(mockedTwitterNotif.SendCalls())
func (mock *TwitterNotifMock) SendCalls() []struct {
	Item feed.Item
} {
	var calls []struct {
		Item feed.Item
	}
	mock.lockSend.RLock()
	calls = mock.calls.Send
	mock.lockSend.RUnlock()
	return calls
}