text:
/*
 * ThinClientRedundancyManager.cpp
 *
 *  Created on: Dec 1, 2008
 *      Author: abhaware
 */

#include "ThinClientRedundancyManager.hpp"
#include "TcrHADistributionManager.hpp"
#include "RemoteQueryService.hpp"
#include "CacheImpl.hpp"
#include "ThinClientRegion.hpp"
#include <geode/SystemProperties.hpp>
#include "ClientProxyMembershipID.hpp"
#include "ExpiryHandler_T.hpp"

#include "ThinClientPoolHADM.hpp"
#include "ServerLocation.hpp"
#include "ThinClientLocatorHelper.hpp"
#include "UserAttributes.hpp"
#include "ProxyCache.hpp"
#include <set>
#include <algorithm>

using namespace apache::geode::client;

const char* ThinClientRedundancyManager::NC_PerodicACK = "NC PerodicACK";

ThinClientRedundancyManager::ThinClientRedundancyManager(
    TcrConnectionManager* theConnManager, int redundancyLevel,
    ThinClientPoolHADM* poolHADM, bool sentReadyForEvents,
    bool globalProcessedMarker)
    : m_globalProcessedMarker(globalProcessedMarker),
      m_IsAllEpDisCon(false),
      m_server(0),
      m_sentReadyForEvents(sentReadyForEvents),
      m_redundancyLevel(redundancyLevel),
      m_loggedRedundancyWarning(false),
      m_poolHADM(poolHADM),
      m_theTcrConnManager(theConnManager),
      m_locators(NULLPTR),
      m_servers(NULLPTR),
      m_periodicAckTask(NULL),
      m_processEventIdMapTaskId(-1),
      m_nextAckInc(0),
      m_HAenabled(false) {}

std::list<ServerLocation> ThinClientRedundancyManager::selectServers(
    int howMany, std::set<ServerLocation> exclEndPts) {
  LOGFINE("Selecting %d servers with %d in exclude list", howMany,
          exclEndPts.size());

  std::list<ServerLocation> outEndpoints;

  if (m_locators->length() > 0) {
    try {
      std::string additionalLoc;
      ClientProxyMembershipID* m_proxyID = m_poolHADM->getMembershipId();
      m_poolHADM->getLocatorHelper()->getEndpointForNewCallBackConn(
          *m_proxyID, outEndpoints, additionalLoc, howMany, exclEndPts,
          m_poolHADM->getServerGroup());
    } catch (const AuthenticationRequiredException&) {
      return outEndpoints;
    } catch (const NoAvailableLocatorsException&) {
      LOGFINE("No locators available");
      return outEndpoints;
    }
  } else if (m_servers->length() > 0) {
    if (howMany == -1) howMany = m_servers->length();
    for (int attempts = 0; attempts < m_servers->length() && howMany > 0;
         attempts++) {
      if (m_server >= m_servers->length()) {
        m_server = 0;
      }
      ServerLocation location(
          Utils::convertHostToCanonicalForm(m_servers[m_server++]->asChar())
              .c_str());
      if (exclEndPts.find(location) != exclEndPts.end()) {
        // exclude this one
        continue;
      }
      outEndpoints.push_back(location);
      howMany--;
    }
  } else {
    throw IllegalStateException(
        "No locators or servers available to select from");
  }
  return outEndpoints;
}

GfErrType ThinClientRedundancyManager::maintainRedundancyLevel(
    bool init, const TcrMessage* request, TcrMessageReply* reply,
    ThinClientRegion* region) {
  // Preconditions:
  // 1. m_redundantEndpoints UNION m_nonredundantEndpionts = All Endpoints
  // 2. m_redundantEndpoints INTERSECTION m_nonredundantEndpoints = Empty
  GfErrType err = GF_NOTCON;
  // save any fatal errors that occur during maintain redundancy so
  // that we can send it back to the caller, to avoid missing out due
  // to nonfatal errors such as server not available
  GfErrType fatalError = GF_NOERR;
  bool fatal = false;
  ACE_Guard<ACE_Recursive_Thread_Mutex> guard(m_redundantEndpointsLock);
  bool isRedundancySatisfied = false;
  int secondaryCount = 0;
  bool isPrimaryConnected = false;
  bool isPrimaryAtBack = false;
  // TODO: isPrimaryAtBack can be removed by simplifying
  // removeEndpointsInOrder().

  std::vector<TcrEndpoint*>::iterator itRedundant =
      m_redundantEndpoints.begin();
  std::vector<TcrEndpoint*> tempRedundantEndpoints;
  std::vector<TcrEndpoint*> tempNonredundantEndpoints;

  // Redundancy level is maintained as follows using the following vectors:
  // m_redundantEndpoints, m_nonredundantEndpoints, tempRedundantEndpoints,
  // tempNonredundantEndpoints.
  // m_redundantEndpoints and m_nonredundantEndpoints contain the current
  // primary-secondary and non-redundant endpoints respectively.
  // tempRedundantEndpoints and tempNonredundantEndpoints are vectors that are
  // used to adjust the status of endpoints:
  // tempRedundantEndpoints: contains endpoints which have been changed from
  // nonredundant to either primary or secondary, i.e.
  // those endpoints whose status has been changed in order to satisfy
  // redundancy level.
  // tempNonredundantEndpoints: contains endpoints which were earlier redundant
  // but are now not connected. These endpoints will
  // be moved from the redundant list to the non redundant list.

  // Step 1: Scan all the endpoints in one pass and arrange them according to
  // connection status in the following order:
  // m_redundantEndpoints,tempRedundantEndpoints,m_nonredundantEndpoints,tempNonredundantEndpoints
  // The endpoints are maintained in order in the redundant endpoints lists.
  // This order is maintained when the
  // endpoints in the temporary list are moved into
  // m_redundantEndpoints/m_nonredundantEndpoints.
  // Note that although endpoint status may change, the endpoints are not
  // deleted from the lists (instead, the endpoints are
  // copied to temporary lists).
  // Scanning the endpoints is done in two stages:
  // 1. First scan the redundant endpoints to find the failed endpoints list,
  // whether redundancy has been satisfied.
  // 2. If redundancy has not been satisfied, scan the nonredundant list to find
  // available endpoints that can be made redundant.

  LOGDEBUG(
      "ThinClientRedundancyManager::maintainRedundancyLevel(): checking "
      "redundant list, size = %d",
      m_redundantEndpoints.size());
  while (!isRedundancySatisfied && itRedundant != m_redundantEndpoints.end()) {
    if (!isPrimaryConnected) {
      if (itRedundant == m_redundantEndpoints.begin()) {
        if ((*itRedundant)->connected()) {
          isPrimaryConnected = true;
          if (m_redundancyLevel == 0) isRedundancySatisfied = true;
        } else {
          tempNonredundantEndpoints.push_back(*itRedundant);
        }
      } else {
        if (sendMakePrimaryMesg(*itRedundant, request, region)) {
          isPrimaryConnected = true;
        } else {
          tempNonredundantEndpoints.push_back(*itRedundant);
        }
      }
    } else {
      if ((*itRedundant)->connected()) {
        secondaryCount++;
        if (secondaryCount == m_redundancyLevel) {
          isRedundancySatisfied = true;
        }
      } else {
        tempNonredundantEndpoints.push_back(*itRedundant);
      }
    }
    ++itRedundant;
  }

  // If redundancy is not satisfied, find nonredundant endpoints that can be
  // made redundant.
  // For queue locators, fetch an initial list of endpoints which can host
  // queues.
  if (!isRedundancySatisfied && m_poolHADM && !init) {
    LOGDEBUG(
        "ThinClientRedundancyManager::maintainRedundancyLevel(): building "
        "nonredundant list via pool.");

    std::list<ServerLocation> outEndpoints;
    std::set<ServerLocation> exclEndPts;
    for (std::vector<TcrEndpoint*>::iterator itr = m_redundantEndpoints.begin();
         itr != m_redundantEndpoints.end(); itr++) {
      LOGDEBUG(
          "ThinClientRedundancyManager::maintainRedundancyLevel(): excluding "
          "endpoint %s from queue list.",
          (*itr)->name().c_str());
      ServerLocation serverLoc((*itr)->name());
      exclEndPts.insert(serverLoc);
    }

    m_nonredundantEndpoints.clear();
    int howMany = -1;
    if (m_locators != NULLPTR && m_locators->length() > 0 &&
        m_servers != NULLPTR && m_servers->length() == 0) {
      // if we are using locators only request the required number of servers.
      howMany = m_redundancyLevel - static_cast<int>(exclEndPts.size()) + 1;
    }
    outEndpoints = selectServers(howMany, exclEndPts);
    for (std::list<ServerLocation>::iterator it = outEndpoints.begin();
         it != outEndpoints.end(); it++) {
      TcrEndpoint* ep = m_poolHADM->addEP(*it);
      LOGDEBUG(
          "ThinClientRedundancyManager::maintainRedundancyLevel(): Adding "
          "endpoint %s to nonredundant list.",
          ep->name().c_str());
      m_nonredundantEndpoints.push_back(ep);
    }
  }

  LOGDEBUG(
      "ThinClientRedundancyManager::maintainRedundancyLevel(): finding "
      "nonredundant endpoints, size = %d",
      m_nonredundantEndpoints.size());
  std::vector<TcrEndpoint*>::iterator itNonredundant =
      m_nonredundantEndpoints.begin();
  while (!isRedundancySatisfied &&
         itNonredundant != m_nonredundantEndpoints.end()) {
    if (!isPrimaryConnected) {
      if (secondaryCount == m_redundancyLevel) {
        // 38196:Make last endpoint from the non redundant list as primary.
        if ((!init || *itNonredundant == m_nonredundantEndpoints.back()) &&
            (err = makePrimary(*itNonredundant, request, reply)) == GF_NOERR) {
          tempRedundantEndpoints.push_back(*itNonredundant);
          isRedundancySatisfied = true;
          isPrimaryConnected = true;
          isPrimaryAtBack = true;
        } else {
          if (ThinClientBaseDM::isFatalError(err)) {
            fatal = true;
            fatalError = err;
          }
        }
      } else {
        if ((err = makeSecondary(*itNonredundant, request, reply)) ==
            GF_NOERR) {
          tempRedundantEndpoints.push_back(*itNonredundant);
          secondaryCount++;
        } else {
          if (ThinClientBaseDM::isFatalError(err)) {
            fatal = true;
            fatalError = err;
          }
        }
      }
    } else {
      if ((err = makeSecondary(*itNonredundant, request, reply)) == GF_NOERR) {
        tempRedundantEndpoints.push_back(*itNonredundant);
        secondaryCount++;
        if (secondaryCount == m_redundancyLevel) {
          isRedundancySatisfied = true;
        }
      } else {
        if (ThinClientBaseDM::isFatalError(err)) {
          fatal = true;
          fatalError = err;
        }
      }
    }
    ++itNonredundant;
  }
  // Step 2: After one scan of the endpoints, if the redundancy level is
  // satisifed by changing status of endpoints, the primary
  // endpoint will be present in either m_redundantEndpoints or
  // tempRedundantEndpoints.
  // However, when redundancy level is not satisifed and following condition
  // holds true, a new secondary server (whose status
  // changed in Step 1) will be made primary:
  // A. No primary server was found, and
  // B. secondaryCount <= redundancy level.

  // 38196: Prefer Primary conversion from oldHA .
  TcrEndpoint* convertedPrimary = NULL;
  if (init && !isRedundancySatisfied && !isPrimaryConnected) {
    bool oldHAEndPointPresent = false;
    for (std::vector<TcrEndpoint*>::iterator it =
             tempRedundantEndpoints.begin();
         it != tempRedundantEndpoints.end(); it++) {
      if ((*it)->getServerQueueStatus() != NON_REDUNDANT_SERVER) {
        oldHAEndPointPresent = true;
        break;
      }
    }
    // TODO: Post-38196fix, simplify durable client initialization by removing
    // constraint on primary position.

    //  holds the endpoints that are skipped by the oldHAEndPointPresent
    // check in the loop back
    std::vector<TcrEndpoint*> tempSkippedEndpoints;
    // warning: do not use unsigned type for index since .size() can return 0
    while (!tempRedundantEndpoints.empty()) {
      TcrEndpoint* ep = tempRedundantEndpoints.back();
      if (oldHAEndPointPresent &&
          ep->getServerQueueStatus() == NON_REDUNDANT_SERVER) {
        tempSkippedEndpoints.push_back(ep);
        tempRedundantEndpoints.pop_back();
        continue;
      }
      if (sendMakePrimaryMesg(ep, request, region)) {
        // Primary may be in middle If there are older nonredundant
        // ep in tempRedundantEndpoints
        isPrimaryAtBack = false;
        convertedPrimary = ep;
        isPrimaryConnected = true;
        break;
      } else {
        tempRedundantEndpoints.pop_back();
      }
    }
    //  push back the skipped endpoints into tempRedundantEndpoints
    while (!tempSkippedEndpoints.empty()) {
      TcrEndpoint* ep = tempSkippedEndpoints.back();
      tempSkippedEndpoints.pop_back();
      tempRedundantEndpoints.push_back(ep);
    }
  }
  if (!isRedundancySatisfied && !isPrimaryConnected) {
    // warning: do not use unsigned type for index since .size() can return 0
    while (!tempRedundantEndpoints.empty()) {
      TcrEndpoint* ep = tempRedundantEndpoints.back();
      if (sendMakePrimaryMesg(ep, request, region)) {
        isPrimaryAtBack = true;
        isPrimaryConnected = true;
        break;
      } else {
        tempRedundantEndpoints.pop_back();
      }
    }
  }

  // Step 3: Finally, create the new redundant and nonredundant lists. Copy from
  // m_redundantEndpointsList all the endpoints that were
  // marked as disconnected. Add in order all the new redundant endpoints (whose
  // status changed in Step 2) to m_redundantEndpoints.
  // If primary was at end of temporary list, move it to front on redundant
  // list.
  // Similarly, adjust the nonredundant list.

  removeEndpointsInOrder(m_redundantEndpoints, tempNonredundantEndpoints);
  removeEndpointsInOrder(m_nonredundantEndpoints, tempRedundantEndpoints);

  // 38196:for DurableReconnect case, primary may be in between, put it @ start.
  if (init && !isPrimaryAtBack && convertedPrimary != NULL) {
    moveEndpointToLast(tempRedundantEndpoints, convertedPrimary);
    isPrimaryAtBack = true;
  }

  addEndpointsInOrder(m_redundantEndpoints, tempRedundantEndpoints);

  if (isPrimaryConnected && isPrimaryAtBack) {
    TcrEndpoint* primary = m_redundantEndpoints.back();
    m_redundantEndpoints.pop_back();
    m_redundantEndpoints.insert(m_redundantEndpoints.begin(), primary);
  }

  // Unregister DM for the new non-redundant endpoints
  for (std::vector<TcrEndpoint*>::const_iterator iter =
           tempNonredundantEndpoints.begin();
       iter != tempNonredundantEndpoints.end(); ++iter) {
    (*iter)->unregisterDM(true);
  }
  addEndpointsInOrder(m_nonredundantEndpoints, tempNonredundantEndpoints);

  // Postconditions:
  // 1. If redundancy level is satisifed, m_redundantEndpoints.size = r + 1,
  // m_redundantEndpoints[0] is primary.
  // 2. If redundancy level is not satisifed, m_redundantEndpoints.size <= r.
  // 3. If primary is connected, m_redundantEndpoints[0] is primary. ( Not
  // checked. To verify, We may have to modify
  //    TcrEndpoint class.)

  GF_DEV_ASSERT(!isRedundancySatisfied || ((m_redundantEndpoints.size() ==
                                            (size_t)(m_redundancyLevel + 1)) &&
                                           isPrimaryConnected));
  GF_DEV_ASSERT(isRedundancySatisfied ||
                ((int)m_redundantEndpoints.size() <= m_redundancyLevel) ||
                m_redundancyLevel == -1);

  RemoteQueryServicePtr queryServicePtr;
  ThinClientPoolDM* poolDM = dynamic_cast<ThinClientPoolDM*>(m_poolHADM);
  if (poolDM) {
    queryServicePtr =
        dynCast<RemoteQueryServicePtr>(poolDM->getQueryServiceWithoutCheck());
  }
  if (queryServicePtr != NULLPTR) {
    if (isPrimaryConnected) {
      // call CqStatusListener connect
      LOGDEBUG(
          "invoke invokeCqConnectedListeners for connected for CQ status "
          "listener");
      queryServicePtr->invokeCqConnectedListeners(poolDM, true);
    } else {
      // call CqStatusListener disconnect
      LOGDEBUG(
          "invoke invokeCqDisConnectedListeners for disconnected for CQ status "
          "listener");
      queryServicePtr->invokeCqConnectedListeners(poolDM, false);
    }
  }

#if GF_DEVEL_ASSERTS == 1
  if (isPrimaryConnected && !m_redundantEndpoints[0]->connected()) {
    LOGWARN(
        "GF_DEV_ASSERT Warning: Failed condition ( isPrimaryConnected ==> "
        "m_redundantEndpoints[ 0 ]->connected( ) )");
  }
#endif

  // Invariants:
  // 1. m_redundantEndpoints UNION m_nonredundantEndpionts = All Endpoints
  // 2. m_redundantEndpoints INTERSECTION m_nonredundantEndpoints = Empty

  // The global endpoint list does not change ever for HA so getAllEndpoints
  // result or redundantEndpoints/nonredundantEndpoints cannot have stale or
  // deleted endpoints
  if (!m_poolHADM) {
    GF_DEV_ASSERT(m_redundantEndpoints.size() +
                      m_nonredundantEndpoints.size() ==
                  (unsigned)m_theTcrConnManager->getNumEndPoints());
  }

  // Update pool stats
  if (m_poolHADM) {
    m_poolHADM->getStats().setSubsServers(
        static_cast<int32_t>(m_redundantEndpoints.size()));
  }

  if (isRedundancySatisfied) {
    m_IsAllEpDisCon = false;
    m_loggedRedundancyWarning = false;
    return GF_NOERR;
  } else if (isPrimaryConnected) {
    if (fatal && err != GF_NOERR) {
      return fatalError;
    }
    m_IsAllEpDisCon = false;
    if (m_redundancyLevel == -1) {
      LOGINFO("Current subscription redundancy level is %d",
              m_redundantEndpoints.size() - 1);
      return GF_NOERR;
    }
    if (!m_loggedRedundancyWarning) {
      LOGWARN(
          "Requested subscription redundancy level %d is not satisfiable with "
          "%d "
          "servers available",
          m_redundancyLevel, m_redundantEndpoints.size());
      m_loggedRedundancyWarning = true;
    }
    return GF_NOERR;
  } else {
    // save any fatal errors that occur during maintain redundancy so
    // that we can send it back to the caller, to avoid missing out due
    // to nonfatal errors such as server not available
    if (m_poolHADM && !m_IsAllEpDisCon) {
      m_poolHADM->sendNotConMesToAllregions();
      m_IsAllEpDisCon = true;
    }
    if (fatal && err != GF_NOERR) {
      return fatalError;
    }
    return err;
  }
}

void ThinClientRedundancyManager::removeEndpointsInOrder(
    std::vector<TcrEndpoint*>& destVector,
    const std::vector<TcrEndpoint*>& srcVector) {
#if GF_DEVEL_ASSERTS == 1
  size_t destInitSize = destVector.size();
#endif

  std::vector<TcrEndpoint*> tempDestVector;
  std::vector<TcrEndpoint*>::iterator itDest;
  std::vector<TcrEndpoint*>::const_iterator itSrc;

  itSrc = srcVector.begin();
  while ((itDest = destVector.begin()) != destVector.end()) {
    if ((itSrc != srcVector.end()) && (*itDest == *itSrc)) {
      destVector.erase(itDest);
      ++itSrc;
    } else {
      tempDestVector.push_back(*itDest);
      destVector.erase(itDest);
    }
  }
  destVector = tempDestVector;

  // Postconditions:
  // 1. size of destVector decreases by the size of srcVector

  GF_DEV_ASSERT(destInitSize == destVector.size() + srcVector.size());
}

void ThinClientRedundancyManager::addEndpointsInOrder(
    std::vector<TcrEndpoint*>& destVector,
    const std::vector<TcrEndpoint*>& srcVector) {
#if GF_DEVEL_ASSERTS == 1
  size_t destInitSize = destVector.size();
#endif

  destVector.insert(destVector.end(), srcVector.begin(), srcVector.end());

  // Postconditions:
  // 1. Length of destVector increases by the length of srcVector

  GF_DEV_ASSERT(destVector.size() == destInitSize + srcVector.size());
}

GfErrType ThinClientRedundancyManager::createQueueEP(TcrEndpoint* ep,
                                                     const TcrMessage* request,
                                                     TcrMessageReply* reply,
                                                     bool isPrimary) {
  LOGFINE("Recovering subscriptions on endpoint [%s]", ep->name().c_str());
  GfErrType err = GF_NOERR;
  if ((err = ep->registerDM(true, !isPrimary)) == GF_NOERR) {
    if ((err = m_theTcrConnManager->registerInterestAllRegions(
             ep, request, reply)) != GF_NOERR ||
        !readyForEvents(ep)) {
      ep->unregisterDM(true);
      if (err == GF_NOERR) {
        err = GF_NOTCON;
      }
    } else {
      // recover CQs
      CacheImpl* cache = m_theTcrConnManager->getCacheImpl();
      RemoteQueryServicePtr rqsService =
          dynCast<RemoteQueryServicePtr>(cache->getQueryService(true));
      if (rqsService != NULLPTR) {
        try {
          err = rqsService->executeAllCqs(ep);
        } catch (const Exception& excp) {
          LOGFINE("Failed to recover CQs on endpoint[%s]: %s",
                  ep->name().c_str(), excp.getMessage());
          ep->unregisterDM(true);
          err = GF_NOTCON;
        } catch (...) {
          LOGFINE("Failed to recover CQs on endpoint[%s]", ep->name().c_str());
          ep->unregisterDM(true);
          err = GF_NOTCON;
        }
      }
    }
  }
  LOGFINE("Done subscription recovery");
  return err;
}

GfErrType ThinClientRedundancyManager::createPoolQueueEP(
    TcrEndpoint* ep, const TcrMessage* request, TcrMessageReply* reply,
    bool isPrimary) {
  LOGFINE("Recovering subscriptions on endpoint [%s] from pool %s",
          ep->name().c_str(), m_poolHADM->getName());

  GfErrType err = GF_NOERR;
  if ((err = ep->registerDM(false, !isPrimary, false, m_poolHADM)) ==
      GF_NOERR) {
    if ((err = m_poolHADM->registerInterestAllRegions(ep, request, reply)) !=
            GF_NOERR ||
        !readyForEvents(ep)) {
      ep->unregisterDM(false);
      if (err == GF_NOERR) {
        err = GF_NOTCON;
      }
    } else {
      // recover CQs
      RemoteQueryServicePtr rqsService = dynCast<RemoteQueryServicePtr>(
          m_poolHADM->getQueryServiceWithoutCheck());
      if (rqsService != NULLPTR) {
        try {
          err = rqsService->executeAllCqs(ep);
        } catch (const Exception& excp) {
          LOGFINE("Failed to recover CQs on endpoint[%s]: %s",
                  ep->name().c_str(), excp.getMessage());
          ep->unregisterDM(false);  // Argument is useless
          err = GF_NOTCON;
        } catch (...) {
          LOGFINE("Failed to recover CQs on endpoint[%s]", ep->name().c_str());
          ep->unregisterDM(false);  // Argument is useless
          err = GF_NOTCON;
        }
      }
    }
  }
  LOGFINE("Done subscription recovery");
  return err;
}

GfErrType ThinClientRedundancyManager::makePrimary(TcrEndpoint* ep,
                                                   const TcrMessage* request,
                                                   TcrMessageReply* reply) {
  if (m_poolHADM) {
    return createPoolQueueEP(ep, request, reply, true);
  } else {
    return createQueueEP(ep, request, reply, true);
  }
}

GfErrType ThinClientRedundancyManager::makeSecondary(TcrEndpoint* ep,
                                                     const TcrMessage* request,
                                                     TcrMessageReply* reply) {
  if (m_poolHADM) {
    return createPoolQueueEP(ep, request, reply, false);
  } else {
    return createQueueEP(ep, request, reply, false);
  }
}

void ThinClientRedundancyManager::initialize(int redundancyLevel) {
  LOGDEBUG(
      "ThinClientRedundancyManager::initialize(): initializing redundancy "
      "manager.");
  LOGFINE("Subscription redundancy level set to %d %s %s", redundancyLevel,
          m_poolHADM != NULL ? "for pool" : "",
          m_poolHADM != NULL ? m_poolHADM->getName() : "");
  m_redundancyLevel = redundancyLevel;
  m_HAenabled = (redundancyLevel > 0 || m_theTcrConnManager->isDurable() ||
                 ThinClientBaseDM::isDeltaEnabledOnServer());
  SystemProperties* sysProp = DistributedSystem::getSystemProperties();
  if (m_poolHADM) {
    m_eventidmap.init(m_poolHADM->getSubscriptionMessageTrackingTimeout());
  } else {
    m_eventidmap.init(sysProp->notifyDupCheckLife());
  }
  int millis = 100;
  if (m_HAenabled) {
    if (m_poolHADM) {
      //  Set periodic ack interval in seconds.
      millis = m_poolHADM->getSubscriptionAckInterval();

    } else {
      millis = sysProp->notifyAckInterval();
    }
    if (millis < 100) millis = 100;
    {
      time_t secs = millis / 1000;
      suseconds_t usecs = (millis % 1000) * 1000;
      ACE_Time_Value duration(secs, usecs);
      m_nextAckInc = duration;
    }

    m_nextAck = ACE_OS::gettimeofday();
    m_nextAck += m_nextAckInc;
  }

  if (m_poolHADM) {
    m_locators = m_poolHADM->getLocators();
    if (m_locators->length() == 0) m_servers = m_poolHADM->getServers();
    if (m_locators->length() > 0) {
      std::vector<std::string> locators;
      for (int item = 0; item < m_locators->length(); item++) {
        LOGDEBUG("ThinClientRedundancyManager::initialize: adding locator %s",
                 m_locators[item]->asChar());
        locators.push_back(m_locators[item]->asChar());
      }

    } else if (m_servers->length() > 0) {
      RandGen randgen;
      m_server = randgen(m_servers->length());
    } else {
      throw IllegalStateException(
          "The redundancy manager's pool does not have locators or servers "
          "specified");
    }
  }
  getAllEndpoints(m_nonredundantEndpoints);
}

void ThinClientRedundancyManager::sendNotificationCloseMsgs() {
  ACE_Guard<ACE_Recursive_Thread_Mutex> guard(m_redundantEndpointsLock);

  for (std::vector<TcrEndpoint*>::iterator iter = m_redundantEndpoints.begin();
       iter != m_redundantEndpoints.end(); ++iter) {
    LOGDEBUG(
        "ThinClientRedundancyManager::sendNotificationCloseMsgs(): closing "
        "notification for endpoint %s",
        (*iter)->name().c_str());
    (*iter)->stopNoBlock();
  }

  for (std::vector<TcrEndpoint*>::iterator iter = m_redundantEndpoints.begin();
       iter != m_redundantEndpoints.end(); ++iter) {
    LOGDEBUG(
        "ThinClientRedundancyManager::sendNotificationCloseMsgs(): closing "
        "receiver for endpoint %s",
        (*iter)->name().c_str());
    (*iter)->stopNotifyReceiverAndCleanup();
  }
}

void ThinClientRedundancyManager::close() {
  LOGDEBUG("ThinClientRedundancyManager::close(): closing redundancy manager.");

  if (m_periodicAckTask) {
    if (m_processEventIdMapTaskId >= 0) {
      CacheImpl::expiryTaskManager->cancelTask(m_processEventIdMapTaskId);
    }
    m_periodicAckTask->stopNoblock();
    m_periodicAckSema.release();
    m_periodicAckTask->wait();
    GF_SAFE_DELETE(m_periodicAckTask);
  }

  ACE_Guard<ACE_Recursive_Thread_Mutex> guard(m_redundantEndpointsLock);

  for (std::vector<TcrEndpoint*>::iterator iter = m_redundantEndpoints.begin();
       iter != m_redundantEndpoints.end(); ++iter) {
    LOGDEBUG(
        "ThinClientRedundancyManager::close(): unregistering from endpoint %s",
        (*iter)->name().c_str());
    (*iter)->unregisterDM(true);
  }

  m_redundantEndpoints.clear();
  m_nonredundantEndpoints.clear();
  m_theTcrConnManager = NULL;
  m_globalProcessedMarker = false;
  m_sentReadyForEvents = false;
}

bool ThinClientRedundancyManager::readyForEvents(
    TcrEndpoint* primaryCandidate) {
  if (!m_theTcrConnManager->isDurable() || !m_sentReadyForEvents) {
    return true;
  }

  TcrMessageClientReady request;
  TcrMessageReply reply(true, NULL);

  GfErrType err = GF_NOTCON;
  if (m_poolHADM) {
    err = m_poolHADM->sendRequestToEP(request, reply, primaryCandidate);
  } else {
    err = ThinClientBaseDM::sendRequestToEndPoint(request, reply,
                                                  primaryCandidate);
  }
  if (err == GF_NOERR) {
    return true;
  } else {
    return false;
  }
}

void ThinClientRedundancyManager::moveEndpointToLast(
    std::vector<TcrEndpoint*>& epVector, TcrEndpoint* targetEp) {
  // Pre-condition
  GF_DEV_ASSERT(epVector.size() > 0 && targetEp != NULL);

  // Remove Ep
  for (std::vector<TcrEndpoint*>::iterator it = epVector.begin();
       it != epVector.end(); it++) {
    if (targetEp == *it) {
      epVector.erase(it);
      break;
    }
  }
  // Push it @ end.
  epVector.push_back(targetEp);
}

bool ThinClientRedundancyManager::sendMakePrimaryMesg(
    TcrEndpoint* ep, const TcrMessage* request, ThinClientRegion* region) {
  if (!ep->connected()) {
    return false;
  }
  TcrMessageReply reply(false, NULL);
  const TcrMessageMakePrimary makePrimaryRequest(
      ThinClientRedundancyManager::m_sentReadyForEvents);

  LOGFINE("Making primary subscription endpoint %s", ep->name().c_str());
  GfErrType err = GF_NOTCON;
  if (m_poolHADM) {
    err = m_poolHADM->sendRequestToEP(makePrimaryRequest, reply, ep);
  } else {
    err =
        ThinClientBaseDM::sendRequestToEndPoint(makePrimaryRequest, reply, ep);
  }
  if (err == GF_NOERR) {
    /*  this causes keys to be added to the region even if the reg interest
     * is supposed to fail due to notauthorized exception then causing
     * subsequent maintainredundancy calls to fail for other ops like CQ
    if ( request != NULL && region != NULL ) {
      const VectorOfCacheableKey* keys = request->getKeys( );
      bool isDurable = request->isDurable( );
      if ( keys == NULL || keys->empty( ) ) {
        const std::string& regex = request->getRegex( );
        if ( !regex.empty( ) ) {
          region->addRegex( regex, isDurable );
        }
      } else {
        region->addKeys( *keys, isDurable );
      }
    }
    */
    return true;
  } else {
    return false;
  }
}

GfErrType ThinClientRedundancyManager::sendSyncRequestCq(
    TcrMessage& request, TcrMessageReply& reply, ThinClientBaseDM* theHADM) {
  LOGDEBUG("ThinClientRedundancyManager::sendSyncRequestCq msgType[%d]",
           request.getMessageType());
  ACE_Guard<ACE_Recursive_Thread_Mutex> guard(m_redundantEndpointsLock);

  GfErrType err = GF_NOERR;
  GfErrType opErr;
  TcrEndpoint* primaryEndpoint = NULL;

  if (m_redundantEndpoints.size() >= 1) {
    LOGDEBUG(
        "ThinClientRedundancyManager::sendSyncRequestCq: to secondary size[%d]",
        m_redundantEndpoints.size());
    std::vector<TcrEndpoint*>::iterator iter = m_redundantEndpoints.begin();
    LOGDEBUG("endpoint[%s]", (*iter)->name().c_str());
    for (++iter; iter != m_redundantEndpoints.end(); ++iter) {
      LOGDEBUG("endpoint[%s]", (*iter)->name().c_str());
      LOGDEBUG(
          "msgType[%d] ThinClientRedundancyManager::sendSyncRequestCq: to "
          "secondary",
          request.getMessageType());
      if (request.getMessageType() == TcrMessage::EXECUTECQ_WITH_IR_MSG_TYPE) {
        // Timeout for this message type is set like so...
        reply.setTimeout(
            dynamic_cast<ThinClientPoolDM*>(theHADM)->getReadTimeout() / 1000);
        opErr = theHADM->sendRequestToEP(request, reply, *iter);
      } else {
        opErr = theHADM->sendRequestToEP(request, reply, *iter);
      }
      if (err == GF_NOERR) {
        err = opErr;
      }
    }
    primaryEndpoint = m_redundantEndpoints[0];
    LOGDEBUG("primary endpoint[%s]", primaryEndpoint->name().c_str());
  }

  int32_t attempts = static_cast<int32_t>(m_redundantEndpoints.size()) +
                     static_cast<int32_t>(m_nonredundantEndpoints.size());
  // TODO: FIXME: avoid magic number 5 for retry attempts
  attempts = attempts < 5
                 ? 5
                 : attempts;  // at least 5 attempts if ep lists are small.

  ProxyCachePtr proxyCache = NULLPTR;

  while (attempts--) {
    if (err != GF_NOERR || m_redundantEndpoints.empty()) {
      UserAttributesPtr userAttr =
          TSSUserAttributesWrapper::s_geodeTSSUserAttributes
              ->getUserAttributes();
      if (userAttr != NULLPTR) proxyCache = userAttr->getProxyCache();
      err = maintainRedundancyLevel();
      // we continue on fatal error because MRL only tries a handshake without
      // sending a request (no params passed) so no need to check
      // isFatalClientError.
      if (theHADM->isFatalError(err) && m_redundantEndpoints.empty()) {
        continue;
      }
    }

    if (m_redundantEndpoints.empty()) {
      err = GF_NOTCON;
    } else {
      primaryEndpoint = m_redundantEndpoints[0];
      LOGDEBUG(
          "ThinClientRedundancyManager::sendSyncRequestCq: to primary [%s]",
          primaryEndpoint->name().c_str());
      GuardUserAttribures gua;
      if (proxyCache != NULLPTR) {
        gua.setProxyCache(proxyCache);
      }
      err = theHADM->sendRequestToEP(request, reply, primaryEndpoint);
      if (err == GF_NOERR || err == GF_TIMOUT ||
          ThinClientBaseDM::isFatalClientError(err)) {
        break;
      }
    }
  }

  // top level should only get NotConnectedException
  if (err == GF_IOERR) {
    err = GF_NOTCON;
  }

  return err;
}

GfErrType ThinClientRedundancyManager::sendSyncRequestRegisterInterest(
    TcrMessage& request, TcrMessageReply& reply, bool attemptFailover,
    TcrEndpoint* endpoint, ThinClientBaseDM* theHADM,
    ThinClientRegion* region) {
  LOGDEBUG("ThinClientRedundancyManager::sendSyncRequestRegisterInterest ");
  if (endpoint == NULL) {
    ACE_Guard<ACE_Recursive_Thread_Mutex> guard(m_redundantEndpointsLock);

    GfErrType err = GF_NOERR;
    GfErrType opErr = GF_NOERR;
    TcrEndpoint* primaryEndpoint = NULL;

    if (m_redundantEndpoints.size() >= 1) {
      std::vector<TcrEndpoint*>::iterator iter = m_redundantEndpoints.begin();
      for (++iter; iter != m_redundantEndpoints.end(); ++iter) {
        (*iter)->setDM(request.getDM());
        opErr = theHADM->sendSyncRequestRegisterInterestEP(request, reply,
                                                           false, *iter);
        if (err == GF_NOERR) {
          err = opErr;
        }
      }
      primaryEndpoint = m_redundantEndpoints[0];
    }

    if ((request.getMessageType() == TcrMessage::REGISTER_INTEREST_LIST ||
         request.getMessageType() == TcrMessage::REGISTER_INTEREST ||
         request.getMessageType() == TcrMessage::UNREGISTER_INTEREST_LIST ||
         request.getMessageType() == TcrMessage::UNREGISTER_INTEREST) &&
        (err != GF_NOERR ||
         static_cast<int>(m_redundantEndpoints.size()) <= m_redundancyLevel)) {
      err = maintainRedundancyLevel(false, &request, &reply, region);
      if (theHADM->isFatalClientError(err)) {
        return err;
      }
    }

    // We need to ensure that primary registration is done at the end.
    // So we skip the primary registration when maintainRedundancyLevel()
    // has already done that, i.e. primary endpoint not the same as that
    // before calling maintainRedundancyLevel()

    if (m_redundantEndpoints.empty()) {
      err = GF_NOTCON;
    } else if (primaryEndpoint == m_redundantEndpoints[0]) {
      for (size_t count = 0;
           count < m_redundantEndpoints.size() + m_nonredundantEndpoints.size();
           count++) {
        primaryEndpoint->setDM(request.getDM());
        opErr = theHADM->sendSyncRequestRegisterInterestEP(
            request, reply, false, primaryEndpoint);
        if (opErr == GF_NOERR) {
          break;
        } else {
          err = maintainRedundancyLevel(false, &request, &reply, region);
          primaryEndpoint = m_redundantEndpoints[0];
        }
      }
      if (theHADM->isFatalError(opErr)) {
        err = opErr;
      }
    }
    return err;
  } else {
    return theHADM->sendSyncRequestRegisterInterestEP(request, reply, false,
                                                      endpoint);
  }
}

void ThinClientRedundancyManager::getAllEndpoints(
    std::vector<TcrEndpoint*>& endpoints) {
  // 38196 Fix: For durable clients reconnect
  // 1. Get list of endpoints which have HA queue.
  // 2. Get HA endpoint with max queuesize;
  // 3. Move other HA endpoints at start of list in ascending order of
  // queuesize.
  // 4. Add HA endpoint with Max queueSize at the end.
  // Primary : Secondaries in ascending order of QSiz : Nonredundant : Secondary
  // with QSizemax
  // e.g. R[#70]:Old Primary,  R [#80] , R[#90], NR.... , R[#100]:New Primary
  // Exception: For R =0 ( or when no EP with Max queuesize ),
  //  Old primary would be considered as new. Hence it would be at the end

  ACE_Map_Manager<std::string, TcrEndpoint*, ACE_Recursive_Thread_Mutex>*
      tempContainer;
  if (m_poolHADM) {
    tempContainer = &m_poolHADM->m_endpoints;
    // fetch queue servers
    // send queue servers for sorting
    std::set<ServerLocation> exclEndPts;
    std::list<ServerLocation> outEndpoints;

    outEndpoints = selectServers(-1, exclEndPts);
    for (std::list<ServerLocation>::iterator it = outEndpoints.begin();
         it != outEndpoints.end(); it++) {
      m_poolHADM->addEP(*it);
    }
  } else {
    tempContainer = &m_theTcrConnManager->m_endpoints;
  }

  TcrEndpoint* maxQEp = NULL;
  TcrEndpoint* primaryEp = NULL;

  for (ACE_Map_Manager<std::string, TcrEndpoint*,
                       ACE_Recursive_Thread_Mutex>::iterator currItr =
           (*tempContainer).begin();
       currItr != (*tempContainer).end(); currItr++) {
    if (isDurable()) {
      TcrEndpoint* ep = (*currItr).int_id_;
      int32_t queueSize = 0;
      TcrConnection* statusConn = NULL;
      ServerQueueStatus status =
          ep->getFreshServerQueueStatus(queueSize, !m_poolHADM, statusConn);
      if (m_poolHADM && status != NON_REDUNDANT_SERVER) {
        m_poolHADM->addConnection(statusConn);
      }
      if (status == REDUNDANT_SERVER) {
        if (maxQEp == NULL) {
          maxQEp = ep;
        } else if (ep->getServerQueueSize() > maxQEp->getServerQueueSize()) {
          insertEPInQueueSizeOrder(maxQEp, endpoints);
          maxQEp = ep;
        } else {
          insertEPInQueueSizeOrder(ep, endpoints);
        }
        LOGDEBUG(
            "ThinClientRedundancyManager::getAllEndpoints(): sorting "
            "endpoints, found redundant endpoint.");
      } else if (status == PRIMARY_SERVER) {
        // Primary should be unique
        GF_DEV_ASSERT(primaryEp == NULL);
        primaryEp = ep;
        LOGDEBUG(
            "ThinClientRedundancyManager::getAllEndpoints(): sorting "
            "endpoints, found primary endpoint.");
      } else {
        endpoints.push_back((*currItr).int_id_);
        LOGDEBUG(
            "ThinClientRedundancyManager::getAllEndpoints(): sorting "
            "endpoints, found nonredundant endpoint.");
      }
    } else {
      endpoints.push_back((*currItr).int_id_);
    }
    //(*currItr)++;
  }

  // Add Endpoint with Max Queuesize at the last and Primary at first position
  if (isDurable()) {
    if (maxQEp != NULL) {
      endpoints.push_back(maxQEp);
      LOGDEBUG(
          "ThinClientRedundancyManager::getAllEndpoints(): sorting endpoints, "
          "pushing max-q endpoint at back.");
    }
    if (primaryEp != NULL) {
      if (m_redundancyLevel == 0 || maxQEp == NULL) {
        endpoints.push_back(primaryEp);
        LOGDEBUG(
            "ThinClientRedundancyManager::getAllEndpoints(): sorting "
            "endpoints, pushing primary at back.");
      } else {
        endpoints.insert(endpoints.begin(), primaryEp);
        LOGDEBUG(
            "ThinClientRedundancyManager::getAllEndpoints(): sorting "
            "endpoints, inserting primary at head.");
      }
    }
  } else {
    RandGen randgen;
    std::random_shuffle(endpoints.begin(), endpoints.end(), randgen);
  }
}

void ThinClientRedundancyManager::insertEPInQueueSizeOrder(
    TcrEndpoint* inputEp, std::vector<TcrEndpoint*>& endpoints) {
  // need to sort out redundant ep in ascending order. other eps will be at
  // back.
  std::vector<TcrEndpoint*>::iterator it = endpoints.begin();
  while (it != endpoints.end()) {
    TcrEndpoint* thisEp = *it;
    if ((thisEp->getServerQueueSize() >= inputEp->getServerQueueSize()) ||
        (thisEp->getServerQueueStatus() != REDUNDANT_SERVER)) {
      break;
    }
    it++;
  }
  endpoints.insert(it, inputEp);
}

bool ThinClientRedundancyManager::isDurable() {
  return m_theTcrConnManager->isDurable();
}

void ThinClientRedundancyManager::readyForEvents() {
  TcrMessageClientReady request;
  TcrMessageReply reply(true, NULL);
  GfErrType result = GF_NOTCON;
  unsigned int epCount = 0;

  ACE_Guard<ACE_Recursive_Thread_Mutex> guardEPs(m_redundantEndpointsLock);

  if (m_sentReadyForEvents) {
    throw IllegalStateException("Already called readyForEvents");
  }

  TcrEndpoint* primary = NULL;
  if (m_redundantEndpoints.size() > 0) {
    primary = m_redundantEndpoints[0];
    if (m_poolHADM) {
      result = m_poolHADM->sendRequestToEP(request, reply, primary);
    } else {
      result = primary->send(request, reply);
    }
  }

  epCount++;

  while (result != GF_NOERR &&
         epCount <=
             m_redundantEndpoints.size() + m_nonredundantEndpoints.size()) {
    TcrMessageReply reply(true, NULL);
    maintainRedundancyLevel();
    if (m_redundantEndpoints.size() > 0) {
      primary = m_redundantEndpoints[0];
      if (m_poolHADM) {
        result = m_poolHADM->sendRequestToEP(request, reply, primary);
      } else {
        result = primary->send(request, reply);
      }
    }
    epCount++;
  }

  if (result != GF_NOERR) {
    throw NotConnectedException("No endpoints available");
  }

  m_sentReadyForEvents = true;
}

int ThinClientRedundancyManager::processEventIdMap(const ACE_Time_Value&,
                                                   const void*) {
  m_periodicAckSema.release();
  return 0;
}

int ThinClientRedundancyManager::periodicAck(volatile bool& isRunning) {
  while (isRunning) {
    m_periodicAckSema.acquire();
    if (isRunning) {
      doPeriodicAck();
      while (m_periodicAckSema.tryacquire() != -1) {
        ;
      }
    }
  }
  return 0;
}

void ThinClientRedundancyManager::doPeriodicAck() {
  LOGDEBUG(
      "ThinClientRedundancyManager::processEventIdMap( ): Examining eventid "
      "map.");
  // do periodic ack if HA is enabled and the time has come
  if (m_HAenabled && (m_nextAck < ACE_OS::gettimeofday())) {
    LOGFINER("Doing periodic ack");
    m_nextAck += m_nextAckInc;

    EventIdMapEntryList entries = m_eventidmap.getUnAcked();
    uint32_t count = static_cast<uint32_t>(entries.size());

    if (count > 0) {
      bool acked = false;

      ACE_Guard<ACE_Recursive_Thread_Mutex> guardEPs(m_redundantEndpointsLock);

      std::vector<TcrEndpoint*>::iterator endpoint =
          m_redundantEndpoints.begin();

      if (endpoint != m_redundantEndpoints.end()) {
        TcrMessagePeriodicAck request(entries);
        TcrMessageReply reply(true, NULL);

        GfErrType result = GF_NOERR;
        if (m_poolHADM) {
          result = m_poolHADM->sendRequestToEP(request, reply, *endpoint);
        } else {
          result = (*endpoint)->send(request, reply);
        };

        if (result == GF_NOERR && reply.getMessageType() == TcrMessage::REPLY) {
          LOGFINE("Sent subscription ack message for %d sources to endpoint %s",
                  count, (*endpoint)->name().c_str());
          acked = true;
        } else {
          LOGWARN(
              "Failure sending subscription ack message for %d sources to "
              "endpoint %s",
              count, (*endpoint)->name().c_str());
          LOGFINER("Ack result is %d and reply message type is %d", result,
                   reply.getMessageType());
        }
      } else {
        LOGWARN(
            "No subscription servers available for periodic ack for %d sources",
            count);
      }

      if (!acked) {
        // clear entries' acked flag for next periodic ack
        uint32_t cleared = m_eventidmap.clearAckedFlags(entries);
        GF_D_ASSERT(cleared <= count);
        cleared = 0;  // this line to avoid a 'unused var' warning in gcc
      }
    }
  }

  // check the event id map for expiry
  uint32_t expired = m_eventidmap.expire(m_HAenabled /* onlyacked */);

  if (expired > 0) {
    LOGFINE("Expired %d sources from subscription map", expired);
  }
}

void ThinClientRedundancyManager::startPeriodicAck() {
  m_periodicAckTask = new Task<ThinClientRedundancyManager>(
      this, &ThinClientRedundancyManager::periodicAck, NC_PerodicACK);
  m_periodicAckTask->start();
  SystemProperties* props = DistributedSystem::getSystemProperties();
  // start the periodic ACK task handler
  ACE_Event_Handler* periodicAckTask =
      new ExpiryHandler_T<ThinClientRedundancyManager>(
          this, &ThinClientRedundancyManager::processEventIdMap);
  // m_processEventIdMapTaskId = CacheImpl::expiryTaskManager->
  // scheduleExpiryTask(periodicAckTask, 1, 1, false);
  m_processEventIdMapTaskId = CacheImpl::expiryTaskManager->scheduleExpiryTask(
      periodicAckTask, m_nextAckInc, m_nextAckInc, false);
  LOGFINE(
      "Registered subscription event "
      "periodic ack task with id = %ld, notify-ack-interval = %ld, "
      "notify-dupcheck-life = %ld, periodic ack is %sabled",
      m_processEventIdMapTaskId,
      m_poolHADM ? m_poolHADM->getSubscriptionAckInterval()
                 : props->notifyAckInterval(),
      m_poolHADM ? m_poolHADM->getSubscriptionMessageTrackingTimeout()
                 : props->notifyDupCheckLife(),
      m_HAenabled ? "en" : "dis");
}

// notification dup check with the help of eventidmap - called by
// ThinClientRegion
bool ThinClientRedundancyManager::checkDupAndAdd(EventIdPtr eventid) {
  EventIdMapEntry entry = EventIdMap::make(eventid);
  return m_eventidmap.put(entry.first, entry.second, true);
}

void ThinClientRedundancyManager::netDown() {
  ACE_Guard<ACE_Recursive_Thread_Mutex> guard(m_redundantEndpointsLock);

  if (!m_poolHADM) {
    m_nonredundantEndpoints.insert(m_nonredundantEndpoints.end(),
                                   m_redundantEndpoints.begin(),
                                   m_redundantEndpoints.end());
  }
  m_redundantEndpoints.clear();
}

void ThinClientRedundancyManager::removeCallbackConnection(TcrEndpoint* ep) {
  ACE_Guard<ACE_Recursive_Thread_Mutex> guard(m_redundantEndpointsLock);

  ep->unregisterDM(false, NULL, true);
}
GfErrType ThinClientRedundancyManager::sendRequestToPrimary(
    TcrMessage& request, TcrMessageReply& reply) {
  ACE_Guard<ACE_Recursive_Thread_Mutex> guard(m_redundantEndpointsLock);
  GfErrType err = GF_NOTCON;
  for (size_t count = 0;
       count <= m_redundantEndpoints.size() + m_nonredundantEndpoints.size();
       count++) {
    if (m_poolHADM) {
      err =
          m_poolHADM->sendRequestToEP(request, reply, m_redundantEndpoints[0]);
    } else {
      err = m_redundantEndpoints[0]->send(request, reply);
    }
    if (err == GF_NOERR) break;
    maintainRedundancyLevel();
  }
  return err;
}


text:
#include "stdafx.h"

#define DEBUG_FT_THESAURUS 0

#include <algorithm>                    /* for equal_range() */
#include <cstring>                      /* for strcmp(3) */
#include <limits>

#include <zorba/internal/cxx_util.h>

#include "context/static_context.h"
#include "diagnostics/assert.h"
#include "diagnostics/dict.h"
#include "diagnostics/xquery_diagnostics.h"
#include "system/globalenv.h"
#include "util/fs_util.h"
#include "util/less.h"
#include "util/locale.h"
#if DEBUG_FT_THESAURUS
# include "util/oseparator.h"
#endif
#include "util/utf8_util.h"
#include "system/globalenv.h"

#include "decode_base128.h"
#include "wn_db_segment.h"
#include "wn_synset.h"
#include "wn_thesaurus.h"
#include "wn_types.h"

using namespace std;
using namespace zorba::locale;

namespace zorba {
namespace wordnet {

/**
 * Used to ensure the endianness of the WordNet data file matches that of the
 * CPU we're running on.
 */
uint32_t const Magic_Number = 42;       // same as TIFF -- why not?

////////// Helper functions ///////////////////////////////////////////////////

/**
 * Appends the name of the file Zorba uses for a WordNet thesaurus files.
 *
 * @param path The path to append to.
 * @param lang The language of the thesaurus file.
 */
static void append_wordnet_file( zstring &path, iso639_1::type lang ) {
  fs::append( path, "wordnet-" );
  path += iso639_1::str( lang );
  path += ".zth";
}

/**
 * "Fixes" the "at most" parameter.  The Full Text specification section 3.4.3
 * says in part:
 *
 *    FTThesaurusID specifies the relationship sought between tokens and
 *    phrases written in the query and terms in the thesaurus and the number of
 *    levels to be queried in hierarchical relationships by including an
 *    FTRange "levels". If no levels are specified, the default is to query all
 *    levels in hierarchical relationships or to query an implementation-
 *    defined number of levels in hierarchical relationships.
 *
 * The problem with defaulting to "all levels" is that it makes queries too
 * broad, hence if at_most specifies "all levels" (max int), clamp it at 2
 * (which seems to work well in practice).
 */
inline internal::Thesaurus::level_type
fix_at_most( internal::Thesaurus::level_type at_most ) {
  return at_most == numeric_limits<internal::Thesaurus::level_type>::max() ?
    2 : at_most;
}

/**
 * Checks whether a WordNet synset pointer should be followed.
 *
 * @param ptr_type The pointer's type.
 * @return Returns \c true only if the pointer should be followed.
 */
static bool follow_ptr( pointer::type ptr_type ) {
  using namespace pointer;
  switch ( ptr_type ) {
    case antonym:
    case attribute:
    case cause:
    case derivationally_related_form:
    case derived_from_adjective:
    case domain_of_synset_region:
    case domain_of_synset_topic:
    case domain_of_synset_usage:
    case entailment:
    case member_holonym:
    case member_meronym:
    case member_of_domain_region:
    case member_of_domain_topic:
    case member_of_domain_usage:
    case part_holonym:
    case part_meronym:
    case participle_of_verb:
    case pertainym:
    case substance_holonym:
    case substance_meronym:
      //
      // These pointer types are skipped because it's thought that nobody would
      // want/expect such words as synonyms.
      //
      return false;

    case also_see:
    case hypernym:
    case hyponym:
    case instance_hypernym:
    case instance_hyponym:
    case similar_to:
    case verb_group:
      //
      // These pointer types are thought to be what most would want/expect
      // synonyms to be.
      //
      return true;

    case unknown:
      ZORBA_ASSERT( false );
  }
  return false;                         // suppesses warning -- never gets here
}

/**
 * Gets the "direction" of a WordNet pointer, i.e., whether a pointer leads to
 * a more specific, neutral, or more general lemma.
 *
 * @param ptr_type The WordNet pointer type to get the direction of.
 * @return Returns -1, 0, or +1 if \a ptr_type leads to a more specific,
 * neutral, or more general lemma, respectively.
 */
static iso2788::rel_dir get_ptr_dir( pointer::type ptr_type ) {
  using namespace pointer;
  switch ( ptr_type ) {
    case attribute:
    case derivationally_related_form:
    case domain_of_synset_region:
    case domain_of_synset_topic:
    case domain_of_synset_usage:
    case entailment:
    case hypernym:
    case instance_hypernym:
    case member_holonym:
    case part_holonym:
    case substance_holonym:
      return iso2788::broader;

    case derived_from_adjective:
    case hyponym:
    case instance_hyponym:
    case member_meronym:
    case member_of_domain_region:
    case member_of_domain_topic:
    case member_of_domain_usage:
    case part_meronym:
    case substance_meronym:
      return iso2788::narrower;

    case also_see:
    case antonym:
    case cause:
    case participle_of_verb:
    case pertainym:
    case similar_to:
    case unknown:
    case verb_group:
      return iso2788::neutral;

    default:                            // ensures all cases are handled
      ZORBA_ASSERT( false );
  }
  return iso2788::neutral;              // suppesses warning -- never gets here
}

/**
 * Gets the full path to the WordNet binary file.
 *
 * @param path The candidate path to the WordNet file.
 * @return Returns said path.
 */
static zstring get_wordnet_path( zstring path ) {
  ZORBA_ASSERT( !path.empty() );

  for ( bool loop = true; loop; ) {
    switch ( fs::get_type( path ) ) {
      case fs::directory:
        append_wordnet_file( path, iso639_1::en );
        break;
      case fs::file:
        loop = false;
        break;
      case fs::non_existent:
        throw ZORBA_EXCEPTION(
          zerr::ZOSE0001_FILE_NOT_FOUND, ERROR_PARAMS( path )
        );
      default:
        throw ZORBA_EXCEPTION(
          zerr::ZOSE0002_NOT_PLAIN_FILE, ERROR_PARAMS( path )
        );
    }
  }
  return path;
}

/**
 * Attempts to map an XQuery thesaurus relationship to a WordNet pointer type.
 *
 * @param relationship The XQuery thesaurus relationship.
 * @param lang The language of the relationship.
 * @return Returns the corresponding WordNet pointer type.
 */
static pointer::type map_xquery_rel( zstring const &relationship,
                                     iso639_1::type lang ) {
  zstring rel_lower;
  utf8::to_lower( relationship, &rel_lower );
  if ( iso2788::rel_type iso_rel = iso2788::find_rel( rel_lower, lang ) )
    return pointer::map_iso_rel( iso_rel );
  return pointer::find( rel_lower );
}

///////////////////////////////////////////////////////////////////////////////

thesaurus::iterator::candidate_queue_t::value_type const
  thesaurus::iterator::LevelMarker = make_pair( ~0u, iso2788::neutral );

thesaurus::iterator::iterator( thesaurus const &t, char const *p,
                               pointer::type ptr_type, level_type at_least,
                               level_type at_most ) :
  thesaurus_( t ), query_ptr_type_( ptr_type ),
  at_least_( at_least ), at_most_( fix_at_most( at_most ) ), level_( 0 )
{
  while ( *p++ ) ;                      // skip past lemma
  //
  // Load the candidate_queue_ will all the synsets for the lemma.
  //
  iso2788::rel_dir const query_dir = get_ptr_dir( query_ptr_type_ );
  for ( unsigned num_synsets = decode_base128( &p ); num_synsets-- > 0; ) {
    synset_id_t const synset_id = decode_base128( &p );
    candidate_queue_.push_back( make_pair( synset_id, query_dir ) );
  }
  //
  // All the candidates just added constitute a "level" so add the LevelMarker
  // to the queue.
  //
  candidate_queue_.push_back( LevelMarker );
}

thesaurus::iterator::~iterator() {
  // out-of-line since it's virtual
}

void thesaurus::iterator::destroy() const {
  delete this;
}

bool thesaurus::iterator::next( zstring *synonym ) {
  while ( result_queue_.empty() ) {
#   if DEBUG_FT_THESAURUS
    cout << "--------------------------------------------------" << endl;
    cout << "result_queue is empty" << endl;
#   endif

    if ( candidate_queue_.empty() ) {
#     if DEBUG_FT_THESAURUS
      cout << "candidate_queue is empty --> no more synonyms" << endl;
#     endif
      return false;
    }

    candidate_t const candidate( ztd::pop_front( candidate_queue_ ) );
    synset_id_t const synset_id = candidate.first;
    iso2788::rel_dir const synset_ptr_dir = candidate.second;

    if ( synset_id == LevelMarker.first ) {
#     if DEBUG_FT_THESAURUS
      cout << "+ found LevelMarker" << endl;
#     endif
      if ( ++level_ > at_most_ ) {
#       if DEBUG_FT_THESAURUS
        cout << "+ level (" << level_ << ") > at_most (" << at_most_
             << ") --> no more synonyms" << endl;
#       endif
        return false;
      }

      //
      // We've just incremented the level, so all synsets that have been added
      // to the queue since the last time we were here constitute a "level",
      // therefore add the level marker so we know when to increment the level
      // next time.
      //
      // Note that we do this only if the queue isn't empty, otherwise the
      // queue would never become empty.
      //
      if ( !candidate_queue_.empty() )
        candidate_queue_.push_back( LevelMarker );

      if ( query_ptr_type_ == pointer::antonym ) {
        //
        // Antonyms are a special case.  When the initial batch of synset
        // pointers are followed, we obviously follow only those whose type is
        // antonym.
        //
        // After that, however, we can no longer follow antonym pointers
        // otherwise we'd get antonyms of the antonyms (i.e., synonyms of the
        // original word).  Instead, we need synonyms of the antonyms.  Hence,
        // after the initial batch of antonym synset pointers are followed, we
        // reset query_ptr_type_ to unknown.
        //
        // For example, given the word "poor", we initially follow an antonym
        // pointer to the word "rich".  After that, we don't want antonyms of
        // rich, but synonyms, e.g., "affluent", "wealthy", "well to do", etc.
        //
#       if DEBUG_FT_THESAURUS
        cout << "$ resetting query_ptr_type_ to unknown" << endl;
#       endif
        query_ptr_type_ = pointer::unknown;
      }
      continue;
    }

    synset const ss( thesaurus_.wn_synsets_[ synset_id ] );

    if ( level_ >= at_least_ ) {
#     if DEBUG_FT_THESAURUS
      cout << "+ level (" << level_ << ") >= at_least (" << at_least_ << ')'
           << endl;
#     endif
      if ( !query_ptr_type_ || level_ ) {
        //
        // The first batch of synsets in the queue are from the inital look-up
        // of the query phrase.  All the lemmas in all these synsets are
        // synonyms of the query phrase (they all have the same "sense").
        //
        // If there was no relationship specified in the query (query_ptr_type_
        // is "unknown"), then it's OK to return those lemmas as synonyms.
        //
        // However, if a relationship was specified, then it's not OK to return
        // those lemmas since the user isn't looking for synonyms, but is
        // instead looking for lemmas that have the given relationship.
        //
        // But once we're past the first (zeroth) level (the synsets of the
        // inital look-up), it's always OK to return the lemmas.
        //
        FOR_EACH( synset::lemma_id_list, lemma_id, ss.lemma_ids() ) {
#         if DEBUG_FT_THESAURUS
          cout << "? " << wn_lemmas_[ *lemma_id ] << " -> ";
#         endif
          if ( synonyms_seen_.insert( *lemma_id ).second ) {
            result_queue_.push_back( *lemma_id );
#           if DEBUG_FT_THESAURUS
            cout << "pushed";
#           endif
          }
#         if DEBUG_FT_THESAURUS
          else {
            cout << "skipped";
          }
          cout << endl;
#         endif
        }
#       if DEBUG_FT_THESAURUS
        cout << "  result_queue is now: ";
        oseparator comma;
        FOR_EACH( result_queue_t, lemma_id, result_queue_ ) {
          cout << comma << wn_lemmas_[ *lemma_id ];
        }
        cout << endl;
#       endif
      }
    }

    //
    // Examine, and possibly follow, this synset's pointers.
    //
    FOR_EACH( synset::ptr_list, ptr, ss.ptrs() ) {
      if ( query_ptr_type_ ) {
        //
        // A pointer type (relationship) was given for the thesaurus option: if
        // this ptr's type doesn't match, skip it.
        //
        if ( ptr->type_ != query_ptr_type_ )
          continue;
      } else {
        if ( !follow_ptr( ptr->type_ ) )
          continue;
      }

      //
      // Ensure the "direction" of the pointer that got us to this synset and
      // that of the current pointer are "congruous", i.e., they do not point
      // in opposite directions.  Once we start to follow a pointer that has a
      // direction, we want to keep going in the same direction (or at least
      // laterally) in the semantic tree.
      //
      // For example, given the word "poor" (as in "poor people"), if we
      // followed a hypernym pointer to the broader word "people", we would
      // then not want to follow a hyponym pointers to more specific words like
      // "blind" ("blind people"), "brave" ("brave people"), "clientele",
      // "rich" ("rich people"), i.e., all the different kinds of people, since
      // none of those are synonyms of "poor".
      //
      iso2788::rel_dir const current_ptr_dir = get_ptr_dir( ptr->type_ );
      if ( !iso2788::congruous( synset_ptr_dir, current_ptr_dir ) )
        continue;

#     if DEBUG_FT_THESAURUS
      cout << "+ pushing \"" << ptr->type_
           << "\" synset (ID=" << ptr->synset_id_ << ')' << endl;
      synset const ptr_ss( wn_synsets_[ ptr->synset_id_ ] );
      oseparator comma;
      cout << "  lemmas: ";
      FOR_EACH( synset::lemma_id_list, lemma_id, ptr_ss.lemma_ids() ) {
        cout << comma << wn_lemmas_[ *lemma_id ];
      }
      cout << endl;
#     endif

      candidate_queue_.push_back(
        make_pair( ptr->synset_id_, synset_ptr_dir + current_ptr_dir )
      );

#define DO_PTR_SOURCE 0
#if DO_PTR_SOURCE
      if ( ptr->source_ ) {
        lemma_id_t const source_lemma_id = ss.lemma_ids()[ ptr->source_ - 1 ];

        synset const tt( wn_synsets_[ ptr->synset_id_ ] );
        lemma_id_t const target_lemma_id = tt.lemma_ids()[ ptr->target_ - 1 ];
      }
#endif
    } // FOR_EACH
  } // while

  *synonym = thesaurus_.wn_lemmas_[ ztd::pop_front( result_queue_ ) ];
# if DEBUG_FT_THESAURUS
  cout << "--> synonym=" << *synonym << endl;
# endif
  return true;
}

///////////////////////////////////////////////////////////////////////////////

thesaurus::wordnet_file_checker::wordnet_file_checker( mmap_file const &file ) {
  typedef char version_type[4];
  static char const our_version[] = "ZW01";

  // check version
  char file_version[ sizeof( version_type ) + 1 ];
  char const *byte_ptr = file.begin();
  ::strncpy( file_version, byte_ptr, sizeof( version_type ) );
  file_version[ sizeof( version_type ) ] = '\0';
  if ( ::strcmp( file_version, our_version ) != 0 )
    throw ZORBA_EXCEPTION(
      zerr::ZXQP8401_THESAURUS_VERSION_MISMATCH,
      ERROR_PARAMS( file_version, our_version )
    );

  // check endian-ness
  byte_ptr += sizeof( uint32_t );
  uint32_t const file_endian = *reinterpret_cast<uint32_t const*>( byte_ptr );
  if ( file_endian != Magic_Number )
    throw ZORBA_EXCEPTION( zerr::ZXQP8402_THESAURUS_ENDIANNESS_MISMATCH );
}

///////////////////////////////////////////////////////////////////////////////

thesaurus::thesaurus( zstring const &path, iso639_1::type lang ) :
  wordnet_file_( get_wordnet_path( path ).c_str() ),
  wordnet_file_checker_( wordnet_file_ ),
  wn_lemmas_( wordnet_file_, db_segment::lemma ),
  wn_synsets_( wordnet_file_, db_segment::synset ),
  lang_( lang )
{
}

thesaurus::~thesaurus() {
  // do nothing
}

void thesaurus::destroy() const {
  delete this;
}

char const* thesaurus::find_lemma( zstring const &phrase ) const {
  typedef pair<db_segment::const_iterator,db_segment::const_iterator> range_t;

  char const *const c_phrase = phrase.c_str();
  less<char const*> const comparator;

  range_t const range =
    ::equal_range( wn_lemmas_.begin(), wn_lemmas_.end(), c_phrase, comparator );
  if ( range.first == wn_lemmas_.end() || comparator( c_phrase, *range.first ) )
    return nullptr;
  return *range.first;
}

thesaurus::iterator::ptr
thesaurus::lookup( zstring const &phrase, zstring const &relationship,
                   level_type at_least, level_type at_most ) const {
  iterator::ptr result;
# if DEBUG_FT_THESAURUS
  cout << "==================================================" << endl;
  cout << "query phrase: " << phrase << endl;
# endif
  if ( char const *const p = find_lemma( phrase ) ) {
    result.reset(
      new iterator(
        *this, p, map_xquery_rel( relationship, lang_ ), at_least, at_most
      )
    );
  }
  return std::move( result );
}

///////////////////////////////////////////////////////////////////////////////

provider::provider( zstring const &path ) : path_( path ) {
  ZORBA_ASSERT( !path.empty() );
}

bool provider::getThesaurus( iso639_1::type lang,
                             internal::Thesaurus::ptr *t ) const {
#ifdef ZORBA_WITH_FILE_ACCESS
  zstring file_path;

  switch ( lang ) {
    case iso639_1::unknown:
      lang = iso639_1::en;
      // no break;
    case iso639_1::en:
      file_path = path_;
      append_wordnet_file( file_path, lang );
      break;
    default:
      return false;
  }

  //
  // We want to look for the WordNet thesaurus file on the library path.
  // Unfortunately every static_context can have its own library path and we
  // don't have direct access to the query's static_context here.  So, for now
  // we only look on the root static_context's library path.
  //
  static_context &sctx = GENV.getRootStaticContext();
  std::vector<zstring> lib_path_components;
  sctx.get_full_lib_path( lib_path_components );
  MUTATE_EACH( std::vector<zstring>, path, lib_path_components ) {
    fs::append( *path, file_path );
    if ( fs::get_type( *path ) == fs::file ) {
      if ( t )
        t->reset( new thesaurus( *path, lang ) );
      return true;
    }
  }
  return false;
#else
  switch ( lang ) {
    case iso639_1::unknown:
      lang = iso639_1::en;
      // no break;
    case iso639_1::en:
      if ( t )
        t->reset();
      return true;
    default:
      return false;
  }
#endif /* ZORBA_WITH_FILE_ACCESS */
}

///////////////////////////////////////////////////////////////////////////////

} // namespace wordnet
} // namespace zorba
/* vim:set et sw=2 ts=2: */

redpajama_set_name: RedPajamaGithub


text:
'''
Copyright (c) 2016, Emmanuel Levijarvi
All rights reserved.
License BSD 2-Clause
'''
import logging
import datetime
from iotrelay import Reading
from meter_reader import Gateway, GatewayError

logger = logging.getLogger(__name__)
__version__ = "1.0.2"

POLL_FREQUENCY = 9
SUMM_POLL_FREQUENCY = 600

class Poll(object):
    def __init__(self, config):
        self.config = config
        self.series_key = config['series key']
        self.gateway = Gateway(config['address'])
        self.gateway.timeout = float(config['timeout'])
        # poll_frequency: seconds between readings
        poll_frequency = int(config.get('poll frequency', POLL_FREQUENCY))
        summ_poll_frequency = int(config.get('summation poll frequency',
                                             SUMM_POLL_FREQUENCY))
        self.summ_delta = datetime.timedelta(seconds=summ_poll_frequency)
        self.delta = datetime.timedelta(seconds=poll_frequency)
        self.next_reading_time = datetime.datetime.now()
        self.summ_next_reading_time = datetime.datetime.now()
        self.last_timestamp = None
        self.last_summation = None

    def get_readings(self):
        summation = self.get_summation()
        power = self.get_power()
        if summation is None:
            return power
        summation.append(power)
        return summation

    def get_power(self):
        if datetime.datetime.now() < self.next_reading_time:
            return
        try:
            timestamp, demand = self.gateway.get_instantaneous_demand()
            logger.debug(timestamp, demand)
        except GatewayError as e:
            logger.error(e)
            self.next_reading_time = datetime.datetime.now() + self.delta / 2
            return
        if self.last_timestamp is None or timestamp != self.last_timestamp:
            logger.info("{0!s}, {1!s}".format(timestamp, demand))
            self.last_timestamp = timestamp
            self.next_reading_time = datetime.datetime.now() + self.delta
            return Reading('power', demand, timestamp, self.series_key)
        else:
            logger.warning('duplicate reading')
            self.next_reading_time = datetime.datetime.now() + self.delta / 2

    def get_summation(self):
        if datetime.datetime.now() < self.summ_next_reading_time:
            return
        try:
            summations = self.gateway.run_command(name='get_summation_values')
        except GatewayError as e:
            logger.error(e)
        self.sum_next_reading_time = datetime.datetime.now() + self.summ_delta
        return [Reading('power', s['Value'], s['TimeStamp'], 'summation')
                for s in summations]

    def __str__(self):
        return 'eagle gateway. next reading {}.'.format(self.next_reading_time)


text:
techanModule('plot/ichimoku', function(specBuilder) {
  'use strict';

  var techan = require('../../../../src/techan'),
      data = require('./../_fixtures/data/ichimoku').plot,
      plot = require('../../../../src/plot/plot')(d3.svg.line),
      plotMixin = require('../../../../src/plot/plotmixin')(d3.scale.linear, d3.functor, techan.scale.financetime),
      domFixtures = require('../_fixtures/dom');

  var actualInit = function(module) {
    return module(d3.svg.area, techan.accessor.ichimoku, plot, plotMixin);
  };

  specBuilder.require(require('../../../../src/plot/ichimoku'), function(instanceBuilder) {
    instanceBuilder.instance('actual', actualInit, function(scope) {
      describe('And ichimoku is initialised with defaults', function () {
        plotMixinShouldBeSetup(scope);
        plotShouldRenderWithoutError(scope, data, domFixtures);
      });
    });
  });
});


text:
package org.apache.jena.rdfxml.xmlinput.states;

import java.util.ArrayList;

import org.apache.jena.rdfxml.xmlinput.impl.* ;
import org.xml.sax.Attributes;
import org.xml.sax.SAXParseException;

public class WantPropertyElement extends Frame implements WantsObjectFrameI,
        HasSubjectFrameI {
    int liCounter = 1;

    ANode predicate;

    ANode object;

    ANode reify;

    boolean objectIsBlank = false;

    public WantPropertyElement(HasSubjectFrameI s, AbsXMLContext x) {
        super(s, x);
    }

    // These three are used as bitfields
    static final private int TYPEDLITERAL = 1;

    static final private int EMPTYWITHOBJ = 2;

    static final private int PARSETYPE = 4;

    @Override
    public FrameI startElement(String uri, String localName, String rawName,
            Attributes atts) throws SAXParseException {
        clearObject();
        if (nonWhiteMsgGiven)
            taint.isTainted();
        nonWhiteMsgGiven = false;
        if (uri==null || uri.equals("")) {
            warning(WARN_UNQUALIFIED_ELEMENT,"Unqualified property elements are not allowed. Treated as a relative URI.");
        }
        ElementLexer el = new ElementLexer(taint, this, uri, localName,
                rawName, E_LI, CoreAndOldTerms | E_DESCRIPTION, false);
        // if (el.badMatch)
        // warning(ERR_SYNTAX_ERROR,"bad use of " + rawName);
        predicate = el.goodMatch ? (AResourceInternal) rdf_n(liCounter++)
                : URIReference.fromQName(this, uri, localName);
        if (taint.isTainted())
            predicate.taint();
        taint = new TaintImpl();

        AttributeLexer ap = new AttributeLexer(this,
        // xml:
                A_XMLLANG | A_XMLBASE | A_XML_OTHER
                // legal rdf:
                        | A_DATATYPE | A_ID | A_NODEID | A_PARSETYPE
                        | A_RESOURCE | A_TYPE,
                // bad rdf:
                A_BADATTRS);
        int cnt = ap.processSpecials(taint, atts);

        // These three states are intended as mutually
        // incompatible, but all three can occur
        // together. Any two of the three, or all
        // three is a syntax errror.
        // Having none of these is legal.
        final int nextStateCode = (ap.datatype == null ? 0 : TYPEDLITERAL)
                | (ap.parseType == null ? 0 : PARSETYPE)
                | (mustBeEmpty(ap, atts, cnt) ? EMPTYWITHOBJ : 0);

        if (this.badStateCode(nextStateCode)) {
            warning(errorNumber(nextStateCode), descriptionOfCases(ap,
                    nextStateCode, propertyAttributeDescription(atts, ap, cnt)));
        }

        AbsXMLContext x = ap.xml(xml);

        reify = ap.id == null ? null : URIReference.fromID(this, x, ap.id);
        if (taint.isTainted())
            predicate.taint();

        if (mustBeEmpty(ap, atts, cnt)) {
            if (ap.nodeID != null) {

                object = new ARPResource(arp, ap.nodeID);
                checkNodeID_XMLName(object, ap.nodeID);
                objectIsBlank = true;
            }
            if (ap.resource != null) {
                if (object != null) {
                    if (!badStateCode(nextStateCode))
                        // otherwise warning already given
                        warning(ERR_SYNTAX_ERROR,
                                "On a property element, only one of the attributes rdf:nodeID or rdf:resource is permitted.");
                } else
                    object = URIReference.resolve(this, x, ap.resource);
            }
            if (object == null) {
                object = new ARPResource(arp);
                objectIsBlank = true;
            }
            if (taint.isTainted())
                object.taint();
            processPropertyAttributes(ap, atts, x);
        }

        FrameI nextFrame = nextFrame(atts, ap, cnt, nextStateCode, x);
        if (object != null) {
            if (taint.isTainted())
                object.taint();
            theObject(object);
        }
        if (taint.isTainted())
            predicate.taint();
        return nextFrame;

    }

    private boolean mustBeEmpty(AttributeLexer ap, Attributes atts, int cnt) {
        return cnt < atts.getLength() || ap.type != null || ap.nodeID != null
                || ap.resource != null;
    }

    private FrameI nextFrame(Attributes atts, AttributeLexer ap, int cnt,
            int nextStateCode, AbsXMLContext x) throws SAXParseException {
        switch (nextStateCode) {
        case 0:
            return new WantLiteralValueOrDescription(this, x);
        case PARSETYPE | TYPEDLITERAL:
        case PARSETYPE | TYPEDLITERAL | EMPTYWITHOBJ:
        case PARSETYPE | EMPTYWITHOBJ:
        case PARSETYPE:
            return withParsetype(ap.parseType, x);
        case TYPEDLITERAL | EMPTYWITHOBJ:
        case TYPEDLITERAL:
            return new WantTypedLiteral(this, ap.datatype, x);
        case EMPTYWITHOBJ:
            return new WantEmpty(this, x);
        }
        throw new IllegalStateException("impossible");
    }

    private FrameI withParsetype(String pt, AbsXMLContext x)
            throws SAXParseException {
        if (pt.equals("Collection")) {
            return new RDFCollection(this, x);
        }
        if (pt.equals("Resource")) {
            if (object == null) {
                // in some error cases the object has already been set.
                object = new ARPResource(arp);
                objectIsBlank = true;
            }
            return new WantPropertyElement(this, x);
        }
        if (!pt.equals("Literal")) {
            warning(WARN_UNKNOWN_PARSETYPE, "Unknown rdf:parseType: '" + pt
                    + "' (treated as 'Literal'.");
        }
        return new OuterXMLLiteral(this, x, pt);
    }

    @Override
    String suggestParsetypeLiteral() {
        return (getParent() instanceof WantTopLevelDescription) ? "" : super
                .suggestParsetypeLiteral();
    }

    @Override
    public void aPredAndObj(ANode p, ANode o) {
        triple(object, p, o);
    }

    @Override
    public void makeSubjectReificationWith(ANode r) {
        triple(r, RDF_SUBJECT, object);
    }

    @Override
    public void theObject(ANode o) {
        HasSubjectFrameI p = (HasSubjectFrameI) getParent();
        p.aPredAndObj(predicate, o);
        if (reify != null) {
            triple(reify, RDF_TYPE, RDF_STATEMENT);
            triple(reify, RDF_OBJECT, o);
            triple(reify, RDF_PREDICATE, predicate);
            p.makeSubjectReificationWith(reify);
        }
    }

    @Override
    public void endElement() {
        clearObject();
    }

    @Override
    public void abort() {
        clearObject();
    }

    private void clearObject() {
        if (objectIsBlank)
            arp.endLocalScope(object);
        objectIsBlank = false;
        object = null;
    }

    // Allocate the first 20 rdf:_i ; calculate at run time any others.
    static private final int RDF_N = 20;
    static private URIReference _rdf_n_uri[] = new URIReference[RDF_N];
    static {
        for ( int i = 0 ; i < RDF_N; i++) {
            _rdf_n_uri[i] = makeRDF_i(i);
        }
    }

    static private URIReference rdf_n(int i) {
        if ( i < 0 )
            throw new IllegalStateException("Negative rdf:_i");
        if (i < _rdf_n_uri.length)
            return _rdf_n_uri[i];
        return makeRDF_i(i);
    }

    static URIReference makeRDF_i(int i) {
        return URIReference.createNoChecks(rdfns + "_" + i);
    }

    // Not thread safe. JENA-2172
//    static private URIReference _rdf_n[] = new URIReference[0];
//
//    static private URIReference rdf_n(int i) {
//        if (i >= _rdf_n.length) {
//            int newLength = (i + 10) * 3 / 2;
//            URIReference new_rdf_n[] = new URIReference[newLength];
//            System.arraycopy(_rdf_n, 0, new_rdf_n, 0, _rdf_n.length);
//            for (int j = _rdf_n.length; j < newLength; j++) {
//                new_rdf_n[j] = URIReference.createNoChecks(rdfns + "_" + j);
//            }
//            _rdf_n = new_rdf_n;
//        }
//        return _rdf_n[i];
//    }

    /***************************************************************************
     *
     * ERROR HANDLING CODE
     *
     **************************************************************************/

    // Error detection
    private boolean badStateCode(int nextStateCode) {
        switch (nextStateCode) {
        case PARSETYPE | TYPEDLITERAL:
        case PARSETYPE | TYPEDLITERAL | EMPTYWITHOBJ:
        case PARSETYPE | EMPTYWITHOBJ:
        case TYPEDLITERAL | EMPTYWITHOBJ:
            return true;
        case 0:
        case PARSETYPE:
        case TYPEDLITERAL:
        case EMPTYWITHOBJ:
            return false;
        }
        throw new IllegalStateException("impossible");
    }

    // Error classification

    private int errorNumber(int nextStateCode) {
        // TODO: not for 2.3. refine this error code.
        return ERR_SYNTAX_ERROR;
    }

    /***************************************************************************
     *
     * ERROR MESSAGES
     *
     **************************************************************************/
   private String descriptionOfCases(AttributeLexer ap, int nextStateCode,
            String propAttrs) {
        return ((propAttrs == null && ap.type == null)
                || (ap.nodeID == null && ap.resource == null && ap.type == null) || (ap.nodeID == null
                && ap.resource == null && propAttrs == null)) ? pairwiseIncompatibleErrorMessage(
                nextStateCode, ap, propAttrs)
                : complicatedErrorMessage(nextStateCode, ap, propAttrs);
    }

    private String pairwiseIncompatibleErrorMessage(int nextStateCode,
            AttributeLexer ap, String propAttrs) {
        ArrayList<String> cases = new ArrayList<>();
        if ((nextStateCode & PARSETYPE) != 0)
            cases.add("rdf:parseType");
        if ((nextStateCode & TYPEDLITERAL) != 0)
            cases.add("rdf:datatype");
        if (ap.nodeID != null)
            cases.add("rdf:nodeID");
        if (ap.resource != null)
            cases.add("rdf:resource");
        if (ap.type != null)
            cases.add("rdf:type");

        if (cases.size() == 1) {
            if (propAttrs == null)
                throw new IllegalStateException("Shouldn't happen.");
            return "The attribute " + cases.get(0) + " is not permitted with "
                    + propAttrs + " on a property element.";
        }
        String rslt = "On a property element, only one of the ";
        if (propAttrs == null)
            rslt += "attributes ";
        for (int i = 0; i < cases.size(); i++) {
            rslt += cases.get(i);
            switch (cases.size() - i) {
            case 1:
                break;
            case 2:
                rslt += " or ";
                break;
            default:
                rslt += ", ";
                break;
            }
        }
        if (propAttrs != null) {
            rslt += " attributes or " + propAttrs;
        }
        rslt += " is permitted.";
        return rslt;
    }

    private String complicatedErrorMessage(int nextStateCode,
            AttributeLexer ap, String propAttrs) {
        String subjectIs;

        if (ap.nodeID == null && ap.resource == null
                && (ap.type == null || propAttrs == null))
            throw new IllegalStateException("precondition failed.");

        switch (nextStateCode & (TYPEDLITERAL | PARSETYPE)) {
        case TYPEDLITERAL | PARSETYPE:
            subjectIs = "the mutually incompatible attributes rdf:datatype and rdf:parseType are";
            break;
        case TYPEDLITERAL:
            subjectIs = "the attribute rdf:datatype is";
            break;
        case PARSETYPE:
            subjectIs = "the attribute rdf:parseType is";
            break;
        default:
            throw new IllegalStateException("precondition failed");
        }

        String nodeIDResource = null;
        if (ap.nodeID != null && ap.resource != null) {
            nodeIDResource = "the mutually incompatible attributes rdf:nodeID and rdf:resource";
        } else if (ap.nodeID != null) {
            nodeIDResource = "the attribute rdf:nodeID";
        } else if (ap.resource != null) {
            nodeIDResource = "the attribute rdf:resource";
        }

        int otherAttCount = nodeIDResource == null ? 0 : 1;
        String otherAtts;
        if (ap.type != null)
            otherAttCount++;
        if (propAttrs != null)
            otherAttCount++;
        if (otherAttCount < 2)
            throw new IllegalStateException("logic error");
        otherAtts = otherAttCount == 2 ? "both " : "each of ";

        if (ap.type != null && propAttrs != null) {
            if (nodeIDResource == null)
                otherAtts += "the attribute rdf:type and the " + propAttrs;
            else
                otherAtts += "the attribute rdf:type, the " + propAttrs;
        } else if (ap.type != null) {
            otherAtts += "the attribute rdf:type";
        } else {
            otherAtts = "the " + propAttrs;
        }

        if (nodeIDResource != null)
            otherAtts += " and "+nodeIDResource;

        return "On a property element, " + subjectIs + " incompatible with "
                + otherAtts +".";
    }

    private String propertyAttributeDescription(Attributes atts,
            AttributeLexer ap, int cnt) {
        String propAttrs = "";
        int propAttrCount = atts.getLength() - cnt;
        int found = 0;
        if (propAttrCount == 0)
            return null;
        switch (propAttrCount) {
        case 0:
            break;
        case 1:
        case 2:
        case 3:
            for (int i = 0; i < atts.getLength(); i++)
                if (!ap.done(i)) {
                    propAttrs += atts.getQName(i);
                    found++;
                    switch (propAttrCount - found) {
                    case 0:
                        break;
                    case 1:
                        propAttrs += " and ";
                        break;
                    default:
                        propAttrs += ", ";
                    }
                }
            break;
        default:
            if (propAttrCount < 0)
                throw new IllegalStateException("Shouldn't happen.");
            for (int i = 0; i < atts.getLength(); i++)
                if (!ap.done(i)) {
                    found++;
                    switch (found) {
                    case 1:
                        propAttrs += atts.getQName(i) + ", ";
                        break;
                    case 2:
                        propAttrs += atts.getQName(i) + ", ...";
                        break;
                    default:
                    // ignore
                    }
                }
        }
        return "property attributes (" + propAttrs + ")";
    }

}


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">

<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<title>Virtual File System Objects</title>

Small. Fast. Reliable.
Choose any three.

    About
    Documentation
    Download
    License
    Support
    Purchase

    <form name=f method="GET" action="https://www.sqlite.org/search">
      <input id=q name=q type=text
       onfocus="entersearch()" onblur="leavesearch()" style="width:24ex;padding:1px 1ex; border:solid white 1px; font-size:0.9em ; font-style:italic;color:#044a64;" value="Search SQLite Docs...">
      <input type=submit value="Go" style="border:solid white 1px;background-color:#044a64;color:white;font-size:0.9em;padding:0 1ex">
    </form>

SQLite C Interface

Virtual File System Objects

sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName);
int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt);
int sqlite3_vfs_unregister(sqlite3_vfs*);

A virtual filesystem (VFS) is an sqlite3_vfs object
that SQLite uses to interact
with the underlying operating system.  Most SQLite builds come with a
single default VFS that is appropriate for the host computer.
New VFSes can be registered and existing VFSes can be unregistered.
The following interfaces are provided.

The sqlite3_vfs_find() interface returns a pointer to a VFS given its name.
Names are case sensitive.
Names are zero-terminated UTF-8 strings.
If there is no match, a NULL pointer is returned.
If zVfsName is NULL then the default VFS is returned.

New VFSes are registered with sqlite3_vfs_register().
Each new VFS becomes the default VFS if the makeDflt flag is set.
The same VFS can be registered multiple times without injury.
To make an existing VFS into the default VFS, register it again
with the makeDflt flag set.  If two different VFSes with the
same name are registered, the behavior is undefined.  If a
VFS is registered with a name that is NULL or an empty string,
then the behavior is undefined.

Unregister a VFS with the sqlite3_vfs_unregister() interface.
If the default VFS is unregistered, another VFS is chosen as
the default.  The choice for the new VFS is arbitrary.

See also lists of
  Objects,
  Constants, and
  Functions.


text:
package com.marklogic.client.row;

import com.marklogic.client.expression.PlanBuilder;

/**
 * An abstraction for a serialization of a plan
 * such as an AST in JSON format, a Query DSL in JavaScript syntax,
 * or a SQL or SPARQL SELECT query.
 */
public interface RawPlan extends PlanBuilder.Plan {
}


text:
from msrest.serialization import Model

class CustomActivityReferenceObject(Model):
    """Reference objects for custom activity.

    :param linked_services: Linked service references.
    :type linked_services:
     list[~azure.mgmt.datafactory.models.LinkedServiceReference]
    :param datasets: Dataset references.
    :type datasets: list[~azure.mgmt.datafactory.models.DatasetReference]
    """

    _attribute_map = {
        'linked_services': {'key': 'linkedServices', 'type': '[LinkedServiceReference]'},
        'datasets': {'key': 'datasets', 'type': '[DatasetReference]'},
    }

    def __init__(self, linked_services=None, datasets=None):
        self.linked_services = linked_services
        self.datasets = datasets


text:
var mongoose = require('mongoose'),
    Category = require('../models/Category'),
    User = require('../models/User');

module.exports.loadForumSchema = function () {
    var forumSchema = new mongoose.Schema({
        categories: [Category],
        users: [User]
    });

    var Forum = mongoose.model('forum', forumSchema);
};


text:
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
              xmlns:tools="http://schemas.android.com/tools"
              android:orientation="vertical"
              android:layout_width="match_parent"
              android:layout_height="match_parent"
              android:background="@color/colorWhite">

    <ImageButton
        android:id="@+id/image_button"
        android:layout_width="match_parent"
        android:layout_height="220dp"
        android:scaleType="fitXY"
        android:minHeight="0dp"
        android:minWidth="0dp"
        android:padding="0dp"/>

    <TextView
        android:id="@+id/title"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_marginTop="15dp"
        android:layout_gravity="center"
        android:textSize="14sp"
        android:textColor="@color/colorBlack"
        tools:text="sgsdgsdg"/>

    <TextView
        android:id="@+id/info"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_marginTop="5dp"
        android:layout_marginBottom="5dp"
        android:layout_gravity="center"
        android:textColor="@color/colorBlack"
        android:textSize="12sp"
        tools:text="sgsgshdfh"/>
</LinearLayout>


text:
"use strict";

const Constants = require("../../../../Constants");
const Events = Constants.Events;
const Endpoints = Constants.Endpoints;
const apiRequest = require("../../../../core/ApiRequest");

module.exports = function(channelId, recipients) {
  return new Promise((rs, rj) => {
    var request = apiRequest
    .post(this, {
      url: Endpoints.CALL_STOP_RINGING(channelId),
      body: recipients ? {recipients} : []
    });

    this._queueManager.put(request, (err, res) => {
      return (!err && res.ok) ? rs() : rj(err);
    });
  });
};


text:
package com.reason.lang.core.psi.impl;

import com.intellij.psi.PsiElement;
import com.intellij.psi.tree.IElementType;
import com.reason.lang.core.CompositeTypePsiElement;
import com.reason.lang.core.ORUtil;
import com.reason.lang.core.psi.PsiConditional;
import com.reason.lang.core.type.ORTypes;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

public class PsiTernary extends CompositeTypePsiElement<ORTypes> implements PsiConditional {

  protected PsiTernary(@NotNull ORTypes types, @NotNull IElementType elementType) {
    super(types, elementType);
  }

  @Override
  public @Nullable PsiBinaryCondition getCondition() {
    return findChildByClass(PsiBinaryCondition.class);
  }

  @Override
  public @Nullable PsiElement getThenExpression() {
    PsiElement element = ORUtil.findImmediateFirstChildOfType(this, m_types.QUESTION_MARK);
    return element == null ? null : ORUtil.nextSibling(element);
  }

  @Override
  public @Nullable PsiElement getElseExpression() {
    PsiElement element = ORUtil.findImmediateFirstChildOfType(this, m_types.COLON);
    return element == null ? null : ORUtil.nextSibling(element);
  }
}


text:
package util

import (
	"runtime"
	"strings"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/hyperledger/fabric/common/metadata"
	"github.com/hyperledger/fabric/core/config"
	"github.com/spf13/viper"
)

//NewDockerClient creates a docker client
func NewDockerClient() (client *docker.Client, err error) {
	endpoint := viper.GetString("vm.endpoint")
	tlsenabled := viper.GetBool("vm.docker.tls.enabled")
	if tlsenabled {
		cert := config.GetPath("vm.docker.tls.cert.file")
		key := config.GetPath("vm.docker.tls.key.file")
		ca := config.GetPath("vm.docker.tls.ca.file")
		client, err = docker.NewTLSClient(endpoint, cert, key, ca)
	} else {
		client, err = docker.NewClient(endpoint)
	}
	return
}

func ParseDockerfileTemplate(template string) string {
	r := strings.NewReplacer(
		"$(ARCH)", runtime.GOARCH,
		"$(PROJECT_VERSION)", metadata.Version,
		"$(BASE_VERSION)", metadata.BaseVersion,
		"$(DOCKER_NS)", metadata.DockerNamespace,
		"$(BASE_DOCKER_NS)", metadata.BaseDockerNamespace)

	return r.Replace(template)
}

func GetDockerfileFromConfig(path string) string {
	return ParseDockerfileTemplate(viper.GetString(path))
}


text:
title: More on compilers: load, and templates
author: Jasper Van der Jeugt
---

Loading items
-------------

The compiler Monad is a complex beast, but this is nicely hidden for the user of
the Hakyll library.

Suppose that you're generating `index.html` which shows your latest brilliant
blogpost. This requires `posts/foo.markdown` to be generated *before*
`index.html` (so we don't have to generate it twice). But you don't have to care
about any of that: Hakyll will sort this out for you automatically!

Let's see some quick examples. We can load a specific item:

```haskell
load "posts/foo.markdown" :: Compiler (Item String)
```

Or a whole bunch of them:

```haskell
loadAll "posts/*" :: Compiler [Item String]
```

Sometimes you just want the *contents* and not the `Item`:

```haskell
loadBody "posts/foo.markdown" :: Compiler String
```

This is all useful if we want to use Hakyll's templating system.

Templates
---------

### Basic templates

Let's have a look at a simple template:

```html
<h1>$title$</h1>
<div class="info">Posted on $date$</div>
$body$
```

As you can probably guess, template files just contain text and only the `$`
character has special meaning: text between dollar signs ("fields") is replaced
when the template is applied. If you want an actual dollar sign in the output,
use `$$`.

You usually compile the templates from disk using the aptly named
`templateCompiler`:

```haskell
match "templates/*" $ compile templateCompiler
```

Notice the lack of `route` here: this is because we don't need to write the
templates to your `_site` folder, we just want to use them elsewhere.

### Templates: Context

We can easily guess the meaning of `$title$`, `$date$`, and `$body$`, but these
are not hard-coded fields: they belong to a certain [Context]. A `Context`
determines how the fields are interpreted. It's a [Monoid] and therefore very
composable.

[Context]: /reference/Hakyll-Web-Template-Context.html
[Monoid]: http://learnyouahaskell.com/functors-applicative-functors-and-monoids

`field` allows us to create a `Context` for a single field:

```haskell
field :: String -> (Item a -> Compiler String) -> Context a
```

Let's try this out. Note that this is for illustration purposes only: you
shouldn't have to write complicated fields often.  We can implement the `$body$`
field like this:

```haskell
field "body" $ \item -> return (itemBody item) :: Context String
```

And `$title$` like this:

```haskell
titleContext :: Context a
titleContext = field "title" $ \item -> do
    metadata <- getMetadata (itemIdentifier item)
    return $ fromMaybe "No title" $ M.lookup "title" metadata
```

And compose them using the `Monoid` instance:

```haskell
context :: Context String
context = mconcat
    [ titleContext
    , field "body" $ return . itemBody
    ]
```

Obviously, it would be tedious to implement things like `titleContext` over and
over again for different websites and different fields. This is why hakyll
provides `defaultContext`. `defaultContext` is a composed `Context` and allows
you to use:

- `$body$` for the body of the page;
- `$url$` for the destination URL of the page;
- `$path$` for the original filepath of the page;
- `$foo$` where foo is specified in the metadata.

`$date$` is not provided by default, you can see how we add it in the definition
of `postCtx` in `site.hs`:

```haskell
postCtx :: Context String
postCtx =
    dateField "date" "%B %e, %Y" `mappend`
    defaultContext
```

### Loading and applying templates

Now we know about templates, context and how to load arbitrary items. This gives
us enough background information in order to understand you can apply a
template:

```haskell
compile $ do
    tpl <- loadBody "templates/post.html"
    pandocCompiler >>= applyTemplate tpl postCtx
```

Loading and then immediately applying a template is so common there's a
shorthand function:

```haskell
compile $
    pandocCompiler >>= loadAndApplyTemplate "templates/post.html" postCtx
```

Control flow in templates
-------------------------

Sometimes string interpolation does not suffice, and you want a little more
control over how your templates are layed out. Hakyll provides a few control
structures for this. The syntax for these structures was based on the syntax
used in pandoc templates, since Hakyll already has tight integration with
pandoc.

### Conditionals

In `templates/post.html` of the example site we generated using `hakyll-init`,
we see an example of a conditional:

```html
<div class="info">
    Posted on $date$
    $if(author)$
        by $author$
    $endif$
</div>
```

This example should be pretty straightforward. One important thing to notice is
that `$if(foo)$` **does not** check the truth value of `$foo$`: it merely checks
if such a key is present.

Note that an if-else form is supported as well:

```html
<div class="info">
    Posted on $date$
    $if(author)$
        by $author$
    $else$
        by some unknown author
    $endif$
</div>
```

### Partials

Partials allow you to [DRY] up your templates by putting repetitive actions into
separate template files. You can then include them using
`$partial("filename.html")$`.

[DRY]: http://en.wikipedia.org/wiki/Don%27t_repeat_yourself

An example can be found in `templates/archive.html`:

```html
Here you can find all my previous posts:
$partial("templates/post-list.html")$
```

This partial is just another template and uses the same syntax. Note that in
order to use something like this, we also need to load the partial template in
our `site.hs`:

```haskell
match "templates/post-list.html" $ compile templateCompiler
```

Fortunately, we usually don't need to add this since we already have:

```haskell
match "templates/*" $ compile templateCompiler
```

### Producing a list of items: for

At this point, everything in the example site we generated should be clear to
you, except for how we produce the list of posts in `archive.html` and
`index.html`. Let's look at the `templates/post-list.html` template:

```html
<ul>
    $for(posts)$
        <li>
            <a href="$url$">$title$</a> - $date$
        </li>
    $endfor$
</ul>
```

This uses the `$for(foo)$` construct. This construct allows you loop over a
list, in this case, `$posts$`. Inside the body of this for loop, all fields
refer to the current post, e.g.: `$url$`, `$title$` and `$date$`.

Of course, $posts$ does not magically appear. We have to specify this in
`site.hs`. Let's look at how `archive.html` is generated:

```haskell
posts <- recentFirst =<< loadAll "posts/*"
let archiveCtx =
        listField "posts" postCtx (return posts) `mappend`
        constField "title" "Archives"            `mappend`
        defaultContext
```

We discussed `loadAll` earlier in this tutorial.

`recentFirst` sorts items by date. This relies on the convention that posts are
always named `YYYY-MM-DD-title.extension` in Hakyll -- or that the date must be
present in the metadata.

```haskell
recentFirst :: [Item a] -> Compiler [Item a]
```

After loading and sorting the items, we use `listField` to create the `$posts$`
key.

```haskell
listField :: String -> Context a -> Compiler [Item a] -> Context b
```

The first parameter is simply the name of the key (`"posts"`). Secondly we have
a `Context` with which all items should be rendered -- for our example site, we
already wrote such a `Context` for posts: `postCtx`. Lastly, we have a
`Compiler` which loads the items. We already loaded the items so we can just use
`return posts`.

The following snippet would produce the same result:

```haskell
let archiveCtx =
        listField "posts" postCtx (recentFirst =<< loadAll "posts/*") `mappend`
        constField "title" "Archives"                                 `mappend`
        defaultContext
```