text:
#include <stdlib.h>
#include <string.h> /* memmove */
#include "xmlvect.h"
#include "xmlcfg.h"

/******************************************************************************/

/*
 *  Dinand Vanvelzen edit:
 *  Added this to build easily with CMake
 */
#ifdef PARSIFAL_CUSTOM_GUCEF_BUILD

#undef XMLAPI
#define XMLAPI

#endif /* PARSIFAL_CUSTOM_GUCEF_BUILD ? */

/******************************************************************************/

#define SETCAPACITY _SetCapacity
#define _SetCapacity(v,newsize) \
( (((v)->array = (BYTE*)realloc((v)->array, \
((newsize) * (v)->itemSize))) == (BYTE*)NULL) ? (LPXMLVECTOR)NULL : ((v)->capacity=(newsize), (v)) )
/*static LPXMLVECTOR SetCapacity(LPXMLVECTOR vector, int newsize);*/

/*
	Creates new vector. initialCapacity 0 sets capacity to
	default capacityIncrement which is 6.
	If you want to change capacityIncrement to 2 for example call:
	LPXMLVECTOR v = XMLVector_Create(&v, 2, sizeof(int));
	and after that set
	v->capacityIncrement = 2;

	NOTE: All indices are 0-based.
*/
LPXMLVECTOR XMLAPI XMLVector_Create(LPXMLVECTOR *vector, int initialCapacity, int itemSize)
{
	if ((*vector = (LPXMLVECTOR)malloc(sizeof(XMLVECTOR))) == NULL)
		return (LPXMLVECTOR)NULL;

	((LPXMLVECTOR)*vector)->length = 0;
	((LPXMLVECTOR)*vector)->capacityIncrement = 6;
	((LPXMLVECTOR)*vector)->array = (BYTE*)NULL;
	((LPXMLVECTOR)*vector)->itemSize = itemSize;
	return SETCAPACITY((LPXMLVECTOR)*vector,
		((initialCapacity) ? initialCapacity : ((LPXMLVECTOR)*vector)->capacityIncrement));
}

/*
	Frees the vector and data associated with it.
*/
void XMLAPI XMLVector_Free(LPXMLVECTOR vector)
{
	if (vector->array != (BYTE*)NULL) free(vector->array);
	free(vector);
}

/*
	Resizes vector. Can be used to truncate Vector i.e. to remove
	items from the end of the vector or to clear the whole thing,
	in this case works as XMLVector_RemoveAll macro.
	If you set newsize to 0, capacity will be set default 6.
	XMLVector_Resize Can be used to ensure capacity of newsize too (capacity
	will be set nearest blocksize (capacityIncrement) that can
	hold newsize num of items)
	returns 1 if succesful, 0 otherwise.
*/
int XMLAPI XMLVector_Resize(LPXMLVECTOR vector, int newsize)
{
	int newCapacity;

	if (!newsize) {
		if (vector->capacity > vector->capacityIncrement) {
			if(!(vector = SETCAPACITY(vector, vector->capacityIncrement))) return 0;
		}
		vector->length = 0;
		return 1;
	}

	newCapacity = COUNTBUFSIZE(newsize, vector->capacityIncrement);
	if (newCapacity != vector->capacity) {
		if(!(vector = SETCAPACITY(vector, newCapacity))) return 0;
	}

	if (vector->length > newsize) vector->length = newsize;
	return 1;
}

void XMLAPI *XMLVector_Get(LPXMLVECTOR vector, int index)
{
	return(_XMLVector_Get(vector, index));
}

/*
	Appends item at the end of the vector
	returns pointer to current item if succesful, NULL otherwise.
*/
void XMLAPI *XMLVector_Append(LPXMLVECTOR vector, void *item)
{
	void *cur;
	if ((vector->length + 1) > vector->capacity) {
		vector->capacity = COUNTBUFSIZE(vector->length + 1,
			vector->capacityIncrement);
		if(!(vector = SETCAPACITY(vector, vector->capacity))) return NULL;
	}
	cur = vector->array+vector->length*vector->itemSize;
	if (item != NULL) memcpy(cur, item, vector->itemSize);
	vector->length++;
	return(cur);
}

/*
	Inserts item before specified index.
	returns pointer to current item if succesful, NULL otherwise.
*/
void XMLAPI *XMLVector_InsertBefore(LPXMLVECTOR vector, int index, void *item)
{
	void *cur;
	if (!index && !vector->length) return XMLVector_Append(vector, item);
	if (index < 0 || index > (vector->length - 1)) return NULL;

	if ((vector->length + 1) > vector->capacity) {
		vector->capacity = COUNTBUFSIZE(vector->length + 1,
			vector->capacityIncrement);
		if(!(vector = SETCAPACITY(vector, vector->capacity))) return NULL;
	}

	vector->length++;
	cur = vector->array+index*vector->itemSize;
	memmove(vector->array+((index+1)*vector->itemSize),
		cur, (vector->length - index) * vector->itemSize);
	if (item != NULL) memcpy(cur, item, vector->itemSize);
	return(cur);
}

/*
	Replaces item at specified index.
	returns pointer to current item if succesful, NULL otherwise.
*/
void XMLAPI *XMLVector_Replace(LPXMLVECTOR vector, int index, void *item)
{
	if (index < 0 || index > (vector->length - 1)) {
		return NULL;
	}
	else {
		void *cur;
		cur = vector->array+index*vector->itemSize;
		if (item != NULL) memcpy(cur, item, vector->itemSize);
		return(cur);
	}
}

/*
	Removes item at specified index.
	returns 1 if succesful, 0 otherwise.
*/
int XMLAPI XMLVector_Remove(LPXMLVECTOR vector, int index)
{
	int newCapacity;
	if (index < 0 || index > (vector->length - 1)) return 0;

	if (index != (--vector->length)) {
		memmove(vector->array+index*vector->itemSize,
			vector->array+((index+1)*vector->itemSize),
				(vector->length - index) * vector->itemSize);
	}

	newCapacity = COUNTBUFSIZE(vector->length, vector->capacityIncrement);
	if (newCapacity != vector->capacity) {
		vector->capacity = newCapacity;
		if(!(vector = SETCAPACITY(vector, vector->capacity))) return 0;
	}
	return 1;
}

/*static
LPXMLVECTOR SetCapacity(LPXMLVECTOR vector, int newsize)
{
	if ((vector->array = (BYTE*)realloc(vector->array,
		(newsize * vector->itemSize))) == (BYTE*)NULL) return NULL;
	vector->capacity = newsize;
	return vector;
}*/

redpajama_set_name: RedPajamaGithub


text:
#pragma once
#include <aws/ec2/EC2_EXPORTS.h>
#include <aws/core/utils/memory/stl/AWSStreamFwd.h>
#include <aws/core/utils/memory/stl/AWSString.h>
#include <utility>

namespace Aws
{
namespace Utils
{
namespace Xml
{
  class XmlNode;
} // namespace Xml
} // namespace Utils
namespace EC2
{
namespace Model
{

  /**
   *
Describes a request to purchase Scheduled Instances.

See Also:

   * AWS
   * API Reference

   */
  class AWS_EC2_API PurchaseRequest
  {
  public:
    PurchaseRequest();
    PurchaseRequest(const Aws::Utils::Xml::XmlNode& xmlNode);
    PurchaseRequest& operator=(const Aws::Utils::Xml::XmlNode& xmlNode);

    void OutputToStream(Aws::OStream& ostream, const char* location, unsigned index, const char* locationValue) const;
    void OutputToStream(Aws::OStream& oStream, const char* location) const;

    /**
     *
The number of instances.

     */
    inline int GetInstanceCount() const{ return m_instanceCount; }

    /**
     *
The number of instances.

     */
    inline bool InstanceCountHasBeenSet() const { return m_instanceCountHasBeenSet; }

    /**
     *
The number of instances.

     */
    inline void SetInstanceCount(int value) { m_instanceCountHasBeenSet = true; m_instanceCount = value; }

    /**
     *
The number of instances.

     */
    inline PurchaseRequest& WithInstanceCount(int value) { SetInstanceCount(value); return *this;}

    /**
     *
The purchase token.

     */
    inline const Aws::String& GetPurchaseToken() const{ return m_purchaseToken; }

    /**
     *
The purchase token.

     */
    inline bool PurchaseTokenHasBeenSet() const { return m_purchaseTokenHasBeenSet; }

    /**
     *
The purchase token.

     */
    inline void SetPurchaseToken(const Aws::String& value) { m_purchaseTokenHasBeenSet = true; m_purchaseToken = value; }

    /**
     *
The purchase token.

     */
    inline void SetPurchaseToken(Aws::String&& value) { m_purchaseTokenHasBeenSet = true; m_purchaseToken = std::move(value); }

    /**
     *
The purchase token.

     */
    inline void SetPurchaseToken(const char* value) { m_purchaseTokenHasBeenSet = true; m_purchaseToken.assign(value); }

    /**
     *
The purchase token.

     */
    inline PurchaseRequest& WithPurchaseToken(const Aws::String& value) { SetPurchaseToken(value); return *this;}

    /**
     *
The purchase token.

     */
    inline PurchaseRequest& WithPurchaseToken(Aws::String&& value) { SetPurchaseToken(std::move(value)); return *this;}

    /**
     *
The purchase token.

     */
    inline PurchaseRequest& WithPurchaseToken(const char* value) { SetPurchaseToken(value); return *this;}

  private:

    int m_instanceCount;
    bool m_instanceCountHasBeenSet;

    Aws::String m_purchaseToken;
    bool m_purchaseTokenHasBeenSet;
  };

} // namespace Model
} // namespace EC2
} // namespace Aws


text:
@implementation TUserActionSheet
- (void)dealloc
{
    NSLog(@"dealloc for : %p",self);
}

- (void)didTapSomething:(void (^)(void))block{}
@end


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<TITLE>
org.pentaho.di.core.plugins
</TITLE>

<META NAME="date" CONTENT="2012-11-21">

<LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../../stylesheet.css" TITLE="Style">

      Overview
    Package
      Class
      Use
      Tree
      Deprecated
      Index
      Help

 PREV PACKAGE
 NEXT PACKAGE

  FRAMES
 NO FRAMES

  All Classes

Package org.pentaho.di.core.plugins

Interface Summary

PluginFolderInterface
Describes a possible location for a plugin

PluginInterface
This interface describes the plugin itself, the IDs it listens too, what libraries (jar files) it uses, the names, the i18n detailes, etc.

PluginTypeInterface
This interface describes a plugin type.

 It expresses the ID and the name of the plugin type.

 Then it also explains what the plugin meta class is called and classes the plugin interface itself.

 It also explains us where to load plugins of this type.

Class Summary

BasePluginType

DatabasePluginType
This class represents the step plugin type.

ImportRulePluginType
This is the import rule plugin type.

JarFileAnnotationPlugin

JarFileCache

JobEntryPluginType
This plugin type handles the job entries.

KettleLifecyclePluginType
Defines a Kettle Environment lifecycle plugin type.

KettleURLClassLoader

LifecyclePluginType
This class represents the repository plugin type.

PartitionerPluginType
This is the partitioner plugin type.

Plugin
This describes the plugin itself, the IDs it listens too, what libraries (jar
 files) it uses, the names, the i18n details, etc.

PluginFolder
A folder to search plugins in.

PluginRegistry
This singleton provides access to all the plugins in the Kettle universe.

 It allows you to register types and plugins, query plugin lists per category, list plugins per type, etc.

PluginRegistryTest

RepositoryPluginType
This class represents the repository plugin type.

StepPluginType
This class represents the step plugin type.

Annotation Types Summary

DatabaseMetaPlugin
This annotation signals to the plugin system that the class is a database metadata plugin.

PluginAnnotationType

PluginClassTypeMapping
This annotation describes extra plugin-specific class types to be managed by the registry.

PluginExtraClassTypes

PluginMainClassType
This annotation when applied to a PluginType expresses the main class to
 associate with that type.

PluginTypeCategoriesOrder
This annotation when applied to a PluginTypeInterface instance describes an ordered list of
 categories.

      Overview
    Package
      Class
      Use
      Tree
      Deprecated
      Index
      Help

 PREV PACKAGE
 NEXT PACKAGE

  FRAMES
 NO FRAMES

  All Classes


text:
@echo off
rem #
rem # info   : lfcfs example
rem # author : Piter T. de Boer
rem # MS DOs adaptation: F. Michel
rem #

setlocal ENABLEDELAYEDEXPANSION
set JAVA="%JAVA_HOME%\bin\java"

rem -- Collect command line arguments
set ARGS=
:loop
    if [%1] == [] goto end
    set ARGS=%ARGS% %1
    shift
    goto loop
:end

rem -- Build classpath from jars in the lib folder
set CLASSPATH=.
FOR /R .\lib %%G IN (*.jar) DO set CLASSPATH=!CLASSPATH!;%%G
echo

echo CLASSPATH = %CLASSPATH%
%JAVA% -cp %CLASSPATH% nl.uva.vlet.glite.lfc.main.LfcLs %ARGS%


text:
package org.omg.DynamicAny;

//
// IDL:omg.org/DynamicAny/FieldName:1.0
//
public class FieldNameHelper
{
    public static void
    insert(org.omg.CORBA.Any any, String val)
    {
        org.omg.CORBA.portable.OutputStream out = any.create_output_stream();
        write(out, val);
        any.read_value(out.create_input_stream(), type());
    }

    public static String
    extract(org.omg.CORBA.Any any)
    {
        if(any.type().equivalent(type()))
            return read(any.create_input_stream());
        else
            throw new org.omg.CORBA.BAD_OPERATION();
    }

    private static org.omg.CORBA.TypeCode typeCode_;

    public static org.omg.CORBA.TypeCode
    type()
    {
        if(typeCode_ == null)
        {
            org.omg.CORBA.ORB orb = org.omg.CORBA.ORB.init();
            typeCode_ = orb.create_alias_tc(id(), "FieldName", orb.get_primitive_tc(org.omg.CORBA.TCKind.tk_string));
        }

        return typeCode_;
    }

    public static String
    id()
    {
        return "IDL:omg.org/DynamicAny/FieldName:1.0";
    }

    public static String
    read(org.omg.CORBA.portable.InputStream in)
    {
        String _ob_v;
        _ob_v = in.read_string();
        return _ob_v;
    }

    public static void
    write(org.omg.CORBA.portable.OutputStream out, String val)
    {
        out.write_string(val);
    }
}


text:
<?php

namespace Adi\Html\Table;

use Adi\Html\AbstractPairedStringTag;

/**
 * Helper for Th tag
 *
 * @author adrian
 */
class Th extends AbstractPairedStringTag {

}


text:
<?php

namespace iiifx\cache\dependency;

use yii\base\InvalidConfigException;
use yii\caching\Dependency;
use yii\helpers\FileHelper;
use Yii;

/**
 * Class FolderDependency
 *
 * @author  Vitaliy IIIFX Khomenko <iiifx@yandex.com
 *
 * @package iiifx\cache\dependency
 */
class FolderDependency extends Dependency {

    /**
     * Path to folder
     *
     * @var string[]
     */
    public $folder;

    /**
     * @inheritdoc
     */
    public $reusable = TRUE;

    /**
     * @inheritdoc
     *
     * @throws InvalidConfigException
     */
    public function init () {
        parent::init();
        if ( $this->folder === NULL ) {
            throw new InvalidConfigException( 'FolderDependency::$folder must be set' );
        }
        if ( !is_array( $this->folder ) ) {
            $this->folder = (array) $this->folder;
        }
        if ( !$this->folder ) {
            throw new InvalidConfigException( 'FolderDependency::$folder must be the path to the folder' );
        }
        foreach ( $this->folder as $i => $folder ) {
            $folder = FileHelper::normalizePath( Yii::getAlias( $folder ) );
            if ( !is_dir( $folder ) ) {
                throw new InvalidConfigException( 'FolderDependency::$folder must be the path to the folder' );
            }
            $this->folder[ $i ] = $folder;
        }
    }

    /**
     * @param \yii\caching\Cache $cache
     *
     * @return array|null
     */
    protected function generateDependencyData ( $cache ) {
        $result = [ ];
        foreach ( $this->folder as $folder ) {
            if ( ( $stat = @stat( $folder ) ) ) {
                $result[ ] = $folder . ':' . $stat[ 'atime' ] . '-' . $stat[ 'mtime' ] . '-' . $stat[ 'ctime' ];
            }
        }
        if ( $result ) {
            return implode( '/', $result );
        }
        return NULL;
    }

}


text:
package org.wso2.carbon.siddhi.editor.core.util.designview.constants.query;

/**
 * Type of a Query's Input.
 */
public enum QueryInputType {
    WINDOW_FILTER_PROJECTION,
    WINDOW,
    FUNCTION,
    FILTER,
    PROJECTION,
    JOIN,
    PATTERN,
    SEQUENCE
}


text:
package graph;

import static java.util.Collections.emptyList;
import static java.util.Collections.emptySet;
import static org.junit.jupiter.api.Assertions.assertEquals;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Queue;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;

public class Graph {

  List<Node> getNodes() {
    return Collections.emptyList();
  }

  static class Node {
    int id;
    STATE state;
    List<Integer> adjacent;

    public Node(int id, STATE state, List<Integer> adjacent) {
      this.id = id;
      this.state = state;
      this.adjacent = adjacent;
    }
  }

  enum STATE {
    UNVISITED,
    VISITING,
    VISITED
  }

  /*
  static boolean route(Graph g, Node start, Node end) {
    for (Node n : g.getNodes()) {
      n.state = STATE.UNVISITED;
    }

    Queue<Node> queue = new LinkedList<>();
    queue.add(start);

    while (!queue.isEmpty()) {
      Node q = queue.remove();
      if (q != null) {
        for (Node n : q.adjacent) {
          if (n.state == STATE.UNVISITED) {
            if (n == end) {
              return true;
            } else {
              n.state = STATE.VISITING;
              queue.add(n);
            }
          }
        }
        q.state = STATE.VISITED;
      }
    }

    return false;
  }
   */

  @Test
  public void test() {
    final List<Integer> descendents = descendents(1, new int[][] {{1, 2}, {1, 3}, {3, 5}, {5, 1}});

    assertEquals(1, descendents);
  }

  static List<Integer> descendents(int startNode, int[][] graph) {

    final Map<Integer, Set<Integer>> nodes =
        Arrays.stream(graph)
            .collect(
                Collectors.groupingBy(
                    item -> item[0], Collectors.mapping(item -> item[1], Collectors.toSet())));

    if (!nodes.containsKey(startNode)) {
      return emptyList();
    }

    Set<Integer> visited = new HashSet<>();
    List<Integer> results = new ArrayList<>();
    Queue<Integer> queue = new LinkedList<>();
    queue.add(startNode);

    while(!queue.isEmpty()) {
      Integer n = queue.remove();
      final Set<Integer> adj = nodes.getOrDefault(n, emptySet());
      for(Integer q : adj) {
        if (!visited.contains(q)) {
          results.add(q);
          queue.add(q);
        }
      }
      visited.add(n);
    }

    return results.stream().sorted().collect(Collectors.toList());
  }
}


text:
<?php

declare(strict_types=1);

namespace DCarbone\PHPConsulAPI\ACL;

use DCarbone\PHPConsulAPI\AbstractValuedQueryResponse;
use DCarbone\PHPConsulAPI\UnmarshalledResponseInterface;

/**
 * Class ACLEntriesResponse
 */
class ACLEntriesResponse extends AbstractValuedQueryResponse implements UnmarshalledResponseInterface
{
    /** @var \DCarbone\PHPConsulAPI\ACL\ACLEntry[]|null */
    public array $ACLEntries = [];

    /**
     * @return \DCarbone\PHPConsulAPI\ACL\ACLEntry[]|null
     */
    public function getValue(): ?array
    {
        return $this->ACLEntries;
    }

    /**
     * @param mixed $decodedData
     */
    public function unmarshalValue($decodedData): void
    {
        foreach ($decodedData as $entry) {
            $this->ACLEntries[] = new ACLEntry($entry);
        }
    }
}


text:
@interface PWGetNearestZoneRequest : PWRequest

@property CLLocationCoordinate2D coordinate;

@property (nonatomic, assign) CLLocationDistance distance;

@end


text:
<!DOCTYPE html>

    <title>Efficient anti-sampling with the Google Analytics Reporting API · Mark Edmondson</title>
    <meta name="generator" content="Hugo 0.55.5" />
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
    <meta name="author" content="Mark Edmondson">

      <meta name="description" content="The code bit of the blog">

    <link rel="canonical" href="https://code.markedmondson.me/anti-sampling-google-analytics-api/"/>
    <link rel="icon" href="https://code.markedmondson.me/favicon.ico">
    <link rel="apple-touch-icon" href="https://code.markedmondson.me/apple-touch-icon.png"/>
    <link rel="stylesheet" href="https://code.markedmondson.me/css/style.css">
    <link rel="stylesheet" href="https://code.markedmondson.me/css/github.css" rel="stylesheet" id="theme-stylesheet">
    <link rel="stylesheet" href="https://code.markedmondson.me/css/font-awesome.min.css">
    <link rel="stylesheet" href="https://code.markedmondson.me/fancybox/jquery.fancybox.css">

    <link href='https://fonts.googleapis.com/css?family=Open+Sans:400italic,400,600' rel='stylesheet' type='text/css'>
    <link href='https://fonts.googleapis.com/css?family=Source+Code+Pro' rel='stylesheet' type='text/css'>
    <meta property="og:title" content="Efficient anti-sampling with the Google Analytics Reporting API" />
<meta property="og:description" content="Avoiding sampling is one of the most common reasons people start using the Google Analytics API. This blog lays out some pseudo-code to do so in an efficient manner, avoiding too many unnecessary API calls. The approach is used in the v4 calls for the R package googleAnalyticsR.
Avoiding the daily walk The most common approach to mitigate sampling is to break down the API calls into one call per day." />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://code.markedmondson.me/anti-sampling-google-analytics-api/" />
<meta property="article:published_time" content="2016-08-05T23:03:57+01:00"/>
<meta property="article:modified_time" content="2016-08-05T23:03:57+01:00"/>

<meta itemprop="name" content="Efficient anti-sampling with the Google Analytics Reporting API">
<meta itemprop="description" content="Avoiding sampling is one of the most common reasons people start using the Google Analytics API. This blog lays out some pseudo-code to do so in an efficient manner, avoiding too many unnecessary API calls. The approach is used in the v4 calls for the R package googleAnalyticsR.
Avoiding the daily walk The most common approach to mitigate sampling is to break down the API calls into one call per day.">

<meta itemprop="datePublished" content="2016-08-05T23:03:57+01:00" />
<meta itemprop="dateModified" content="2016-08-05T23:03:57+01:00" />
<meta itemprop="wordCount" content="932">

<meta itemprop="keywords" content="R,google-analytics," />

    <meta name="twitter:card" content="summary"/>
<meta name="twitter:title" content="Efficient anti-sampling with the Google Analytics Reporting API"/>
<meta name="twitter:description" content="Avoiding sampling is one of the most common reasons people start using the Google Analytics API. This blog lays out some pseudo-code to do so in an efficient manner, avoiding too many unnecessary API calls. The approach is used in the v4 calls for the R package googleAnalyticsR.
Avoiding the daily walk The most common approach to mitigate sampling is to break down the API calls into one call per day."/>
<meta name="twitter:site" content="@HoloMarkeD"/>

          Mark Edmondson

          Home

          My R Packages

          Non-code blog

          Past Presentations

          <form action="//google.com/search" method="get" accept-charset="UTF-8" class="search-form">
              <input type="search" name="q" class="search-form-input" placeholder="Search">
              <button type="submit" class="search-form-submit">
              </button>
              <input type="hidden" name="sitesearch" value="https://code.markedmondson.me/" />
         </form>

          Home

          My R Packages

          Non-code blog

          Past Presentations

          <form action="//google.com/search" method="get" accept-charset="UTF-8" class="search-form">
          <input type="search" name="q" class="search-form-input" placeholder="Search">
          <input type="hidden" name="sitesearch" value="https://code.markedmondson.me/" />
          </form>

Mark Edmondson

Coding in digital analytics

      Copenhagen, Denmark

              Follow

        28
        Posts

          17

            Tags

        Efficient anti-sampling with the Google Analytics Reporting API

            <time datetime="2016-08-05 23:03:57 +0100 +0100" itemprop="datePublished">2016-08-05</time>
            ·
            932
            words
            ·
            5
            minute read

                R
                ·

                google-analytics

Avoiding sampling is one of the most common reasons people start using the Google Analytics API.  This blog lays out some pseudo-code to do so in an efficient manner, avoiding too many unnecessary API calls.  The approach is used in the v4 calls for the R package googleAnalyticsR.

Avoiding the daily walk

The most common approach to mitigate sampling is to break down the API calls into one call per day.  This has some problems:

Its inefficient.  If you have 80% sampling or 10% sampling, you use the same number of API calls.

It takes a long time.  A year long fetch is 365 calls of 5+ seconds that can equate to a 30mins+ wait.

It doesn't always work. If you have so many sessions its sampled for one day, you will still have sampling, albeit at a lower rate.

Anti-sampling based on session size

Google Analytics sampling works as outlined in this Google article.  The main points are that if your API call covers a date range greater than set session limits, it will return a sampled call.

The session limits vary according to if you are using Google Analytics 360 and other unknown factors in a sampling algorithm.  Fortunately, this information is available in the API responses via the samplesReadCounts and samplingSpaceSizes meta data.  See the v4 API reference for their definitions.

These values change per API call, so the general strategy is to make two exploratory API calls first to get the sampling information and the number of sessions over the desired date period, then use that information to construct batches of calls over date ranges that are small enough to avoid sampling, but large enough to not waste API calls.

The two exploratory API calls to find the meta data are more than made up for once you have saved calls in the actual data fetch.

How it works in practice - 80%+ quicker data

Following this approach, I have found a huge improvement in time spent for sampled calls, making it much more useable in say dynamic dashboards where waiting 30 mins for data is not an option.

An example response from the googleAnalyticsR library is shown below - for a month’s worth of unsampled data  that would have taken 30 API calls via a daily walk, I get the same in 5 (2 to find batch sizes, 3 to get the data).

> library(googleAnalyticsR)
> ga_auth()
> ga_data <-
    google_analytics_4(id,
                       date_range = c("2016-01-01",
                                      "2016-02-01"),
                       metrics = c("sessions",
                                   "bounceRate"),
                       dimensions = c("date",
                                      "landingPagePath",
                                      "source"),
                       anti_sample = TRUE)

anti_sample set to TRUE. Mitigating sampling via multiple API calls.
Finding how much sampling in data request...
Data is sampled, based on 54.06% of visits.
Downloaded [10] rows from a total of [76796].
Finding number of sessions for anti-sample calculations...
Downloaded [32] rows from a total of [32].
Calculated [3] batches are needed to download [113316] rows unsampled.
Anti-sample call covering 10 days: 2016-01-01, 2016-01-10
Downloaded [38354] rows from a total of [38354].
Anti-sample call covering 20 days: 2016-01-11, 2016-01-30
Downloaded [68475] rows from a total of [68475].
Anti-sample call covering 2 days: 2016-01-31, 2016-02-01
Downloaded [6487] rows from a total of [6487].
Finished unsampled data request, total rows [113316]
Successfully avoided sampling

The time saved gets even greater the longer the time period you request.

Limitations

As with daily walk anti-sample techniques, user metrics and unique users are linked to the date range you are querying, so this technique will not match the numbers as if you queried over the whole date range.

The sampling session limit is also applied at a web property level, not View for non-GA360 accounts, so its best to use this on a Raw data View, as filters will cause the session calculations be incorrect.

Example pseudo-code

R code that implements the above is available here, but the pseudo-code below is intended for you to port to different languages:

// Get the unsampled data
test_call = get_ga_api(full_date_range,
                       metrics = your_metrics,
                       dimensions = your_dimensions)

// # Read the sample meta data

// read_counts is the number of sessions before sampling starts
// I make it 10% smaller to ensure its small enough as
// it seems a bit flakey
read_counts = meta_data(test_call, "sampledReadCounts")
read_counts = read_counts * 0.9

// space_size is total amount of sessions sampling was used for
space_size = meta_data(test_call, "samplingSpaceSizes")

// dividing by each gives the % of sampling in the API call
samplingPercent = read_counts / space_size

// if there is no sample meta data, its not sampled. We're done!
if(read_counts = NULL or space_size = NULL):
  return(test_call)

// ..otherwise its sampled
// # get info for batch size calculations

// I found rowCount returned from a sampled call was not equal
// to an unsampled call, so I add 20% to rowCount to adjust
rowCount = meta_data(test_call, "rowCount")
rowCount = rowCount * 1.2

// get the number of sessions per day
date_sessions = get_ga_api(full_date_range,
                           metric = "sessions",
                           dimensions = "date")

// get the cumulative number of sessions over the year
date_sessions.cumulative = cumsum(date_sessions.sessions)

// modulus divide the cumulative sessions by the
// sample read_counts.
date_sessions.sample_bucket = date_sessions.cumulative %%
                                            read_counts

// get the new date ranges per sample_bucket group
new_date_ranges = get_min_and_max_date(date_sessions)

// new_date_ranges should now hold the smaller date ranges
// for each batched API call

// # call GA api with new date ranges

total = empty_matrix

for i in new_date_ranges:

    // multi-day batching
    batch_call = get_ga_api(new_date_ranges[i],
                            metrics = your_metrics,
                            dimensions = your_dimensions)

  total = total + batch_call

If you need anti-sampling for sub-daily, then you should really be looking at using the BigQuery Google Analytics 360 exports. :)

        Share

        Comments

          Older

SEO keyword research using searchConsoleR and googleAnalyticsR

          Newer

A digital analytics workflow through the Google Cloud using R

Please enable JavaScript to view the comments powered by Disqus.
comments powered by Disqus

        Recents

Data Privacy Engineering with Google Tag Manager Server Side and Consent Mode

                        <time datetime="2020-11-27 09:29:50 +0100 CET" itemprop="datePublished">2020-11-27</time>

Using Google Tag Manager Server Side to Trigger Webhook Events

                        <time datetime="2020-09-04 12:50:50 +0100 +0100" itemprop="datePublished">2020-09-04</time>

Google Tag Manager Server Side on Cloud Run - Pros and Cons

                        <time datetime="2020-08-21 00:00:00 +0000 UTC" itemprop="datePublished">2020-08-21</time>

Shiny on Google Cloud Run - Scale-to-Zero R Web Apps

                        <time datetime="2020-08-01 00:00:00 +0000 UTC" itemprop="datePublished">2020-08-01</time>

Online payments for data science apps (DSaaS) using R, Shiny, Firebase, Paddle and Google Cloud Functions

                        <time datetime="2020-06-28 00:00:00 +0000 UTC" itemprop="datePublished">2020-06-28</time>

        Tags

                    R

                20

                    big-query

                5

                    blogging

                2

                    cloud-functions

                4

                    cloud-run

                3

                    docker

                9

                    firebase

                1

                    google-analytics

                10

                    google-app-engine

                4

                    google-auth

                4

                    google-cloud-storage

                4

                    google-compute-engine

                7

                    google-language

                2

                    google-tag-manager

                6

                    python

                5

                    rstudio-server

                4

                    search-console

                3

      © 2020
      Powered by Hugo. Theme by PPOffice.


text:
<?php

namespace Tystr\RedisOrm\Criteria;

/**
 * @author Justin Taft <justin.t@zeetogroup.com>
 */
class AndGroup extends Restriction implements AndGroupInterface
{
    /**
     * @return Restrictions[]
     */
    public function getValue() {
        return $this->value;
    }
}


text:
<action>
    <field name="type">report_odt</field>
    <field name="model">account.payment</field>
    <field name="method">get_report_data</field>
    <field name="multi_page">1</field>
</action>


text:
#pragma once

#include "PDFParserTokenizer.h"
#include "EStatusCode.h"
#include "IOBasicTypes.h"
#include "IReadPositionProvider.h"

#include <list>
#include <string>
#include <utility>

class PDFObject;
class IByteReader;
class IPDFParserExtender;
class DecryptionHelper;

typedef std::pair<bool,IOBasicTypes::Byte> BoolAndByte;
typedef std::list<std::string> StringList;

class PDFObjectParser
{
public:
	PDFObjectParser(void);
	~PDFObjectParser(void);

	// Assign the stream to read from (does not take ownership of the stream)
	void SetReadStream(IByteReader* inSourceStream,IReadPositionProvider* inCurrentPositionProvider);

	PDFObject* ParseNewObject();

	// calls this when changing underlying stream position
	void ResetReadState();
	// reset, but with an external tokenizer, which is useful if you are using
	// another pdf parser tokenizer and want to start parsing objects, and don't want to lose
	// buffered characters
	void ResetReadState(const PDFParserTokenizer& inExternalTokenizer);

	void SetDecryptionHelper(DecryptionHelper* inDecryptionHelper);
	void SetParserExtender(IPDFParserExtender* inParserExtender);

	// helper method for others who need to parse encoded pdf data
	std::string DecodeHexString(const std::string inStringToDecode);

private:
	PDFParserTokenizer mTokenizer;
	StringList mTokenBuffer;
	IByteReader* mStream;
	IReadPositionProvider* mCurrentPositionProvider;
	IPDFParserExtender* mParserExtender;
	DecryptionHelper* mDecryptionHelper;

	bool GetNextToken(std::string& outToken);
	void SaveTokenToBuffer(std::string& inToken);
	void ReturnTokenToBuffer(std::string& inToken);

	bool IsBoolean(const std::string& inToken);
	PDFObject* ParseBoolean(const std::string& inToken);

	bool IsLiteralString(const std::string& inToken);
	PDFObject* ParseLiteralString(const std::string& inToken);

	bool IsHexadecimalString(const std::string& inToken);
	PDFObject* ParseHexadecimalString(const std::string& inToken);

	bool IsNull(const std::string& inToken);

	bool IsName(const std::string& inToken);
	PDFObject* ParseName(const std::string& inToken);

	bool IsNumber(const std::string& inToken);
	PDFObject* ParseNumber(const std::string& inToken);

	bool IsArray(const std::string& inToken);
	PDFObject* ParseArray();

	bool IsDictionary(const std::string& inToken);
	PDFObject* ParseDictionary();

	bool IsComment(const std::string& inToken);

	BoolAndByte GetHexValue(IOBasicTypes::Byte inValue);

	std::string MaybeDecryptString(const std::string& inString);

};


text:
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from matplotlib.externals import six
from matplotlib.externals.six.moves import zip

import warnings

import numpy as np

from matplotlib.path import Path
from matplotlib import rcParams
import matplotlib.font_manager as font_manager
from matplotlib.ft2font import FT2Font, KERNING_DEFAULT, LOAD_NO_HINTING
from matplotlib.ft2font import LOAD_TARGET_LIGHT
from matplotlib.mathtext import MathTextParser
import matplotlib.dviread as dviread
from matplotlib.font_manager import FontProperties
from matplotlib.transforms import Affine2D
from matplotlib.externals.six.moves.urllib.parse import quote as urllib_quote

class TextToPath(object):
    """
    A class that convert a given text to a path using ttf fonts.
    """

    FONT_SCALE = 100.
    DPI = 72

    def __init__(self):
        """
        Initialization
        """
        self.mathtext_parser = MathTextParser('path')
        self.tex_font_map = None

        from matplotlib.cbook import maxdict
        self._ps_fontd = maxdict(50)

        self._texmanager = None

        self._adobe_standard_encoding = None

    def _get_adobe_standard_encoding(self):
        enc_name = dviread.find_tex_file('8a.enc')
        enc = dviread.Encoding(enc_name)
        return dict([(c, i) for i, c in enumerate(enc.encoding)])

    def _get_font(self, prop):
        """
        find a ttf font.
        """
        fname = font_manager.findfont(prop)
        font = FT2Font(fname)
        font.set_size(self.FONT_SCALE, self.DPI)

        return font

    def _get_hinting_flag(self):
        return LOAD_NO_HINTING

    def _get_char_id(self, font, ccode):
        """
        Return a unique id for the given font and character-code set.
        """
        sfnt = font.get_sfnt()
        try:
            ps_name = sfnt[(1, 0, 0, 6)].decode('macroman')
        except KeyError:
            ps_name = sfnt[(3, 1, 0x0409, 6)].decode('utf-16be')
        char_id = urllib_quote('%s-%x' % (ps_name, ccode))
        return char_id

    def _get_char_id_ps(self, font, ccode):
        """
        Return a unique id for the given font and character-code set (for tex).
        """
        ps_name = font.get_ps_font_info()[2]
        char_id = urllib_quote('%s-%d' % (ps_name, ccode))
        return char_id

    def glyph_to_path(self, font, currx=0.):
        """
        convert the ft2font glyph to vertices and codes.
        """
        verts, codes = font.get_path()
        if currx != 0.0:
            verts[:, 0] += currx
        return verts, codes

    def get_text_width_height_descent(self, s, prop, ismath):
        if rcParams['text.usetex']:
            texmanager = self.get_texmanager()
            fontsize = prop.get_size_in_points()
            w, h, d = texmanager.get_text_width_height_descent(s, fontsize,
                                                               renderer=None)
            return w, h, d

        fontsize = prop.get_size_in_points()
        scale = float(fontsize) / self.FONT_SCALE

        if ismath:
            prop = prop.copy()
            prop.set_size(self.FONT_SCALE)

            width, height, descent, trash, used_characters = \
                self.mathtext_parser.parse(s, 72, prop)
            return width * scale, height * scale, descent * scale

        font = self._get_font(prop)
        font.set_text(s, 0.0, flags=LOAD_NO_HINTING)
        w, h = font.get_width_height()
        w /= 64.0  # convert from subpixels
        h /= 64.0
        d = font.get_descent()
        d /= 64.0
        return w * scale, h * scale, d * scale

    def get_text_path(self, prop, s, ismath=False, usetex=False):
        """
        convert text *s* to path (a tuple of vertices and codes for
        matplotlib.path.Path).

        *prop*
          font property

        *s*
          text to be converted

        *usetex*
          If True, use matplotlib usetex mode.

        *ismath*
          If True, use mathtext parser. Effective only if usetex == False.

        """
        if not usetex:
            if not ismath:
                font = self._get_font(prop)
                glyph_info, glyph_map, rects = self.get_glyphs_with_font(
                                                    font, s)
            else:
                glyph_info, glyph_map, rects = self.get_glyphs_mathtext(
                                                    prop, s)
        else:
            glyph_info, glyph_map, rects = self.get_glyphs_tex(prop, s)

        verts, codes = [], []

        for glyph_id, xposition, yposition, scale in glyph_info:
            verts1, codes1 = glyph_map[glyph_id]
            if len(verts1):
                verts1 = np.array(verts1) * scale + [xposition, yposition]
                verts.extend(verts1)
                codes.extend(codes1)

        for verts1, codes1 in rects:
            verts.extend(verts1)
            codes.extend(codes1)

        return verts, codes

    def get_glyphs_with_font(self, font, s, glyph_map=None,
                             return_new_glyphs_only=False):
        """
        convert the string *s* to vertices and codes using the
        provided ttf font.
        """

        # Mostly copied from backend_svg.py.

        cmap = font.get_charmap()
        lastgind = None

        currx = 0
        xpositions = []
        glyph_ids = []

        if glyph_map is None:
            glyph_map = dict()

        if return_new_glyphs_only:
            glyph_map_new = dict()
        else:
            glyph_map_new = glyph_map

        # I'm not sure if I get kernings right. Needs to be verified. -JJL

        for c in s:
            ccode = ord(c)
            gind = cmap.get(ccode)
            if gind is None:
                ccode = ord('?')
                gind = 0

            if lastgind is not None:
                kern = font.get_kerning(lastgind, gind, KERNING_DEFAULT)
            else:
                kern = 0

            glyph = font.load_char(ccode, flags=LOAD_NO_HINTING)
            horiz_advance = (glyph.linearHoriAdvance / 65536.0)

            char_id = self._get_char_id(font, ccode)
            if char_id not in glyph_map:
                glyph_map_new[char_id] = self.glyph_to_path(font)

            currx += (kern / 64.0)

            xpositions.append(currx)
            glyph_ids.append(char_id)

            currx += horiz_advance

            lastgind = gind

        ypositions = [0] * len(xpositions)
        sizes = [1.] * len(xpositions)

        rects = []

        return (list(zip(glyph_ids, xpositions, ypositions, sizes)),
                     glyph_map_new, rects)

    def get_glyphs_mathtext(self, prop, s, glyph_map=None,
                            return_new_glyphs_only=False):
        """
        convert the string *s* to vertices and codes by parsing it with
        mathtext.
        """

        prop = prop.copy()
        prop.set_size(self.FONT_SCALE)

        width, height, descent, glyphs, rects = self.mathtext_parser.parse(
            s, self.DPI, prop)

        if not glyph_map:
            glyph_map = dict()

        if return_new_glyphs_only:
            glyph_map_new = dict()
        else:
            glyph_map_new = glyph_map

        xpositions = []
        ypositions = []
        glyph_ids = []
        sizes = []

        currx, curry = 0, 0
        for font, fontsize, ccode, ox, oy in glyphs:
            char_id = self._get_char_id(font, ccode)
            if char_id not in glyph_map:
                font.clear()
                font.set_size(self.FONT_SCALE, self.DPI)
                glyph = font.load_char(ccode, flags=LOAD_NO_HINTING)
                glyph_map_new[char_id] = self.glyph_to_path(font)

            xpositions.append(ox)
            ypositions.append(oy)
            glyph_ids.append(char_id)
            size = fontsize / self.FONT_SCALE
            sizes.append(size)

        myrects = []
        for ox, oy, w, h in rects:
            vert1 = [(ox, oy), (ox, oy + h), (ox + w, oy + h),
                     (ox + w, oy), (ox, oy), (0, 0)]
            code1 = [Path.MOVETO,
                     Path.LINETO, Path.LINETO, Path.LINETO, Path.LINETO,
                     Path.CLOSEPOLY]
            myrects.append((vert1, code1))

        return (list(zip(glyph_ids, xpositions, ypositions, sizes)),
                glyph_map_new, myrects)

    def get_texmanager(self):
        """
        return the :class:`matplotlib.texmanager.TexManager` instance
        """
        if self._texmanager is None:
            from matplotlib.texmanager import TexManager
            self._texmanager = TexManager()
        return self._texmanager

    def get_glyphs_tex(self, prop, s, glyph_map=None,
                       return_new_glyphs_only=False):
        """
        convert the string *s* to vertices and codes using matplotlib's usetex
        mode.
        """

        # codes are modstly borrowed from pdf backend.

        texmanager = self.get_texmanager()

        if self.tex_font_map is None:
            self.tex_font_map = dviread.PsfontsMap(
                                    dviread.find_tex_file('pdftex.map'))

        if self._adobe_standard_encoding is None:
            self._adobe_standard_encoding = self._get_adobe_standard_encoding()

        fontsize = prop.get_size_in_points()
        if hasattr(texmanager, "get_dvi"):
            dvifilelike = texmanager.get_dvi(s, self.FONT_SCALE)
            dvi = dviread.DviFromFileLike(dvifilelike, self.DPI)
        else:
            dvifile = texmanager.make_dvi(s, self.FONT_SCALE)
            dvi = dviread.Dvi(dvifile, self.DPI)
        try:
            page = next(iter(dvi))
        finally:
            dvi.close()

        if glyph_map is None:
            glyph_map = dict()

        if return_new_glyphs_only:
            glyph_map_new = dict()
        else:
            glyph_map_new = glyph_map

        glyph_ids, xpositions, ypositions, sizes = [], [], [], []

        # Gather font information and do some setup for combining
        # characters into strings.
        # oldfont, seq = None, []
        for x1, y1, dvifont, glyph, width in page.text:
            font_and_encoding = self._ps_fontd.get(dvifont.texname)
            font_bunch = self.tex_font_map[dvifont.texname]

            if font_and_encoding is None:
                font = FT2Font(font_bunch.filename)

                for charmap_name, charmap_code in [("ADOBE_CUSTOM",
                                                    1094992451),
                                                   ("ADOBE_STANDARD",
                                                    1094995778)]:
                    try:
                        font.select_charmap(charmap_code)
                    except ValueError:
                        pass
                    else:
                        break
                else:
                    charmap_name = ""
                    warnings.warn("No supported encoding in font (%s)." %
                                  font_bunch.filename)

                if charmap_name == "ADOBE_STANDARD" and font_bunch.encoding:
                    enc0 = dviread.Encoding(font_bunch.encoding)
                    enc = dict([(i, self._adobe_standard_encoding.get(c, None))
                                for i, c in enumerate(enc0.encoding)])
                else:
                    enc = dict()
                self._ps_fontd[dvifont.texname] = font, enc

            else:
                font, enc = font_and_encoding

            ft2font_flag = LOAD_TARGET_LIGHT

            char_id = self._get_char_id_ps(font, glyph)

            if char_id not in glyph_map:
                font.clear()
                font.set_size(self.FONT_SCALE, self.DPI)
                if enc:
                    charcode = enc.get(glyph, None)
                else:
                    charcode = glyph

                if charcode is not None:
                    glyph0 = font.load_char(charcode, flags=ft2font_flag)
                else:
                    warnings.warn("The glyph (%d) of font (%s) cannot be "
                                  "converted with the encoding. Glyph may "
                                  "be wrong" % (glyph, font_bunch.filename))

                    glyph0 = font.load_char(glyph, flags=ft2font_flag)

                glyph_map_new[char_id] = self.glyph_to_path(font)

            glyph_ids.append(char_id)
            xpositions.append(x1)
            ypositions.append(y1)
            sizes.append(dvifont.size / self.FONT_SCALE)

        myrects = []

        for ox, oy, h, w in page.boxes:
            vert1 = [(ox, oy), (ox + w, oy), (ox + w, oy + h),
                     (ox, oy + h), (ox, oy), (0, 0)]
            code1 = [Path.MOVETO,
                     Path.LINETO, Path.LINETO, Path.LINETO, Path.LINETO,
                     Path.CLOSEPOLY]
            myrects.append((vert1, code1))

        return (list(zip(glyph_ids, xpositions, ypositions, sizes)),
                glyph_map_new, myrects)

text_to_path = TextToPath()

class TextPath(Path):
    """
    Create a path from the text.
    """

    def __init__(self, xy, s, size=None, prop=None,
                 _interpolation_steps=1, usetex=False,
                 *kl, **kwargs):
        """
        Create a path from the text. No support for TeX yet. Note that
        it simply is a path, not an artist. You need to use the
        PathPatch (or other artists) to draw this path onto the
        canvas.

        xy : position of the text.
        s : text
        size : font size
        prop : font property
        """

        if prop is None:
            prop = FontProperties()

        if size is None:
            size = prop.get_size_in_points()

        self._xy = xy
        self.set_size(size)

        self._cached_vertices = None

        self._vertices, self._codes = self.text_get_vertices_codes(
                                            prop, s,
                                            usetex=usetex)

        self._should_simplify = False
        self._simplify_threshold = rcParams['path.simplify_threshold']
        self._has_nonfinite = False
        self._interpolation_steps = _interpolation_steps

    def set_size(self, size):
        """
        set the size of the text
        """
        self._size = size
        self._invalid = True

    def get_size(self):
        """
        get the size of the text
        """
        return self._size

    def _get_vertices(self):
        """
        Return the cached path after updating it if necessary.
        """
        self._revalidate_path()
        return self._cached_vertices

    def _get_codes(self):
        """
        Return the codes
        """
        return self._codes

    vertices = property(_get_vertices)
    codes = property(_get_codes)

    def _revalidate_path(self):
        """
        update the path if necessary.

        The path for the text is initially create with the font size
        of FONT_SCALE, and this path is rescaled to other size when
        necessary.

        """
        if (self._invalid or
            (self._cached_vertices is None)):
            tr = Affine2D().scale(
                    self._size / text_to_path.FONT_SCALE,
                    self._size / text_to_path.FONT_SCALE).translate(*self._xy)
            self._cached_vertices = tr.transform(self._vertices)
            self._invalid = False

    def is_math_text(self, s):
        """
        Returns True if the given string *s* contains any mathtext.
        """
        # copied from Text.is_math_text -JJL

        # Did we find an even number of non-escaped dollar signs?
        # If so, treat is as math text.
        dollar_count = s.count(r'$') - s.count(r'\$')
        even_dollars = (dollar_count > 0 and dollar_count % 2 == 0)

        if rcParams['text.usetex']:
            return s, 'TeX'

        if even_dollars:
            return s, True
        else:
            return s.replace(r'\$', '$'), False

    def text_get_vertices_codes(self, prop, s, usetex):
        """
        convert the string *s* to vertices and codes using the
        provided font property *prop*. Mostly copied from
        backend_svg.py.
        """

        if usetex:
            verts, codes = text_to_path.get_text_path(prop, s, usetex=True)
        else:
            clean_line, ismath = self.is_math_text(s)
            verts, codes = text_to_path.get_text_path(prop, clean_line,
                                                      ismath=ismath)

        return verts, codes


text:
import { AllHtmlEntities as Entities } from 'html-entities';
import fetch from 'node-fetch';
import xss from 'xss';

const decoder = new Entities();

const attemptLyricsWikia = (path) =>
  new Promise((resolve, reject) => {
    fetch(`http://lyrics.wikia.com/wiki/${path}`)
      .then((data) => data.text())
      .then((html) => {
        try {
          let lyrics = (/('|")lyricbox('|")>(.+<\/script>)?(.+)(<!--)?/g.exec(html)[4]);
          lyrics = lyrics.replace(/<br \/>/gi, '\n');
          lyrics = decoder.decode(lyrics);
          lyrics = xss(lyrics, {
            whiteList: { br: [], i: [], b: [], strong: [], em: [] },
            stripIgnoreTag: true,
            stripIgnoreTagBody: ['script'],
          });
          resolve(lyrics);
        } catch (err) {
          reject(err);
        }
      })
      .catch(reject);
  });

export default attemptLyricsWikia;


text:
<!--

    Copyright © 2016-2022 The Thingsboard Authors

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.

-->
<div class="input-wrapper" [ngClass]="{'focus': focus}" fxLayoutAlign="start center" fxLayoutGap="16px">
  <mat-icon>search</mat-icon>
  <input type="text"
         [(ngModel)]="searchText"
         (ngModelChange)="updateSearchText()"
         (focus)="toggleFocus()"
         (blur)="toggleFocus()"
         [placeholder]=placeholder>
  <button mat-button type="button" *ngIf="searchText" mat-icon-button (click)="clear($event)">
    <mat-icon class="close">close</mat-icon>
  </button>
</div>


text:
"""

Dialogs.
Cross-platform and cross-application compatible. Some of them anyway.
(Not all dialogs work on PCs outside of FontLab. Some dialogs are for FontLab only. Sorry.)

Mac and FontLab implementation written by the RoboFab development team.
PC implementation by Eigi Eigendorf and is (C)2002 Eigi Eigendorf.

"""

import os
import sys
from robofab import RoboFabError
from warnings import warn

MAC = False
PC = False
haveMacfs = False

if sys.platform in ('mac', 'darwin'):
	MAC = True
elif os.name == 'nt':
	PC = True
else:
	warn("dialogs.py only supports Mac and PC platforms.")
pyVersion = sys.version_info[:3]

inFontLab = False
try:
	from FL import *
	inFontLab = True
except ImportError: pass

try:
	import W
	hasW = True
except ImportError:
	hasW = False

try:
	import dialogKit
	hasDialogKit = True
except ImportError:
	hasDialogKit = False

try:
	import EasyDialogs
	hasEasyDialogs = True
except:
	hasEasyDialogs = False

if MAC:
	if pyVersion < (2, 3, 0):
		import macfs
		haveMacfs = True
elif PC and not inFontLab:
	from win32com.shell import shell
	import win32ui
	import win32con

def _raisePlatformError(dialog):
	"""error raiser"""
	if MAC:
		p = 'Macintosh'
	elif PC:
		p = 'PC'
	else:
		p = sys.platform
	raise RoboFabError("%s is not currently available on the %s platform"%(dialog, p))

class _FontLabDialogOneList:
	"""A one list dialog for FontLab. This class should not be called directly. Use the OneList function."""

	def __init__(self, list, message, title='RoboFab'):
		self.message = message
		self.selected = None
		self.list = list
		self.d = Dialog(self)
		self.d.size = Point(250, 250)
		self.d.title = title
		self.d.Center()
		self.d.AddControl(LISTCONTROL, Rect(12, 30, 238, 190), "list", STYLE_LIST, self.message)
		self.list_index = 0

	def Run(self):
		return self.d.Run()

	def on_cancel(self, code):
		self.selected = None

	def on_ok(self, code):
		self.d.GetValue('list')
		# Since FLS v5.2, the GetValue() method of the Dialog() class returns
		# a 'wrong' index value from the specified LISTCONTROL.
		# If the selected index is n, it will return n-1. For example, when
		# the index is 1, it returns 0; when it's 2, it returns 1, and so on.
		# If the selection is empty, FLS v5.2 returns -2, while the old v5.0
		# returned None.
		# See also:
		# - http://forum.fontlab.com/index.php?topic=8807.0
		# - http://forum.fontlab.com/index.php?topic=9003.0
		#
		# Edited based on feedback from Adam Twardoch
		if fl.buildnumber > 4600 and sys.platform == 'win32':
			if self.list_index == -2:
				self.selected = None
			else:
				self.selected = self.list_index + 1
		else:
			self.selected = self.list_index

class _FontLabDialogSearchList:
	"""A dialog for searching through a list. It contains a text field and a results list FontLab. This class should not be called directly. Use the SearchList function."""

	def __init__(self, aList, message, title="RoboFab"):
		self.d = Dialog(self)
		self.d.size = Point(250, 290)
		self.d.title = title
		self.d.Center()

		self.message = message
		self._fullContent = aList
		self.possibleHits = list(aList)
		self.possibleHits.sort()
		self.possibleHits_index = 0
		self.entryField = ""
		self.selected = None

		self.d.AddControl(STATICCONTROL, Rect(10, 10, 240, 30), "message", STYLE_LABEL, message)
		self.d.AddControl(EDITCONTROL, Rect(10, 30, 240, aAUTO), "entryField", STYLE_EDIT, "")
		self.d.AddControl(LISTCONTROL, Rect(12, 60, 238, 230), "possibleHits", STYLE_LIST, "")

	def run(self):
		self.d.Run()

	def on_entryField(self, code):
		self.d.GetValue("entryField")
		entry = self.entryField
		count = len(entry)
		possibleHits = [
				i for i in self._fullContent
				if len(i) >= count
				and i[:count] == entry
				]
		possibleHits.sort()
		self.possibleHits = possibleHits
		self.possibleHits_index = 0
		self.d.PutValue("possibleHits")

	def on_ok(self, code):
		self.d.GetValue("possibleHits")
		sel = self.possibleHits_index
		if sel == -1:
			self.selected = None
		else:
			self.selected = self.possibleHits[sel]

	def on_cancel(self, code):
		self.selected = None

class _FontLabDialogTwoFields:
	"""A two field dialog for FontLab. This class should not be called directly. Use the TwoFields function."""

	def __init__(self, title_1, value_1, title_2, value_2, title='RoboFab'):
		self.d = Dialog(self)
		self.d.size = Point(200, 125)
		self.d.title = title
		self.d.Center()
		self.d.AddControl(EDITCONTROL, Rect(120, 10, aIDENT2, aAUTO), "v1edit", STYLE_EDIT, title_1)
		self.d.AddControl(EDITCONTROL, Rect(120, 40, aIDENT2, aAUTO), "v2edit", STYLE_EDIT, title_2)
		self.v1edit = value_1
		self.v2edit = value_2

	def Run(self):
		return self.d.Run()

	def on_cancel(self, code):
		self.v1edit = None
		self.v2edit = None

	def on_ok(self, code):
		self.d.GetValue("v1edit")
		self.d.GetValue("v2edit")
		self.v1 = self.v1edit
		self.v2 = self.v2edit

class _FontLabDialogTwoChecks:
	"""A two check box dialog for FontLab. This class should not be called directly. Use the TwoChecks function."""

	def __init__(self, title_1, title_2, value1=1, value2=1, title='RoboFab'):
		self.d = Dialog(self)
		self.d.size = Point(200, 105)
		self.d.title = title
		self.d.Center()
		self.d.AddControl(CHECKBOXCONTROL, Rect(10, 10, aIDENT2, aAUTO), "check1", STYLE_CHECKBOX, title_1)
		self.d.AddControl(CHECKBOXCONTROL, Rect(10, 30, aIDENT2, aAUTO), "check2", STYLE_CHECKBOX, title_2)
		self.check1 = value1
		self.check2 = value2

	def Run(self):
		return self.d.Run()

	def on_cancel(self, code):
		self.check1 = None
		self.check2 = None

	def on_ok(self, code):
		self.d.GetValue("check1")
		self.d.GetValue("check2")

class _FontLabDialogAskString:
	"""A one simple string prompt dialog for FontLab. This class should not be called directly. Use the GetString function."""

	def __init__(self, message, value, title='RoboFab'):
		self.d = Dialog(self)
		self.d.size = Point(350, 130)
		self.d.title = title
		self.d.Center()
		self.d.AddControl(STATICCONTROL, Rect(aIDENT, aIDENT, aIDENT, aAUTO), "label", STYLE_LABEL, message)
		self.d.AddControl(EDITCONTROL, Rect(aIDENT, 40, aIDENT, aAUTO), "value", STYLE_EDIT, '')
		self.value=value

	def Run(self):
		return self.d.Run()

	def on_cancel(self, code):
		self.value = None

	def on_ok(self, code):
		self.d.GetValue("value")

class _FontLabDialogMessage:
	"""A simple message dialog for FontLab. This class should not be called directly. Use the SimpleMessage function."""

	def __init__(self, message, title='RoboFab'):
		self.d = Dialog(self)
		self.d.size = Point(350, 130)
		self.d.title = title
		self.d.Center()
		self.d.AddControl(STATICCONTROL, Rect(aIDENT, aIDENT, aIDENT, 80), "label", STYLE_LABEL, message)

	def Run(self):
		return self.d.Run()

class _FontLabDialogGetYesNoCancel:
	"""A yes no cancel message dialog for FontLab. This class should not be called directly. Use the YesNoCancel function."""

	def __init__(self, message, title='RoboFab'):
		self.d = Dialog(self)
		self.d.size = Point(350, 130)
		self.d.title = title
		self.d.Center()
		self.d.ok = 'Yes'
		self.d.AddControl(STATICCONTROL, Rect(aIDENT, aIDENT, aIDENT, 80), "label", STYLE_LABEL, message)
		self.d.AddControl(BUTTONCONTROL, Rect(100, 95, 172, 115), "button", STYLE_BUTTON, "No")
		self.value = 0

	def Run(self):
		return self.d.Run()

	def on_ok(self, code):
		self.value = 1

	def on_cancel(self, code):
		self.value = -1

	def on_button(self, code):
		self.value = 0
		self.d.End()

class _MacOneListW:
	"""A one list dialog for Macintosh. This class should not be called directly. Use the OneList function."""

	def __init__(self, list, message='Make a selection'):
		import W
		self.list = list
		self.selected = None
		self.w = W.ModalDialog((200, 240))
		self.w.message = W.TextBox((10, 10, -10, 30), message)
		self.w.list = W.List((10, 35, -10, -50), list)
		self.w.l = W.HorizontalLine((10, -40, -10, 1), 1)
		self.w.cancel = W.Button((10, -30, 87, -10), 'Cancel', self.cancel)
		self.w.ok = W.Button((102, -30, 88, -10), 'OK', self.ok)
		self.w.setdefaultbutton(self.w.ok)
		self.w.bind('cmd.', self.w.cancel.push)
		self.w.open()

	def ok(self):
		if len(self.w.list.getselection()) == 1:
			self.selected = self.w.list.getselection()[0]
			self.w.close()

	def cancel(self):
		self.selected = None
		self.w.close()

class _MacTwoChecksW:
	""" Version using W """

	def __init__(self, title_1, title_2, value1=1, value2=1, title='RoboFab'):
		import W
		self.check1 = value1
		self.check2 = value2
		self.w = W.ModalDialog((200, 100))
		self.w.check1 = W.CheckBox((10, 10, -10, 16), title_1, value=value1)
		self.w.check2 = W.CheckBox((10, 35, -10, 16), title_2, value=value2)
		self.w.l = W.HorizontalLine((10, 60, -10, 1), 1)
		self.w.cancel = W.Button((10, 70, 85, 20), 'Cancel', self.cancel)
		self.w.ok = W.Button((105, 70, 85, 20), 'OK', self.ok)
		self.w.setdefaultbutton(self.w.ok)
		self.w.bind('cmd.', self.w.cancel.push)
		self.w.open()

	def ok(self):
		self.check1 = self.w.check1.get()
		self.check2 = self.w.check2.get()
		self.w.close()

	def cancel(self):
		self.check1 = None
		self.check2 = None
		self.w.close()

class ProgressBar:
	def __init__(self, title='RoboFab...', ticks=0, label=''):
		"""
		A progress bar.
		Availability: FontLab, Mac
		"""
		self._tickValue = 1

		if inFontLab:
			fl.BeginProgress(title, ticks)
		elif MAC and hasEasyDialogs:
			import EasyDialogs
			self._bar = EasyDialogs.ProgressBar(title, maxval=ticks, label=label)
		else:
			_raisePlatformError('Progress')

	def getCurrentTick(self):
		return self._tickValue

	def tick(self, tickValue=None):
		"""
		Tick the progress bar.
		Availability: FontLab, Mac
		"""
		if not tickValue:
			tickValue = self._tickValue

		if inFontLab:
			fl.TickProgress(tickValue)
		elif MAC:
			self._bar.set(tickValue)
		else:
			pass

		self._tickValue = tickValue + 1

	def label(self, label):
		"""
		Set the label on the progress bar.
		Availability: Mac
		"""
		if inFontLab:
			pass
		elif MAC:
			self._bar.label(label)
		else:
			pass

	def close(self):
		"""
		Close the progressbar.
		Availability: FontLab, Mac
		"""
		if inFontLab:
			fl.EndProgress()
		elif MAC:
			del self._bar
		else:
			pass

def SelectFont(message="Select a font:", title='RoboFab'):
	"""
	Returns font instance if there is one, otherwise it returns None.
	Availability: FontLab
	"""
	from robofab.world import RFont
	if inFontLab:
		list = []
		for i in range(fl.count):
			list.append(fl[i].full_name)
		name = OneList(list, message, title)
		if name is None:
			return None
		else:
			return RFont(fl[list.index(name)])
	else:
		_raisePlatformError('SelectFont')

def SelectGlyph(font, message="Select a glyph:", title='RoboFab'):
	"""
	Returns glyph instance if there is one, otherwise it returns None.
	Availability: FontLab
	"""
	from fontTools.misc.textTools import caselessSort

	if inFontLab:
		tl = font.keys()
		list = caselessSort(tl)
		glyphname = OneList(list, message, title)
		if glyphname is None:
			return None
		else:
			return font[glyphname]
	else:
		_raisePlatformError('SelectGlyph')

def FindGlyph(font, message="Search for a glyph:", title='RoboFab'):
	"""
	Returns glyph instance if there is one, otherwise it returns None.
	Availability: FontLab
	"""

	if inFontLab:
		glyphname = SearchList(font.keys(), message, title)
		if glyphname is None:
			return None
		else:
			return font[glyphname]
	else:
		_raisePlatformError('SelectGlyph')

def OneList(list, message="Select an item:", title='RoboFab'):
	"""
	Returns selected item, otherwise it returns None.
	Availability: FontLab, Macintosh
	"""
	if inFontLab:
		ol = _FontLabDialogOneList(list, message)
		ol.Run()
		selected = ol.selected
		if selected is None:
			return None
		else:
			try:
				return list[selected]
			except:
				return None
	elif MAC:
		if hasW:
			d = _MacOneListW(list, message)
			sel = d.selected
			if sel is None:
				return None
			else:
				return list[sel]
		else:
			_raisePlatformError('OneList')
	elif PC:
		_raisePlatformError('OneList')

def SearchList(list, message="Select an item:", title='RoboFab'):
	"""
	Returns selected item, otherwise it returns None.
	Availability: FontLab
	"""
	if inFontLab:
		sl = _FontLabDialogSearchList(list, message, title)
		sl.run()
		selected = sl.selected
		if selected is None:
			return None
		else:
			return selected
	else:
		_raisePlatformError('SearchList')

def TwoFields(title_1="One:", value_1="0", title_2="Two:", value_2="0", title='RoboFab'):
	"""
	Returns (value 1, value 2).
	Availability: FontLab
	"""
	if inFontLab:
		tf = _FontLabDialogTwoFields(title_1, value_1, title_2, value_2, title)
		tf.Run()
		try:
			v1 = tf.v1
			v2 = tf.v2
			return (v1, v2)
		except:
			return None
	else:
		_raisePlatformError('TwoFields')

def TwoChecks(title_1="One",  title_2="Two", value1=1, value2=1, title='RoboFab'):
	"""
	Returns check value:
	1 if check box 1 is checked
	2 if check box 2 is checked
	3 if both are checked
	0 if neither are checked
	None if cancel is clicked.

	Availability: FontLab, Macintosh
	"""
	tc = None
	if inFontLab:
		tc = _FontLabDialogTwoChecks(title_1, title_2, value1, value2, title)
		tc.Run()
	elif MAC:
		if hasW:
			tc = _MacTwoChecksW(title_1, title_2, value1, value2, title)
		else:
			_raisePlatformError('TwoChecks')
	else:
		_raisePlatformError('TwoChecks')
	c1 = tc.check1
	c2 = tc.check2
	if c1 == 1 and c2 == 0:
		return 1
	elif c1 == 0 and c2 == 1:
		return 2
	elif c1 == 1 and c2 == 1:
		return 3
	elif c1 == 0 and c2 == 0:
		return 0
	else:
		return None

def Message(message, title='RoboFab'):
	"""
	A simple message dialog.
	Availability: FontLab, Macintosh
	"""
	if inFontLab:
		_FontLabDialogMessage(message, title).Run()
	elif MAC:
		import EasyDialogs
		EasyDialogs.Message(message)
	else:
		_raisePlatformError('Message')

def AskString(message, value='', title='RoboFab'):
	"""
	Returns entered string.
	Availability: FontLab, Macintosh
	"""
	if inFontLab:
		askString = _FontLabDialogAskString(message, value, title)
		askString.Run()
		v = askString.value
		if v is None:
			return None
		else:
			return v
	elif MAC:
		import EasyDialogs
		askString = EasyDialogs.AskString(message)
		if askString is None:
			return None
		if len(askString) == 0:
			return None
		else:
			return askString
	else:
		_raisePlatformError('GetString')

def AskYesNoCancel(message, title='RoboFab', default=0):
	"""
	Returns 1 for 'Yes', 0 for 'No' and -1 for 'Cancel'.
	Availability: FontLab, Macintosh
	("default" argument only available on Macintosh)
	"""
	if inFontLab:
		gync = _FontLabDialogGetYesNoCancel(message, title)
		gync.Run()
		v = gync.value
		return v
	elif MAC:
		import EasyDialogs
		gync = EasyDialogs.AskYesNoCancel(message, default=default)
		return gync
	else:
		_raisePlatformError('GetYesNoCancel')

def GetFile(message=None):
	"""
	Select file dialog. Returns path if one is selected. Otherwise it returns None.
	Availability: FontLab, Macintosh, PC
	"""
	path = None
	if MAC:
		if haveMacfs:
			fss, ok = macfs.PromptGetFile(message)
			if ok:
				path = fss.as_pathname()
		else:
			from robofab.interface.mac.getFileOrFolder import GetFile
			path = GetFile(message)
	elif PC:
		if inFontLab:
			if not message:
				message = ''
			path = fl.GetFileName(1, message, '', '')
		else:
			openFlags = win32con.OFN_FILEMUSTEXIST|win32con.OFN_EXPLORER
			mode_open = 1
			myDialog = win32ui.CreateFileDialog(mode_open,None,None,openFlags)
			myDialog.SetOFNTitle(message)
			is_OK = myDialog.DoModal()
			if is_OK == 1:
				path = myDialog.GetPathName()
	else:
		_raisePlatformError('GetFile')
	return path

def GetFolder(message=None):
	"""
	Select folder dialog. Returns path if one is selected. Otherwise it returns None.
	Availability: FontLab, Macintosh, PC
	"""
	path = None
	if MAC:
		if haveMacfs:
			fss, ok = macfs.GetDirectory(message)
			if ok:
				path = fss.as_pathname()
		else:
			from robofab.interface.mac.getFileOrFolder import GetFileOrFolder
			# This _also_ allows the user to select _files_, but given the
			# package/folder dichotomy, I think we have no other choice.
			path = GetFileOrFolder(message)
	elif PC:
		if inFontLab:
			if not message:
				message = ''
			path = fl.GetPathName('', message)
		else:
			myTuple = shell.SHBrowseForFolder(0, None, message, 64)
			try:
				path = shell.SHGetPathFromIDList(myTuple[0])
			except:
				pass
	else:
		_raisePlatformError('GetFile')
	return path

GetDirectory = GetFolder

def PutFile(message=None, fileName=None):
	"""
	Save file dialog. Returns path if one is entered. Otherwise it returns None.
	Availability: FontLab, Macintosh, PC
	"""
	path = None
	if MAC:
		if haveMacfs:
			fss, ok = macfs.StandardPutFile(message, fileName)
			if ok:
				path = fss.as_pathname()
		else:
			import EasyDialogs
			path = EasyDialogs.AskFileForSave(message, savedFileName=fileName)
	elif PC:
		if inFontLab:
			if not message:
				message = ''
			if not fileName:
				fileName = ''
			path = fl.GetFileName(0, message, fileName, '')
		else:
			openFlags = win32con.OFN_OVERWRITEPROMPT|win32con.OFN_EXPLORER
			mode_save = 0
			myDialog = win32ui.CreateFileDialog(mode_save, None, fileName, openFlags)
			myDialog.SetOFNTitle(message)
			is_OK = myDialog.DoModal()
			if is_OK == 1:
				path = myDialog.GetPathName()
	else:
		_raisePlatformError('GetFile')
	return path

if __name__=='__main__':
	import traceback

	print "dialogs hasW", hasW
	print "dialogs hasDialogKit", hasDialogKit
	print "dialogs MAC", MAC
	print "dialogs PC", PC
	print "dialogs inFontLab", inFontLab
	print "dialogs hasEasyDialogs", hasEasyDialogs

	def tryDialog(dialogClass, args=None):
		print
		print "tryDialog:", dialogClass, "with args:", args
		try:
			if args is not None:
				apply(dialogClass, args)
			else:
				apply(dialogClass)
		except:
			traceback.print_exc(limit=0)

	tryDialog(TwoChecks, ('hello', 'world', 1, 0, 'ugh'))
	tryDialog(TwoFields)
	tryDialog(TwoChecks, ('hello', 'world', 1, 0, 'ugh'))
	tryDialog(OneList, (['a', 'b', 'c'], 'hello world'))
	tryDialog(Message, ('hello world',))
	tryDialog(AskString, ('hello world',))
	tryDialog(AskYesNoCancel, ('hello world',))

	try:
		b = ProgressBar('hello', 50, 'world')
		for i in range(50):
			if i == 25:
				b.label('ugh.')
			b.tick(i)
		b.close()
	except:
		traceback.print_exc(limit=0)


text:
package org.spongepowered.common.data.manipulator.mutable.common;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import org.spongepowered.api.data.DataContainer;
import org.spongepowered.api.data.DataHolder;
import org.spongepowered.api.data.key.Key;
import org.spongepowered.api.data.manipulator.DataManipulator;
import org.spongepowered.api.data.manipulator.ImmutableDataManipulator;
import org.spongepowered.api.data.merge.MergeFunction;
import org.spongepowered.api.data.value.BaseValue;
import org.spongepowered.api.data.value.immutable.ImmutableValue;
import org.spongepowered.api.data.value.mutable.Value;
import org.spongepowered.common.data.DataProcessor;
import org.spongepowered.common.data.SpongeDataRegistry;
import org.spongepowered.common.data.ValueProcessor;
import org.spongepowered.common.util.GetterFunction;
import org.spongepowered.common.util.SetterFunction;

import java.util.Map;
import java.util.Set;

import javax.annotation.Nullable;

/**
 * Basically, this is the default implementation that automatically delegates
 * ALL actual data processing to either their associated
 * {@link DataProcessor}s or {@link ValueProcessor}s to avoid relying on
 * implementation residing in the actua {@link DataManipulator}s themselves.
 * As all vanilla related manipulators are based on data existing from
 * Minecraft's current implementation (i.e. not an ECS), it is required
 * that all processing exists in the associated processors and not within these
 * {@link DataManipulator}s. Unfortunately, due to JDK 6's poor generic
 * inference calculator, most of the delegating methods to their processors
 * require extra casting to {@link Object} and recasting to the practical
 * {@code M} generics.
 *
 *
Note: It is ABSOLUTELY REQUIRED to {@link #registerKeyValue(Key, GetterFunction)}
 * and {@link #registerFieldGetter(Key, GetterFunction)} and
 * {@link #registerFieldSetter(Key, SetterFunction)} for all possible
 * {@link Key}s and {@link Value}s the {@link DataManipulator} may provide as
 * all of the implementation methods provided here are handled using those. This
 * was done to avoid having to override {@link #getKeys()} and {@link #getValues()},
 * let alone using {@link ValueProcessor}s for simple setters and getters. I believe
 * this to be faster than having to retrieve a processor from a map, checking if
 * the class is an instance of the type implementation to access the setters
 * and getters.

 *
 * @param <M> The generic of the DataManipulator from the API
 * @param  The type of the ImmutableDatAManipulator from the API
 */
@SuppressWarnings("unchecked")
public abstract class AbstractData<M extends DataManipulator<M, I>, I extends ImmutableDataManipulator> implements DataManipulator<M, I> {

    // We need this field for referencing to retrieve the processors as needed. This can never be null
    private final Class<M> manipulatorClass;

    // Ok, so, you're probably asking "Why the hell are you doing this type of hackery?"
    // Answer: Because I'd rather have these abstract functions (read method references)
    // to get and set field values according to the key, and get values based on key
    // instead of using a Value/Data Processor. The advantage of course is that
    // in Java 8, this would all be done with lambda expressions, but since we
    // target Java 6, we can't quite do that... so, this is the compromise.
    //
    // There was a possibility for using annotations, but I (gabizou) decided against
    // it since there's a lot of magic that goes on with it, and there is very little
    // customization when it comes to the method calls for setting/getting the values.
    // The largest issue was implementation. Since most fields are simple to get and
    // set, other values, such as ItemStacks require a bit of finer tuning.
    //
    private final Map<Key<?>, GetterFunction<Value<?>>> keyValueMap = Maps.newHashMap();
    private final Map<Key<?>, GetterFunction<?>> keyFieldGetterMap = Maps.newHashMap();
    private final Map<Key<?>, SetterFunction<Object>> keyFieldSetterMap = Maps.newHashMap();

    protected AbstractData(Class<M> manipulatorClass) {
        this.manipulatorClass = checkNotNull(manipulatorClass);
    }

    /**
     * Simple registration method for the keys to value return methods.
     *
     *
Note that this is still going to be usable, but will be made simpler
     * when Java 8 is used, as lambda expressions can refrence methods. The
     * update won't actually change these registration methods, but the
     * {@link DataManipulator}s calling these registration methods will
     * become single line simplifications.

     *
     * @param key The key for the value return type
     * @param function The function for getting the value
     */
    protected final void registerKeyValue(Key<?> key, GetterFunction<Value<?>> function) {
        this.keyValueMap.put(checkNotNull(key), checkNotNull(function));
    }

    /**
     * Simple registration method for the keys to field getter methods.
     *
     *
Note that this is still going to be usable, but will be made simpler
     * when Java 8 is used, as lambda expressions can refrence methods. The
     * update won't actually change these registration methods, but the
     * {@link DataManipulator}s calling these registration methods will
     * become single line simplifications.

     *
     * @param key The key for the value return type
     * @param function The function for getting the field
     */
    protected final void registerFieldGetter(Key<?> key, GetterFunction<?> function) {
        this.keyFieldGetterMap.put(checkNotNull(key), checkNotNull(function));
    }

    /**
     * Simple registration method for the keys to field setter methods.
     *
     *
Note that this is still going to be usable, but will be made simpler
     * when Java 8 is used, as lambda expressions can refrence methods. The
     * update won't actually change these registration methods, but the
     * {@link DataManipulator}s calling these registration methods will
     * become single line simplifications.

     *
     * @param key The key for the value return type
     * @param function The function for setting the field
     */
    protected final void registerFieldSetter(Key<?> key, SetterFunction<Object> function) {
        this.keyFieldSetterMap.put(checkNotNull(key), checkNotNull(function));
    }

    protected abstract void registerGettersAndSetters();

    @Override
    public Optional<M> fill(DataHolder dataHolder) {
        // Basic stuff, getting the processor....
        final Optional<DataProcessor<M, I>> processor = SpongeDataRegistry.getInstance().getProcessor(this.manipulatorClass);
        if (!processor.isPresent()) {
            return Optional.absent();
        }
        // .... and delegate to the processor!
        return processor.get().fill(dataHolder, copy(), MergeFunction.IGNORE_ALL);
    }

    @Override
    public Optional<M> fill(DataHolder dataHolder, MergeFunction overlap) {
        // Basic stuff, getting the processor....
        final Optional<DataProcessor<M, I>> processor = SpongeDataRegistry.getInstance().getProcessor(this.manipulatorClass);
        if (!processor.isPresent()) {
            return Optional.absent();
        }
        // .... and delegate to the processor!
        return processor.get().fill(dataHolder, copy(), overlap);
    }

    // ---------------

    @Override
    public Optional<M> from(DataContainer container) {
        // Basic stuff, getting the processor....
        final Optional<DataProcessor<M, I>> processor = SpongeDataRegistry.getInstance().getProcessor(this.manipulatorClass);
        if (!processor.isPresent()) {
            return Optional.absent();
        }
        return processor.get().fill(container, (M) (Object) this);
    }

    // Beyond this point is all implementation with the getter/setter functions!

    @Override
    public <E> M set(Key<? extends BaseValue<E>> key, E value) {
        checkArgument(supports(key), "This data manipulator doesn't support the following key: " + key.toString());
        this.keyFieldSetterMap.get(key).set(value);
        return (M) this;
    }

    @Override
    public M set(BaseValue<?> value) {
        checkArgument(supports(value), "This data manipulator doesn't support the following key: " + value.getKey().toString());
        this.keyFieldSetterMap.get(value.getKey()).set(value.get());
        return (M) this;
    }

    @Override
    public M set(BaseValue<?>... values) {
        for (BaseValue<?> value : checkNotNull(values)) {
            try { // Though we should be "argument" aware,
                // so we use the try catch.
                set(value);
            } catch (IllegalArgumentException e) {
                e.printStackTrace(); // print the stacktrace, but we continue
            }
        }
        // finally, return this object, casted for JDK 6's really bad generic inference calculations.
        return (M) (Object) this;
    }

    @Override
    public M set(Iterable<? extends BaseValue<?>> values) { // Basically, the exact same as above...
        for (BaseValue<?> value : checkNotNull(values)) {
            try { // Though we should be "argument" aware,
                  // so we use the try catch.
                set(value);
            } catch (IllegalArgumentException e) {
                e.printStackTrace(); // print the stacktrace, but we continue
            }
        }
        // finally, return this object, casted for JDK 6's really bad generic inference calculations.
        return (M) (Object) this;
    }

    @Override
    public <E> M transform(Key<? extends BaseValue<E>> key, Function<E, E> function) {
        checkArgument(supports(key));
        this.keyFieldSetterMap.get(key).set(checkNotNull(function.apply((E) this.keyFieldGetterMap.get(key).get())));
        return (M) this;
    }

    @Override
    public <E> Optional<E> get(Key<? extends BaseValue<E>> key) {
        if (!supports(key)) {
            return Optional.absent();
        }
        return Optional.of((E) this.keyFieldGetterMap.get(key).get());
    }

    @Nullable
    @Override
    public <E> E getOrNull(Key<? extends BaseValue<E>> key) {
        return get(key).orNull(); // Just use the provided optional
    }

    @Override
    public <E> E getOrElse(Key<? extends BaseValue<E>> key, E defaultValue) {
        return get(key).or(checkNotNull(defaultValue)); // Or use the optional with a default value
    }

    @Override
    public <E, V extends BaseValue<E>> Optional<V> getValue(Key<V> key) {
        if (!this.keyValueMap.containsKey(key)) {
            return Optional.absent();
        }
        return Optional.of((V) checkNotNull(this.keyValueMap.get(key).get()));
    }

    @Override
    public boolean supports(Key<?> key) {
        return this.keyFieldSetterMap.containsKey(checkNotNull(key));
    }

    @Override
    public boolean supports(BaseValue<?> baseValue) {
        return this.keyFieldSetterMap.containsKey(checkNotNull(baseValue.getKey()));
    }

    @Override
    public Set<Key<?>> getKeys() {
        return ImmutableSet.copyOf(this.keyFieldSetterMap.keySet());
    }

    @Override
    public Set<ImmutableValue<?>> getValues() {
        ImmutableSet.Builder<ImmutableValue<?>> builder = ImmutableSet.builder();
        for (GetterFunction<Value<?>> function : this.keyValueMap.values()) {
            builder.add(checkNotNull(function.get()).asImmutable());
        }
        return builder.build();
    }

    @Override
    public int hashCode() {
        return Objects.hashCode(this.manipulatorClass);
    }

    @SuppressWarnings("rawtypes")
    @Override
    public boolean equals(Object obj) {
        if (this == obj) {
            return true;
        }
        if (obj == null || getClass() != obj.getClass()) {
            return false;
        }
        final AbstractData other = (AbstractData) obj;
        return Objects.equal(this.manipulatorClass, other.manipulatorClass);
    }

}