text:
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
      package="com.example.hellojni"
      android:installLocation="preferExternal"
      android:versionCode="1"
      android:versionName="1.0">
    <uses-sdk android:minSdkVersion="3" />
    <application android:label="@string/app_name"
                 android:debuggable="true">
        <activity android:name=".HelloJni"
                  android:label="@string/app_name">
            <intent-filter>
                <action android:name="android.intent.action.MAIN" />
                <category android:name="android.intent.category.LAUNCHER" />
            </intent-filter>
        </activity>
    </application>
</manifest>


text:
def _path_inject(paths):
    import sys
    sys.path[:0] = paths

redpajama_set_name: RedPajamaGithub


text:
var React = require("react")
var PureRenderMixin = require('react/addons').addons.PureRenderMixin
var JSONActions = require('../data/actions/JSONActions');
var cx = require("classnames")

var JSONSearch = React.createClass({
	getInitialState: function  () {
		return {
			showSuggestions: false
		}
	},

	render: function () {
		return (

				<input
					className="json-search-inp"
					onBlur={this.hideSuggestions}
					onChange={this.updateSearchTerm}
					onFocus={this.showSuggestions}
					value={this.props.searchTerm}
				/>

				<JSONSearchButton searchType={this.props.searchType} type="std" title="Search">
					...
				</JSONSearchButton>

				<JSONSearchButton searchType={this.props.searchType} type="regexp" title="Search by regular expression">
					[A-Z]
				</JSONSearchButton>

				<JSONSearchButton searchType={this.props.searchType} type="path" title="Search by path">
					a.b.c
				</JSONSearchButton>

				<JSONSearchOption activeSearchOptions={this.props.searchOptions} option="caseSensitive" title="Case Sensitive">
					Aa
				</JSONSearchOption>

				{ this.state.showSuggestions && this.props.searchSuggestions && this.props.searchSuggestions.length > 0 &&

						{ this.props.searchSuggestions.map(suggestion => (

								{suggestion}

						))}

				}

		)
	},

	autoCompleteSearchTerm: function(suggestion, e){
        JSONActions.updateSearchTerm(suggestion);

        e.stopPropagation();
	},

	hideSuggestions: function (e) {
		this.setState({
			showSuggestions: false
		})

		e.stopPropagation();
	},

	showSuggestions: function (e) {
		this.setState({
			showSuggestions: true
		})

		e.stopPropagation();
	},

    updateSearchTerm: function(e){
        JSONActions.updateSearchTerm(e.target.value);

        e.stopPropagation();
    }
})

var JSONSearchButton = React.createClass({
	render: function(){
		var classes = cx("json-edit-search-btn", {
			"json-edit-search-btn-active": this.props.type == this.props.searchType
		})

		return (
			<button {...this.props} className={classes} onClick={this.setSearchType}>
				{ this.props.children }
			</button>
		)
	},

	setSearchType: function(e){
		JSONActions.setSearchType(this.props.type)

		e.stopPropagation()
	}
})

var JSONSearchOption = React.createClass({
	render: function(){
		var classes = cx("json-edit-search-btn", {
			"json-edit-search-btn-active": this.props.activeSearchOptions[this.props.option]
		})

		return (
			<button {...this.props} className={classes} onClick={this.setSearchOption}>
				{ this.props.children }
			</button>
		)
	},

	setSearchOption: function(e){
		JSONActions.setSearchOption(this.props.option)

		e.stopPropagation()
	}
})

module.exports = JSONSearch;


text:
<?php

namespace Predis\Command\Processor;

use PredisTestCase;

/**
 *
 */
class KeyPrefixProcessorTest extends PredisTestCase
{
    /**
     * @group disconnected
     */
    public function testConstructorWithPrefix()
    {
        $prefix = 'prefix:';
        $processor = new KeyPrefixProcessor($prefix);

        $this->assertInstanceOf('Predis\Command\Processor\CommandProcessorInterface', $processor);
        $this->assertEquals($prefix, $processor->getPrefix());
    }

    /**
     * @group disconnected
     */
    public function testChangePrefix()
    {
        $prefix1 = 'prefix:';
        $prefix2 = 'prefix:new:';

        $processor = new KeyPrefixProcessor($prefix1);
        $this->assertEquals($prefix1, $processor->getPrefix());

        $processor->setPrefix($prefix2);
        $this->assertEquals($prefix2, $processor->getPrefix());
    }

    /**
     * @group disconnected
     */
    public function testProcessPrefixableCommands()
    {
        $prefix = 'prefix:';

        $command = $this->getMock('Predis\Command\PrefixableCommand');
        $command->expects($this->once())
                ->method('prefixKeys')
                ->with($prefix);
        $command->expects($this->once())
                ->method('getArguments')
                ->will($this->returnValue('key'));

        $processor = new KeyPrefixProcessor($prefix);

        $processor->process($command);
    }

    /**
     * @group disconnected
     */
    public function testSkipPrefixableCommandsWithNoArguments()
    {
        $prefix = 'prefix:';

        $command = $this->getMock('Predis\Command\PrefixableCommand');
        $command->expects($this->never())
                ->method('prefixKeys');

        $processor = new KeyPrefixProcessor($prefix);

        $processor->process($command);
    }

    /**
     * @group disconnected
     */
    public function testSkipNotPrefixableCommands()
    {
        $prefix = 'prefix:';
        $unprefixed = 'key';
        $expected = "$prefix$unprefixed";

        $command = $this->getMock('Predis\Command\CommandInterface');
        $command->expects($this->never())->method('prefixKeys');

        $processor = new KeyPrefixProcessor($prefix);

        $processor->process($command);
    }

    /**
     * @group disconnected
     */
    public function testInstanceCanBeCastedToString()
    {
        $prefix = 'prefix:';
        $processor = new KeyPrefixProcessor($prefix);

        $this->assertEquals($prefix, (string) $processor);
    }
}


text:
namespace Ignitus.Option
{
    public abstract class Option<T> where T : notnull
    {
        public static Option<T> Optionify(T item)
        {
            if (item != null)
            {
                return new Some<T>(item);
            }

            return new None<T>();
        }

        public abstract bool IsSome();

        public abstract bool IsNone();

        public abstract T GetOrDefault(T value);
    }

    public class Some<T> : Option<T> where T : notnull
    {
        public T Value { get; private set; }

        public Some(T val)
        {
            Value = val;
        }

        public override bool IsSome()
        {
            return true;
        }

        public override bool IsNone()
        {
            return false;
        }

        public override T GetOrDefault(T value)
        {
            return Value;
        }

        public override bool Equals(object obj)
        {
            bool result = false;

            if (obj is Some<T> option)
            {
                if (option.Value.Equals(Value))
                {
                    result = true;
                }
            }

            return result;
        }

        public override int GetHashCode()
        {
            return Value.GetHashCode();
        }

        public override string ToString()
        {
            return $"Some<{typeof(T).Name}>({Value})";
        }
    }

    public class None<T> : Option<T> where T : notnull
    {
        public None() { }

        public override bool IsNone()
        {
            return true;
        }

        public override bool IsSome()
        {
            return false;
        }

        public override T GetOrDefault(T value)
        {
            return value;
        }

        public override bool Equals(object obj)
        {
            bool result = false;

            if (obj is None<T>)
            {
                result = true;
            }

            return result;
        }

        public override int GetHashCode()
        {
            return 0;
        }

        public override string ToString()
        {
            return $"None";
        }
    }
}


text:
<!DOCTYPE html>

        <meta charset="utf-8" />
        <title>Circle</title>


text:
package de.fau.amos.virtualledger.android.views.savings.add;

import android.content.Intent;
import android.os.Bundle;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
import android.widget.ListView;
import android.widget.TextView;

import java.util.ArrayList;
import java.util.logging.Logger;

import butterknife.ButterKnife;
import butterknife.OnClick;
import de.fau.amos.virtualledger.R;
import de.fau.amos.virtualledger.android.views.contacts.ContactsSupplier;
import de.fau.amos.virtualledger.android.views.contacts.add.AddContactsActivity;
import de.fau.amos.virtualledger.android.views.shared.transactionList.DataListening;
import de.fau.amos.virtualledger.dtos.Contact;
import de.fau.amos.virtualledger.dtos.SavingsAccount;

public class AddSavingsAccountAssignPeopleFragment extends AddSavingsAccountPage implements DataListening {
    @SuppressWarnings("unused")
    private static final String TAG = AddSavingsAccountAssignPeopleFragment.class.getSimpleName();
    private PeopleAssignedListener peopleAssignetListener;
    private SavingsAccount dataModel;
    private ContactsSupplier contactSupplier;
    private PeopleAdapter adapter;
    private Contact meContact = new Contact("", "Me", "");

    @Override
    public void consumeDataModel(SavingsAccount account) {
        this.dataModel = account;
        if (this.peopleAssignetListener != null) {
            this.peopleAssignetListener.updateText();
        }
    }

    @Nullable
    @Override
    public View onCreateView(final LayoutInflater inflater, @Nullable final ViewGroup container, @Nullable final Bundle savedInstanceState) {
        final View view = inflater.inflate(R.layout.saving_accounts_add_friends, container, false);
        ButterKnife.bind(this, view);
        this.contactSupplier = new ContactsSupplier(getActivity());
        this.contactSupplier.onResume();
        contactSupplier.addDataListeningObject(this);

        ListView peopleList = (ListView) view.findViewById(R.id.list);

        TextView conclusionTextView = (TextView) view.findViewById(R.id.conclusion_text);

        this.peopleAssignetListener = new PeopleAssignedListener(conclusionTextView, conclusionTextView.getText().toString(), this.dataModel);
        adapter = new PeopleAdapter(getActivity(), R.id.list, getContacts(), peopleAssignetListener);
        peopleList.setAdapter(adapter);
        return view;
    }

    @NonNull
    private ArrayList<Contact> getContacts() {
        ArrayList<Contact> allContacts = new ArrayList<>();
        allContacts.add(meContact);
        allContacts.addAll(contactSupplier.getAll());
        return allContacts;
    }

    @Override
    public void fillInData(final SavingsAccount savingsAccount) {
        logger().info("Contacts already synced into data model:" + dataModel.getAdditionalAssignedUsers());
    }

    private Logger logger() {
        return Logger.getLogger(this.getClass().getCanonicalName());
    }

    @OnClick(R.id.add_contacts)
    public void onClickAddContacts() {
        logger().info("Add contacts called");
        Intent addContactsIntent = new Intent(getActivity(), AddContactsActivity.class);
        startActivity(addContactsIntent);
    }

    @Override
    public void notifyDataChanged() {
        logger().info("Data-Set changed");
        if (this.adapter == null) {
            return;
        }
        logger().info("Refreshing data");
        this.adapter.clear();
        this.adapter.addAll(getContacts());
        this.adapter.notifyDataSetChanged();
    }

    @Override
    public void onResume() {
        logger().info("Resuming");
        super.onResume();
        this.contactSupplier.addDataListeningObject(this);
    }

    @Override
    public void onPause() {
        logger().info("on Pause");
        super.onPause();
        this.contactSupplier.deregister(this);
    }
}


text:
title: "Singapore School Locator App v1"
categories:
  - projects
tags:
  - R
  - Shiny
  - data.gov.sg
  - geospatial
  - googleVis
  - leaflet
  - DT
comments: true
excerpt: "Shiny application for locating schools in Singapore (with inclusive features)."
modified: 2017-03-24T10:01:43-04:00
---
[Application Github Repo]({{ "https://github.com/ryzalk/singapore-school-locator" }}){: .btn .btn--info}{: .align-center}

{% include toc %}

## Introduction
This Shiny application allows users to locate schools (Primary, Secondary or SPED) in Singapore that are near to their submitted location (through postal code). How this app is distinct from existing applications like [OneMap](https://www.onemap.sg), [Elite](https://elite.com.sg/), or undergraduate projects like [iSchool](http://aaronmak.github.io/ischool/) is that this app takes into account different or special educational needs: visual impairment, physically handicapped or autism, just to name some.

![app-screenshot-1]({{ site.url }}{{ site.baseurl }}/assets/images/singapore-school-locator-ss1.png){: .align-center}

## For Inclusiveness
The motivation behind this project is the increasing need for Singapore to be a more inclusive society. From 2019 onwards, the Compulsory Education Act [will include all children with special needs](http://www.channelnewsasia.com/news/singapore/compulsory-education-for-all-special-needs-children-from-2019/3260830.html). Hence, there's greater relevance for parents or guardians of children with special needs to seek and research on schools that are able to cater to their child or kin. The MOE's website is able to provide with information to schools that are relevant (as [such](https://www.moe.gov.sg/education/special-education/list-of-sped-schools)). However, more can be done to assist these people in providing the best for their kin.

## Usage of Application
To use the application, a potential user requires the installation of [R](https://www.r-project.org/). The following packages are required to be installed as well for the application to be run:

|shiny|googleVis|shinythemes|geosphere|ggmap|
|:-:|:-:|:-:|:-:|:-:|
|data.table|plyr|DT|leaflet|shinyjs|

All the aforementioned packages can be installed using the following command:
```r
install.packages(c("shiny", "googleVis", "shinythemes",
                   "geosphere", "ggmap", "data.table", "plyr",
                   "DT", "leaflet", "shinyjs"))
```
To run the app from R, execute the following:
```r
runGitHub("ryzalk/singapore-school-locator", subdir = "singapore-school-locator")
```
The video below provides a demonstration for using the application.
<iframe width="640" height="360" src="https://www.youtube.com/embed/eqbCSVA4J7Q" frameborder="0" allowfullscreen></iframe>

Currently, application would show __up to 10 school markers__ on map visualisations but data table still shows every school fitted with the user's criteria.

## Application Details
_Coming soon!_

## Room for Improvements / Coming Soon
1. Validation of user inputs.
2. Range slider for number of school markers desired by user.
2. User location marker for Google Maps visualisation.
3. More criteria features e.g. academic streams like Express or Normal (Academic).
4. Addition of school website links (tooltip/data table).
5. Deployment of application through Shiny Server.

## Data Source
_Coming soon!_


text:
package client

import (
	"encoding/json"
	"fmt"
	"github.com/etsy/hound/config"
	"github.com/etsy/hound/index"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

type Response struct {
	Results map[string]*index.SearchResponse
	Stats   *struct {
		FilesOpened int
		Duration    int
	} `json:",omitempty"`
}

type Presenter interface {
	Present(
		re *regexp.Regexp,
		ctx int,
		repos map[string]*config.Repo,
		res *Response) error
}

type Config struct {
	HttpHeaders map[string]string `json:"http-headers"`
	Host        string            `json:"host"`
}

// Extract a repo name from the given url.
func repoNameFromUrl(uri string) string {
	ax := strings.LastIndex(uri, "/")
	if ax < 0 {
		return ""
	}

	name := uri[ax+1:]
	if strings.HasSuffix(name, ".git") {
		name = name[:len(name)-4]
	}

	bx := strings.LastIndex(uri[:ax-1], "/")
	if bx < 0 {
		return name
	}

	return fmt.Sprintf("%s/%s", uri[bx+1:ax], name)
}

// Find the proper name for the given repo using the map of repo
// information.
func repoNameFor(repos map[string]*config.Repo, repo string) string {
	data := repos[repo]
	if data == nil {
		return repo
	}

	name := repoNameFromUrl(data.Url)
	if name == "" {
		return repo
	}

	return name
}

func doHttpGet(cfg *Config, uri string) (*http.Response, error) {
	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		return nil, err
	}

	for key, val := range cfg.HttpHeaders {
		if strings.ToLower(key) == "host" {
			req.Host = val
		} else {
			req.Header.Set(key, val)
		}
	}

	var c http.Client
	return c.Do(req)
}

// Executes a search on the API running on host.
func Search(r *Response, cfg *Config, pattern, repos, files string, context int, ignoreCase, stats bool) error {
	u := fmt.Sprintf("http://%s/api/v1/search?%s",
		cfg.Host,
		url.Values{
			"q":     {pattern},
			"repos": {repos},
			"files": {files},
			"ctx":   {fmt.Sprintf("%d", context)},
			"i":     {fmt.Sprintf("%t", ignoreCase)},
			"stats": {fmt.Sprintf("%t", stats)},
		}.Encode())

	res, err := doHttpGet(cfg, u)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return fmt.Errorf("Status %d", res.StatusCode)
	}

	return json.NewDecoder(res.Body).Decode(r)
}

// Load the list of repositories from the API running on host.
func LoadRepos(repos map[string]*config.Repo, cfg *Config) error {
	res, err := doHttpGet(cfg, fmt.Sprintf("http://%s/api/v1/repos", cfg.Host))
	if err != nil {
		return err
	}
	defer res.Body.Close()

	return json.NewDecoder(res.Body).Decode(&repos)
}

// Execute a search and load the list of repositories in parallel on the host.
func SearchAndLoadRepos(cfg *Config, pattern, repos, files string, context int, ignoreCase, stats bool) (*Response, map[string]*config.Repo, error) {
	chs := make(chan error)
	var res Response
	go func() {
		chs <- Search(&res, cfg, pattern, repos, files, context, ignoreCase, stats)
	}()

	chr := make(chan error)
	rep := map[string]*config.Repo{}
	go func() {
		chr <- LoadRepos(rep, cfg)
	}()

	// must ready both channels before returning to avoid routine/channel leak.
	errS, errR := <-chs, <-chr

	if errS != nil {
		return nil, nil, errS
	}

	if errR != nil {
		return nil, nil, errR
	}

	return &res, rep, nil
}


text:
from __future__ import unicode_literals

from django.db import models, migrations
from apps.plea.standardisers import standardise_urn

def standardise_urns(apps, schema_editor):
    Case = apps.get_model("plea", "Case")
    for case in Case.objects.all():
        case.urn = standardise_urn(case.urn)
        case.save()

class Migration(migrations.Migration):

    dependencies = [
        ('plea', '0021_auto_20151208_1619'),
    ]

    operations = [
        migrations.RunPython(standardise_urns)
    ]


text:
        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
        <title>W29641_text</title>
        <link rel="stylesheet" type="text/css" href="style.css" />

                «

    <p class="styleSans6.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">

    <p class="styleSans648.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">m no mar, me.
I. - oo- Iwoa. Wdl bastion. LYLA aux—w. located on them In 0 - mo in NM 0.. 5: 1/4 sw 1/4 of Stella! 10. man. 395w. A - scum comm: Loam. 5th P.M.. mum. county. North Dakota.
WE'LL LOCATION PLAT 7758M R957, 5th RM.
man)? - new
teALl
M'IICAL calm DAWN: NAM
ATION
E E Q I I: E
M II
mum!) - Ammu- mama) mang- Imw 01042.1) um - "new «mu-7) mm I macaw 103mm)
iwmmwunurwm
I 1mm" M1 fifeflu'v'auufifi

    <p class="styleSans12000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">

    <p class="styleSans12000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">

    <p class="styleSans23.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">

    <p class="styleSans12.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">

    <p class="styleSans215.0000<enum PANGO_WEIGHT_NORMAL of type PangoWeight><enum PANGO_STYLE_NORMAL of type PangoStyle>">


text:
package hello

import "fmt"

func Name()  {
	fmt.Println("Go")
}


text:
package org.ftc8702.opmodes;

import com.qualcomm.hardware.bosch.BNO055IMU;
import com.qualcomm.robotcore.hardware.DcMotor;
import com.qualcomm.robotcore.hardware.HardwareMap;
import com.qualcomm.robotcore.hardware.Servo;

import org.ftc8702.opmodes.ultimategoal.UltimateGoalConfiguration;

import ftcbootstrap.RobotConfiguration;
import ftcbootstrap.components.utils.TelemetryUtil;

public class
MecanumConfig extends RobotConfiguration {

    public static final String MOTOR_FR = "wheelFR";
    public static final String MOTOR_FL = "wheelFL";
    public static final String MOTOR_BR = "wheelBR";
    public static final String MOTOR_BL = "wheelBL";
    public DcMotor motorFR;
    public DcMotor motorFL;
    public DcMotor motorBR;
    public DcMotor motorBL;
    public BNO055IMU imu;

    public static final String LIFTER = "lifter";
    public static final String ARM_LEFT= "armLeft";
    public static final String ARM_RIGHT = "armRight";
    public Servo lifter;
    public Servo armLeft;
    public Servo armRight;

    protected static MecanumConfig newConfig(HardwareMap hardwareMap, TelemetryUtil telemetryUtil) {
        MecanumConfig config = new MecanumConfig();
        config.init(hardwareMap, telemetryUtil);
        return config;
    }
    @Override
    protected void init(HardwareMap hardwareMap, TelemetryUtil telemetryUtil) {

        setTelemetry(telemetryUtil);

        motorFR = (DcMotor) getHardwareOn(MOTOR_FR, hardwareMap.dcMotor);
        motorFL = (DcMotor) getHardwareOn(MOTOR_FL, hardwareMap.dcMotor);
        motorBR = (DcMotor) getHardwareOn(MOTOR_BR, hardwareMap.dcMotor);
        motorBL = (DcMotor) getHardwareOn(MOTOR_BL, hardwareMap.dcMotor);

        armLeft = (Servo) getHardwareOn(ARM_LEFT, hardwareMap.servo);
        armRight = (Servo) getHardwareOn(ARM_RIGHT, hardwareMap.servo);
        armLeft = (Servo) getHardwareOn(LIFTER, hardwareMap.servo);
        imu = hardwareMap.get(BNO055IMU.class, "imu");
    }
}


text:
"""Text block"""

from typing import Union, cast, List, Optional, Iterable

BlockType = Union["Block", str]
BlockConvertible = Union[BlockType, List[BlockType]]

class Block:
    """Represents a block of code. See the `__init__` method documentation for
    more information about the block properties.

    Blocks are composed of list of lines or other blocks. Blocks can be
    indented or decorated by suffixes or prefixes. Example use:

    .. code-block::

        body = Block(indent=4)
        body += "do_this()"
        body += "do_that()"

        block = Block()
        block += "begin"
        block += body
        block += "end"

    Which renders as:

    .. code-block::

        begin
            do_this()
            do_that()
        end
    """

    children: List[BlockType]
    indent: int
    prefix: Optional[str]
    suffix: Optional[str]
    first_prefix: Optional[str]
    first_indent: Optional[int]
    last_suffix: Optional[str]

    def __init__(self, block: Optional[BlockConvertible]=None,
            indent: int=0,
            prefix: Optional[str]=None,
            suffix: Optional[str]=None,
            first_prefix: Optional[str]=None,
            first_indent: Optional[int]=None,
            last_suffix: Optional[str]=None) -> None:
        """Create a code block. Arguments:

        * `block` – content of the block, either another block or a string.
        * `indent` – number of spaces before each line of the block body
        * `prefix` – string prepended to every line of the block after adding
          indentation padding.
        * `suffix` – string appended to every line of the block
        * `first_prefix` – if specified, then prepended to the first line of
          the block instead of the common `prefix`
        * `first_indent` – if specified, then it is used for padding of the
          first line of the block instead of the common indent.
        * `last_suffix` – if specified, then appended to the last line of the
          block instead of the common `suffix`

        """

        self.indent = indent
        self.prefix = prefix
        self.suffix = suffix
        self.first_indent = first_indent
        self.first_prefix = first_prefix
        self.last_suffix = last_suffix

        if not block:
            self.children = []
        elif isinstance(block, (Block, str)):
            self.children = [block]
        elif isinstance(block, list):
            self.children = cast(List[BlockType], block)
        else:
            raise Exception("Invalid block type: {}".format(type(block)))

    def __iadd__(self, block:BlockType) -> "Block":
        if not isinstance(block, (Block, str)):
            raise Exception("Invalid block type: {}".format(type(block)))
        self.children.append(block)
        return self

    def lines(self) -> Iterable[str]:
        """Iterate over lines of the block"""

        common_padding = " " * self.indent
        if self.first_indent is None:
            first_padding = common_padding
        else:
            first_padding = " " * self.first_indent

        common_prefix = self.prefix or ""
        if self.first_prefix is None:
            first_prefix = common_prefix
        else:
            first_prefix = self.first_prefix

        common_suffix = self.suffix or ""
        if self.last_suffix is None:
            last_suffix = common_suffix
        else:
            last_suffix = self.last_suffix

        # We gather all the lines, since we need to know which one is the last
        # one
        lines: List[str] = []

        for child in self.children:
            if isinstance(child, str):
                lines.append(child)
            else:
                for line in child.lines():
                    lines.append(line)

        count = len(lines)

        # Print-out the lines
        for i, line in enumerate(lines):
            prefix = common_prefix
            padding = common_padding
            suffix = common_suffix

            if i == 0:
                prefix = first_prefix
                padding = first_padding
            elif i >= count - 1:
                suffix = last_suffix

            yield padding + prefix + line + suffix

    def to_string(self, indent:int=0) -> str:
        """Return block as string with indent `indent`"""

        lines = self.lines()

        if self.suffix:
            lines = [line + self.suffix for line in lines]
        return "\n".join(lines)

    def __str__(self) -> str:
        return self.to_string(indent=0)


text:
namespace Headless
{
    using System;
    using System.Collections.Generic;
    using System.Collections.ObjectModel;
    using System.Globalization;
    using System.Linq;
    using System.Runtime.Serialization;
    using System.Xml.XPath;
    using Headless.Activation;

    /// <summary>
    ///     The <see cref="InvalidHtmlElementException" />
    ///     is used to identify that an HTML element is not supported in the current context.
    /// </summary>
    [Serializable]
    public class InvalidHtmlElementException : Exception
    {
        /// <summary>
        ///     Stores the html node relate to the exception.
        /// </summary>
        [NonSerialized]
        private readonly IXPathNavigable _node;

        /// <summary>
        ///     Stores the tags supported by the element.
        /// </summary>
        [NonSerialized]
        private readonly IEnumerable<SupportedTagAttribute> _supportedTags;

        /// <summary>
        ///     Initializes a new instance of the <see cref="InvalidHtmlElementException" /> class.
        /// </summary>
        public InvalidHtmlElementException()
        {
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="InvalidHtmlElementException"/> class.
        /// </summary>
        /// <param name="node">
        /// The node.
        /// </param>
        /// <param name="supportedTags">
        /// The supported tags.
        /// </param>
        public InvalidHtmlElementException(IXPathNavigable node, ReadOnlyCollection<SupportedTagAttribute> supportedTags)
            : this(BuildSupportedTagsMessage(node, supportedTags))
        {
            _node = node;
            _supportedTags = supportedTags;
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="InvalidHtmlElementException"/> class.
        /// </summary>
        /// <param name="message">
        /// The message that describes the error.
        /// </param>
        public InvalidHtmlElementException(string message) : base(message)
        {
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="InvalidHtmlElementException"/> class.
        /// </summary>
        /// <param name="message">
        /// The message.
        /// </param>
        /// <param name="inner">
        /// The inner.
        /// </param>
        public InvalidHtmlElementException(string message, Exception inner) : base(message, inner)
        {
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="InvalidHtmlElementException"/> class.
        /// </summary>
        /// <param name="info">
        /// The <see cref="T:System.Runtime.Serialization.SerializationInfo"/> that holds the serialized object
        ///     data about the exception being thrown.
        /// </param>
        /// <param name="context">
        /// The <see cref="T:System.Runtime.Serialization.StreamingContext"/> that contains contextual
        ///     information about the source or destination.
        /// </param>
        protected InvalidHtmlElementException(SerializationInfo info, StreamingContext context) : base(info, context)
        {
        }

        /// <summary>
        /// Builds the supported tags message.
        /// </summary>
        /// <param name="node">
        /// The node.
        /// </param>
        /// <param name="tags">
        /// The tags.
        /// </param>
        /// <returns>
        /// A <see cref="string"/> value.
        /// </returns>
        private static string BuildSupportedTagsMessage(IXPathNavigable node, IEnumerable<SupportedTagAttribute> tags)
        {
            var supportedTags = tags.Select(x => x.ToString()).Aggregate((i, j) => i + Environment.NewLine + j);

            var navigator = node.GetNavigator();

            var message = string.Format(
                CultureInfo.CurrentCulture,
                "The specified '{0}' element is invalid. The supported tags for this node are: {1}",
                navigator.Name,
                supportedTags);

            return message;
        }

        /// <summary>
        ///     Gets the node.
        /// </summary>
        /// <value>
        ///     The node.
        /// </value>
        public IXPathNavigable Node
        {
            get
            {
                return _node;
            }
        }

        /// <summary>
        ///     Gets the supported tags.
        /// </summary>
        /// <value>
        ///     The supported tags.
        /// </value>
        public IEnumerable<SupportedTagAttribute> SupportedTags
        {
            get
            {
                return _supportedTags;
            }
        }
    }
}


text:
<?xml version="1.0" encoding="utf-8"?>
<resources>
    <item name="white" type="color">#FFF5F5F5</item>
</resources>


text:
import DetectOS from 'detectos.js';

const Detect: DetectOS = new DetectOS();
const os: string | null = Detect.OS;
const browser: string | null = Detect.OS;
const version: string | null = Detect.version;


text:
package rs.elfak.mosis.marko.discounthunt.api.endpoints;

/**
 * Created by marko on 7/2/16.
 */
public class FriendshipEndpoint extends BaseEndpoint {
    public FriendshipEndpoint() {
        super("/friendships", "friendship");
    }
}


text:
class Solution {
public:
    bool pyramidTransition(string bottom, vector<string>& allowed) {
        map<string, vector<char>> poss;
        map<string, bool> vis;
        // Put the allowed combinations into map
        for(int i = 0; i < allowed.size(); i++) poss[allowed[i].substr(0,2)].push_back(allowed[i][2]);
        return dfs(bottom, poss, vis);
    }

    bool dfs(string bottom, map<string, vector<char>>& poss, map<string, bool>& vis){
        if(vis[bottom]) return false;
        if(bottom.length() == 1) return true;
        vis[bottom] = true;
        vector<string> nextl;
        string s = "";
        nextl.push_back(s);
        // construct all the possible strings for the next level
        for(int i = 0; i < bottom.length() - 1; i++){
            int ncur_size = nextl.size();
            string cur_comb = bottom.substr(i,2);
            for(int j = 0; j < poss[cur_comb].size(); j++){
                for(int k = 0; k < ncur_size; k++){
                    nextl.push_back(nextl[k] + poss[cur_comb][j]);
                }
            }
            nextl.erase(nextl.begin(), nextl.begin() + ncur_size);
        }

        // try all the possible strings
        for(int i = 0; i < nextl.size(); i++){
            if(dfs(nextl[i],poss,vis)) return true;
        }
        return false;
    }
};


text:
- [MIGRATE from Mandrill IMMEDIATELY](https://github.com/kshnurov/mandrill_dm/blob/master/MIGRATE)

### 1.3.7 (2020-03-23)

- [#66](https://github.com/spovich/mandrill_dm/pull/66) Switch to security-patched fork of mandrill-api gem (mandrill-api-json). Patches CVE-2020-10663.

### 1.3.6 (2018-10-19)

- [#63](https://github.com/spovich/mandrill_dm/pull/63) Adopt Mail::Field#unparsed_value public API, thanks @tensho

### 1.3.5 (2017-12-30)

- [#60](https://github.com/spovich/mandrill_dm/pull/60) Add support for mail 2.7, thanks @tensho

### 1.3.4 (2017-05-28)

- [#55](https://github.com/spovich/mandrill_dm/pull/55) Set default ip pool and override with each message, thanks @dorongutman

### 1.3.3 (2017-02-09)

- [#52](https://github.com/spovich/mandrill_dm/pull/52) Mail gem should be a runtime dependency, thanks @tensho
- [#51](https://github.com/spovich/mandrill_dm/pull/51) Send all passed headers to Mandrill, thanks @tensho
- [#49](https://github.com/spovich/mandrill_dm/pull/49) Update readme for use of mandrill templates, thanks @januszm
- [#48](https://github.com/spovich/mandrill_dm/pull/48) Merge settings and options (multiple api keys), thanks @tensho

### 1.3.2 (2016-06-21)

- [#45](https://github.com/spovich/mandrill_dm/pull/45) fix issue with html only email as plain text, thanks @sysqa

### 1.3.1 (2016-04-25)

- [#43](https://github.com/spovich/mandrill_dm/pull/43) add send_at support, thanks @sbauch, @kshnurov
- [#42](https://github.com/spovich/mandrill_dm/pull/42) add metadata support, thanks @sbauch
- [#40](https://github.com/spovich/mandrill_dm/pull/40) drop official support for 1.9.3, ymmv
- [#39](https://github.com/spovich/mandrill_dm/pull/39) fix sending of text-only emails, thanks @piotrze
- [#26](https://github.com/spovich/mandrill_dm/pull/26) add mandrill template support, thanks @genaromadrid
- [#32](https://github.com/spovich/mandrill_dm/pull/32) fix compatibility with `mail` gem >= 2.5.x (rails 3.2+), thanks @sysqa
- [#23](https://github.com/spovich/mandrill_dm/pull/23) global_merge_vars support, thanks @sysqa
- [#7](https://github.com/spovich/mandrill_dm/pull/7) asynchronous message support, thanks @beorc
- [#22](https://github.com/spovich/mandrill_dm/pull/22) merge_vars support, thanks @tomasc
- [#20](https://github.com/spovich/mandrill_dm/pull/20) Inline image attachments support, thanks @ropiku

### 1.2.0 (2015-03-31)

 - [#2](https://github.com/spovich/mandrill_dm/pull/2) Attachment support
 - [#3](https://github.com/spovich/mandrill_dm/pull/3) Add tags and headers to mandrill_dm
 - [#6](https://github.com/spovich/mandrill_dm/pull/6) Update `MandrillDm::DeliveryMethod` to capture the response from the Mandrill API
 - Add travis-ci to builds
 - [#14](https://github.com/spovich/mandrill_dm/pull/14) Add `merge` and `merge_language` flags


text:
import axios from 'axios';
import { apiRoot } from 'common/injectGlobals';

let token;

const ajax = axios.create({ baseURL: apiRoot });

export const getToken = () => token;

export const setToken = t => {
  token = t;
  ajax.defaults.headers.common['Authorization'] = `Bearer ${t}`;
};

export default ajax;


text:
<?xml version="1.0" encoding="UTF-8"?>

<mule xmlns:json="http://www.mulesoft.org/schema/mule/json" xmlns:vm="http://www.mulesoft.org/schema/mule/vm" xmlns:http="http://www.mulesoft.org/schema/mule/http"
	xmlns:jms="http://www.mulesoft.org/schema/mule/jms" xmlns="http://www.mulesoft.org/schema/mule/core"
	xmlns:doc="http://www.mulesoft.org/schema/mule/documentation"
	xmlns:spring="http://www.springframework.org/schema/beans" version="CE-3.5.0"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-current.xsd
http://www.mulesoft.org/schema/mule/core http://www.mulesoft.org/schema/mule/core/current/mule.xsd
http://www.mulesoft.org/schema/mule/jms http://www.mulesoft.org/schema/mule/jms/current/mule-jms.xsd
http://www.mulesoft.org/schema/mule/http http://www.mulesoft.org/schema/mule/http/current/mule-http.xsd
http://www.mulesoft.org/schema/mule/vm http://www.mulesoft.org/schema/mule/vm/current/mule-vm.xsd
http://www.mulesoft.org/schema/mule/json http://www.mulesoft.org/schema/mule/json/current/mule-json.xsd">
	<jms:activemq-connector name="Active_MQ"
		specification="1.1" username="admin" password="admin" brokerURL="tcp://machine:61616"
		validateConnections="false" doc:name="Active MQ" persistentDelivery="true">
		<reconnect frequency="20000" />
	</jms:activemq-connector>

	<flow name="httptojmsFlow-localhost:4442">
		<http:inbound-endpoint exchange-pattern="request-response"
			host="0.0.0.0" port="${startport}" doc:name="HTTP" />
		<http:outbound-endpoint exchange-pattern="request-response"
			method="GET" responseTimeout="1000000" followRedirects="true"
			doc:name="Autobot-Retrieve-Http-Content" address="${httpendpoint}" />
		<byte-array-to-string-transformer
			doc:name="Byte Array to String" />
        <set-variable variableName="cname" value="${cname}" doc:name="CustomerName1"/>
        <set-variable variableName="count" value="#[xpath('count(//event)')] " doc:name="count"/>
        <logger message="CNAME  ** ${cname} *** xpath prop *** ${xpathcount}" level="INFO" doc:name="Logger"/>
        <scatter-gather doc:name="Scatter-Gather">
            <processor-chain>
                <jms:outbound-endpoint queue="${amqhttp}" connector-ref="Active_MQ" doc:name="JMS-ab.out"/>
                <logger message="through the ab queue" level="INFO" doc:name="Logger"/>
            </processor-chain>
            <processor-chain>
                <foreach collection="#[xpath('//root/events/event')]" doc:name="For Each">
                    <component class="Pause" doc:name="Pause"/>
                    <logger message="XML lines *** #[xpath('title/text()').text]" level="INFO" doc:name="Logger"/>
	<!-- xpath('//XML/Result/Surveys/element') -->
                  <!--   <set-payload value="Event title**** #[payload.title]" doc:name="Logger information"/> -->
                    <jms:outbound-endpoint queue="diff" connector-ref="Active_MQ" doc:name="JMS-diff"/>
                </foreach>
                <logger message="payload for logger " level="INFO" doc:name="Logger"/>
            </processor-chain>
            <processor-chain>
                <set-payload value="#[server.dateTime.getDate().toString()] *** Total Event Count *** #[flowVars['count'].substring(0,flowVars['count'].lastIndexOf('.'))]  *** Customer Name *** #[flowVars.cname]" doc:name="Logger information"/>
                <jms:outbound-endpoint queue="log" connector-ref="Active_MQ" doc:name="JMS-log"/>
                <logger message="payload for logger" level="INFO" doc:name="Logger"/>
            </processor-chain>
        </scatter-gather>
        <logger message="POST result *** " level="INFO" doc:name="Logger"/>
        <catch-exception-strategy doc:name="Catch Exception Strategy">
            <logger message="#[exception.exceptions]" level="ERROR" doc:name="HttpToJMS Exception"/>
        </catch-exception-strategy>
	</flow>
</mule>


text:
namespace tastefulserver {

ByteStream::ByteStream(const QString & linebreak)
: m_linebreak(linebreak)
, m_pos(0)
{
}

ByteStream::ByteStream(const QByteArray & bytes, const QString & linebreak)
: m_buffer(bytes)
, m_linebreak(linebreak)
, m_pos(0)
{
}

ByteStream ByteStream::forLinebreak(const QString & linebreak)
{
    return ByteStream(QByteArray(), linebreak);
}

void ByteStream::append(const QByteArray & bytes)
{
    m_buffer.append(bytes);
}

void ByteStream::flush()
{
    m_buffer = m_buffer.remove(0, m_pos);
    m_pos = 0;
}

QByteArray ByteStream::alreadyRead() const
{
    return m_buffer.mid(0, m_pos);
}

bool ByteStream::canReadLine() const
{
    return m_buffer.indexOf(m_linebreak, m_pos)>=0;
}

int ByteStream::availableBytes() const
{
    return m_buffer.size() - m_pos;
}

char ByteStream::readByte()
{
    if (atEnd())
        return 0;

    char c = m_buffer[m_pos];
    ++m_pos;

    return c;
}

QByteArray ByteStream::read(int length)
{
    if (availableBytes()<length)
    {
        m_pos = m_buffer.size();

        return m_buffer.mid(m_pos);
    }
    QByteArray bytes = m_buffer.mid(m_pos, length);
    m_pos += length;

    return bytes;
}

void ByteStream::skip(int length)
{
    if (availableBytes()<length)
    {
        m_pos = m_buffer.size();

        return;
    }
    m_pos += length;
}

QString ByteStream::readLine()
{
    int p = m_buffer.indexOf(m_linebreak, m_pos);

    if (p<0)
    {
        return readAll();
    }
    QString line = QString(m_buffer.mid(m_pos, p - m_pos));
    m_pos = p + m_linebreak.size();

    return line;
}

bool ByteStream::canReadUpTo(const QString & delimiter) const
{
    return m_buffer.indexOf(delimiter, m_pos)>=0;
}

bool ByteStream::canReadUpTo(const QChar & delimiter) const
{
    return m_buffer.indexOf(delimiter, m_pos)>=0;
}

void ByteStream::skipUpTo(const QString & delimiter)
{
    int p = m_buffer.indexOf(delimiter, m_pos);

    if (p<0)
    {
        skipAll();
    }
    else
    {
        m_pos = p;
    }
}

void ByteStream::skipUpTo(const QChar & delimiter)
{
    int p = m_buffer.indexOf(delimiter, m_pos);

    if (p<0)
    {
        skipAll();
    }
    else
    {
        m_pos = p;
    }
}

void ByteStream::skipBehind(const QString & delimiter)
{
    skipUpTo(delimiter);
    skip(delimiter.size());
}

void ByteStream::skipBehind(const QChar & delimiter)
{
    skipUpTo(delimiter);
    skip(1);
}

QByteArray ByteStream::readUpTo(const QString & delimiter, bool skipDelimiter)
{
    int p = m_buffer.indexOf(delimiter, m_pos);

    if (p<0)
    {
        return readAll();
    }
    QByteArray part = m_buffer.mid(m_pos, p - m_pos);
    m_pos = p;
    if (skipDelimiter)
    {
        skip(delimiter.size());
    }

    return part;
}

QByteArray ByteStream::readUpTo(const QChar & delimiter, bool skipDelimiter)
{
    int p = m_buffer.indexOf(delimiter, m_pos);

    if (p<0)
    {
        return readAll();
    }
    QByteArray part = m_buffer.mid(m_pos, p - m_pos);
    m_pos = p;
    if (skipDelimiter)
    {
        skip(1);
    }

    return part;
}

void ByteStream::skipAll()
{
    m_pos = m_buffer.size();
}

QByteArray ByteStream::readAll()
{
    QByteArray bytes = m_buffer.mid(m_pos);

    m_pos = m_buffer.size();

    return bytes;
}

bool ByteStream::atEnd() const
{
    return m_pos == m_buffer.size();
}

} // namespace tastefulserver


text:
package ttlcontroller

import (
	"context"
	"fmt"
	"net/http/httptest"
	"strconv"
	"sync"
	"testing"
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/informers"
	clientset "k8s.io/client-go/kubernetes"
	listers "k8s.io/client-go/listers/core/v1"
	restclient "k8s.io/client-go/rest"
	"k8s.io/kubernetes/pkg/controller/ttl"
	"k8s.io/kubernetes/test/integration/framework"
)

func createClientAndInformers(t *testing.T, server *httptest.Server) (*clientset.Clientset, informers.SharedInformerFactory) {
	config := restclient.Config{
		Host:  server.URL,
		QPS:   500,
		Burst: 500,
	}
	testClient := clientset.NewForConfigOrDie(&config)

	informers := informers.NewSharedInformerFactory(testClient, time.Second)
	return testClient, informers
}

func createNodes(t *testing.T, client *clientset.Clientset, startIndex, endIndex int) {
	var wg sync.WaitGroup
	for i := startIndex; i < endIndex; i++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			node := &v1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Name: fmt.Sprintf("node-%d", idx),
				},
			}
			if _, err := client.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}); err != nil {
				t.Fatalf("Failed to create node: %v", err)
			}
		}(i)
	}
	wg.Wait()
}

func deleteNodes(t *testing.T, client *clientset.Clientset, startIndex, endIndex int) {
	var wg sync.WaitGroup
	for i := startIndex; i < endIndex; i++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			name := fmt.Sprintf("node-%d", idx)
			if err := client.CoreV1().Nodes().Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
				t.Fatalf("Failed to delete node: %v", err)
			}
		}(i)
	}
	wg.Wait()
}

func waitForNodesWithTTLAnnotation(t *testing.T, nodeLister listers.NodeLister, numNodes, ttlSeconds int) {
	if err := wait.Poll(time.Second, 30*time.Second, func() (bool, error) {
		nodes, err := nodeLister.List(labels.Everything())
		if err != nil || len(nodes) != numNodes {
			return false, nil
		}
		for _, node := range nodes {
			if node.Annotations == nil {
				return false, nil
			}
			value, ok := node.Annotations[v1.ObjectTTLAnnotationKey]
			if !ok {
				return false, nil
			}
			currentTTL, err := strconv.Atoi(value)
			if err != nil || currentTTL != ttlSeconds {
				return false, nil
			}
		}
		return true, nil
	}); err != nil {
		t.Fatalf("Failed waiting for all nodes with annotation: %v", err)
	}
}

// Test whether ttlcontroller sets correct ttl annotations.
func TestTTLAnnotations(t *testing.T) {
	_, server, closeFn := framework.RunAMaster(nil)
	defer closeFn()

	testClient, informers := createClientAndInformers(t, server)
	nodeInformer := informers.Core().V1().Nodes()
	ttlc := ttl.NewTTLController(nodeInformer, testClient)

	stopCh := make(chan struct{})
	defer close(stopCh)
	go nodeInformer.Informer().Run(stopCh)
	go ttlc.Run(1, stopCh)

	// Create 100 nodes all should have annotation equal to 0.
	createNodes(t, testClient, 0, 100)
	waitForNodesWithTTLAnnotation(t, informers.Core().V1().Nodes().Lister(), 100, 0)

	// Create 1 more node, all annotation should change to 15.
	createNodes(t, testClient, 100, 101)
	waitForNodesWithTTLAnnotation(t, informers.Core().V1().Nodes().Lister(), 101, 15)

	// Delete 11 nodes, it should still remain at the level of 15.
	deleteNodes(t, testClient, 90, 101)
	waitForNodesWithTTLAnnotation(t, informers.Core().V1().Nodes().Lister(), 90, 15)

	// Delete 1 more node, all should be decreased to 0.
	deleteNodes(t, testClient, 89, 90)
	waitForNodesWithTTLAnnotation(t, informers.Core().V1().Nodes().Lister(), 89, 0)
}


text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.facebook.presto.spark;

import com.facebook.airlift.json.JsonCodec;
import com.facebook.airlift.log.Logger;
import com.facebook.airlift.stats.Distribution;
import com.facebook.presto.Session;
import com.facebook.presto.block.BlockEncodingManager;
import com.facebook.presto.common.Page;
import com.facebook.presto.common.type.Type;
import com.facebook.presto.event.QueryMonitor;
import com.facebook.presto.execution.ExecutionFailureInfo;
import com.facebook.presto.execution.QueryIdGenerator;
import com.facebook.presto.execution.QueryInfo;
import com.facebook.presto.execution.QueryPreparer;
import com.facebook.presto.execution.QueryPreparer.PreparedQuery;
import com.facebook.presto.execution.QueryState;
import com.facebook.presto.execution.QueryStateTimer;
import com.facebook.presto.execution.QueryStats;
import com.facebook.presto.execution.StageExecutionId;
import com.facebook.presto.execution.StageExecutionInfo;
import com.facebook.presto.execution.StageExecutionState;
import com.facebook.presto.execution.StageId;
import com.facebook.presto.execution.StageInfo;
import com.facebook.presto.execution.TaskInfo;
import com.facebook.presto.execution.scheduler.ExecutionWriterTarget;
import com.facebook.presto.execution.scheduler.StreamingPlanSection;
import com.facebook.presto.execution.scheduler.StreamingSubPlan;
import com.facebook.presto.execution.scheduler.TableWriteInfo;
import com.facebook.presto.execution.warnings.WarningCollector;
import com.facebook.presto.metadata.Metadata;
import com.facebook.presto.security.AccessControl;
import com.facebook.presto.server.BasicQueryInfo;
import com.facebook.presto.server.QuerySessionSupplier;
import com.facebook.presto.server.SessionContext;
import com.facebook.presto.spark.classloader_interface.IPrestoSparkQueryExecution;
import com.facebook.presto.spark.classloader_interface.IPrestoSparkQueryExecutionFactory;
import com.facebook.presto.spark.classloader_interface.IPrestoSparkTaskExecutor;
import com.facebook.presto.spark.classloader_interface.MutablePartitionId;
import com.facebook.presto.spark.classloader_interface.PrestoSparkConfInitializer;
import com.facebook.presto.spark.classloader_interface.PrestoSparkMutableRow;
import com.facebook.presto.spark.classloader_interface.PrestoSparkSerializedPage;
import com.facebook.presto.spark.classloader_interface.PrestoSparkSession;
import com.facebook.presto.spark.classloader_interface.PrestoSparkTaskExecutorFactoryProvider;
import com.facebook.presto.spark.classloader_interface.PrestoSparkTaskInputs;
import com.facebook.presto.spark.classloader_interface.PrestoSparkTaskOutput;
import com.facebook.presto.spark.classloader_interface.SerializedPrestoSparkTaskDescriptor;
import com.facebook.presto.spark.classloader_interface.SerializedTaskInfo;
import com.facebook.presto.spark.execution.PrestoSparkExecutionException;
import com.facebook.presto.spark.execution.PrestoSparkExecutionExceptionFactory;
import com.facebook.presto.spark.execution.PrestoSparkTaskExecutorFactory;
import com.facebook.presto.spark.planner.PrestoSparkPlanFragmenter;
import com.facebook.presto.spark.planner.PrestoSparkQueryPlanner;
import com.facebook.presto.spark.planner.PrestoSparkQueryPlanner.PlanAndMore;
import com.facebook.presto.spark.planner.PrestoSparkRddFactory;
import com.facebook.presto.spark.util.PrestoSparkUtils;
import com.facebook.presto.spi.ConnectorId;
import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.spi.PrestoException;
import com.facebook.presto.spi.QueryId;
import com.facebook.presto.spi.connector.ConnectorCapabilities;
import com.facebook.presto.spi.memory.MemoryPoolId;
import com.facebook.presto.spi.page.PagesSerde;
import com.facebook.presto.spi.resourceGroups.ResourceGroupId;
import com.facebook.presto.sql.planner.PlanFragment;
import com.facebook.presto.sql.planner.SubPlan;
import com.facebook.presto.sql.planner.plan.PlanFragmentId;
import com.facebook.presto.transaction.TransactionId;
import com.facebook.presto.transaction.TransactionInfo;
import com.facebook.presto.transaction.TransactionManager;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ListMultimap;
import com.google.common.util.concurrent.UncheckedExecutionException;
import org.apache.spark.SparkContext;
import org.apache.spark.SparkException;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.broadcast.Broadcast;
import org.apache.spark.util.CollectionAccumulator;
import org.joda.time.DateTime;
import scala.Some;
import scala.Tuple2;

import javax.inject.Inject;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

import static com.facebook.airlift.concurrent.MoreFutures.getFutureValue;
import static com.facebook.presto.execution.QueryState.FAILED;
import static com.facebook.presto.execution.QueryState.FINISHED;
import static com.facebook.presto.execution.QueryState.PLANNING;
import static com.facebook.presto.execution.StageInfo.getAllStages;
import static com.facebook.presto.execution.scheduler.StreamingPlanSection.extractStreamingSections;
import static com.facebook.presto.execution.scheduler.TableWriteInfo.createTableWriteInfo;
import static com.facebook.presto.spark.classloader_interface.ScalaUtils.collectScalaIterator;
import static com.facebook.presto.spark.classloader_interface.ScalaUtils.emptyScalaIterator;
import static com.facebook.presto.spark.util.PrestoSparkUtils.toSerializedPage;
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
import static com.facebook.presto.spi.connector.ConnectorCapabilities.SUPPORTS_PAGE_SINK_COMMIT;
import static com.facebook.presto.sql.planner.SystemPartitioningHandle.COORDINATOR_DISTRIBUTION;
import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_BROADCAST_DISTRIBUTION;
import static com.facebook.presto.sql.planner.planPrinter.PlanPrinter.textDistributedPlan;
import static com.facebook.presto.util.Failures.toFailure;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Throwables.propagateIfPossible;
import static com.google.common.base.Ticker.systemTicker;
import static com.google.common.base.Verify.verify;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.util.concurrent.Futures.getUnchecked;
import static io.airlift.units.DataSize.succinctBytes;
import static java.lang.Math.max;
import static java.nio.file.Files.notExists;
import static java.util.Collections.unmodifiableList;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;

public class PrestoSparkQueryExecutionFactory
        implements IPrestoSparkQueryExecutionFactory
{
    private static final Logger log = Logger.get(PrestoSparkQueryExecutionFactory.class);

    private final QueryIdGenerator queryIdGenerator;
    private final QuerySessionSupplier sessionSupplier;
    private final QueryPreparer queryPreparer;
    private final PrestoSparkQueryPlanner queryPlanner;
    private final PrestoSparkPlanFragmenter planFragmenter;
    private final PrestoSparkRddFactory rddFactory;
    private final QueryMonitor queryMonitor;
    private final JsonCodec<TaskInfo> taskInfoJsonCodec;
    private final JsonCodec<PrestoSparkTaskDescriptor> sparkTaskDescriptorJsonCodec;
    private final JsonCodec<QueryInfo> queryInfoJsonCodec;
    private final TransactionManager transactionManager;
    private final AccessControl accessControl;
    private final Metadata metadata;
    private final BlockEncodingManager blockEncodingManager;
    private final PrestoSparkSettingsRequirements settingsRequirements;
    private final PrestoSparkExecutionExceptionFactory executionExceptionFactory;
    private final PrestoSparkTaskExecutorFactory prestoSparkTaskExecutorFactory;

    private final Set<PrestoSparkCredentialsProvider> credentialsProviders;
    private final Set<PrestoSparkAuthenticatorProvider> authenticatorProviders;

    @Inject
    public PrestoSparkQueryExecutionFactory(
            QueryIdGenerator queryIdGenerator,
            QuerySessionSupplier sessionSupplier,
            QueryPreparer queryPreparer,
            PrestoSparkQueryPlanner queryPlanner,
            PrestoSparkPlanFragmenter planFragmenter,
            PrestoSparkRddFactory rddFactory,
            QueryMonitor queryMonitor,
            JsonCodec<TaskInfo> taskInfoJsonCodec,
            JsonCodec<PrestoSparkTaskDescriptor> sparkTaskDescriptorJsonCodec,
            JsonCodec<QueryInfo> queryInfoJsonCodec,
            TransactionManager transactionManager,
            AccessControl accessControl,
            Metadata metadata,
            BlockEncodingManager blockEncodingManager,
            PrestoSparkSettingsRequirements settingsRequirements,
            PrestoSparkExecutionExceptionFactory executionExceptionFactory,
            PrestoSparkTaskExecutorFactory prestoSparkTaskExecutorFactory,
            Set<PrestoSparkCredentialsProvider> credentialsProviders,
            Set<PrestoSparkAuthenticatorProvider> authenticatorProviders)
    {
        this.queryIdGenerator = requireNonNull(queryIdGenerator, "queryIdGenerator is null");
        this.sessionSupplier = requireNonNull(sessionSupplier, "sessionSupplier is null");
        this.queryPreparer = requireNonNull(queryPreparer, "queryPreparer is null");
        this.queryPlanner = requireNonNull(queryPlanner, "queryPlanner is null");
        this.planFragmenter = requireNonNull(planFragmenter, "planFragmenter is null");
        this.rddFactory = requireNonNull(rddFactory, "rddFactory is null");
        this.queryMonitor = requireNonNull(queryMonitor, "queryMonitor is null");
        this.taskInfoJsonCodec = requireNonNull(taskInfoJsonCodec, "taskInfoJsonCodec is null");
        this.sparkTaskDescriptorJsonCodec = requireNonNull(sparkTaskDescriptorJsonCodec, "sparkTaskDescriptorJsonCodec is null");
        this.queryInfoJsonCodec = requireNonNull(queryInfoJsonCodec, "queryInfoJsonCodec is null");
        this.transactionManager = requireNonNull(transactionManager, "transactionManager is null");
        this.accessControl = requireNonNull(accessControl, "accessControl is null");
        this.metadata = requireNonNull(metadata, "metadata is null");
        this.blockEncodingManager = requireNonNull(blockEncodingManager, "blockEncodingManager is null");
        this.settingsRequirements = requireNonNull(settingsRequirements, "settingsRequirements is null");
        this.executionExceptionFactory = requireNonNull(executionExceptionFactory, "executionExceptionFactory is null");
        this.prestoSparkTaskExecutorFactory = requireNonNull(prestoSparkTaskExecutorFactory, "prestoSparkTaskExecutorFactory is null");
        this.credentialsProviders = ImmutableSet.copyOf(requireNonNull(credentialsProviders, "credentialsProviders is null"));
        this.authenticatorProviders = ImmutableSet.copyOf(requireNonNull(authenticatorProviders, "authenticatorProviders is null"));
    }

    @Override
    public IPrestoSparkQueryExecution create(
            SparkContext sparkContext,
            PrestoSparkSession prestoSparkSession,
            String sql,
            Optional<String> sparkQueueName,
            PrestoSparkTaskExecutorFactoryProvider executorFactoryProvider,
            Optional<Path> queryInfoOutputPath)
    {
        PrestoSparkConfInitializer.checkInitialized(sparkContext);

        queryInfoOutputPath.ifPresent(path -> checkArgument(notExists(path), "File already exist: %s", path));

        QueryStateTimer queryStateTimer = new QueryStateTimer(systemTicker());

        queryStateTimer.beginPlanning();

        QueryId queryId = queryIdGenerator.createNextQueryId();
        log.info("Starting execution for presto query: %s", queryId);
        SessionContext sessionContext = PrestoSparkSessionContext.createFromSessionInfo(
                prestoSparkSession,
                credentialsProviders,
                authenticatorProviders);

        // TODO: implement warning collection
        WarningCollector warningCollector = WarningCollector.NOOP;

        Session session = sessionSupplier.createSession(queryId, sessionContext);
        settingsRequirements.verify(sparkContext, session);

        TransactionId transactionId = transactionManager.beginTransaction(true);
        session = session.beginTransactionId(transactionId, transactionManager, accessControl);

        queryMonitor.queryCreatedEvent(
                new BasicQueryInfo(createQueryInfo(
                        session,
                        sql,
                        PLANNING,
                        Optional.empty(),
                        sparkQueueName,
                        Optional.empty(),
                        queryStateTimer,
                        Optional.empty(),
                        warningCollector)));

        PlanAndMore planAndMore = null;
        try {
            queryStateTimer.beginAnalyzing();

            PreparedQuery preparedQuery = queryPreparer.prepareQuery(session, sql, warningCollector);
            planAndMore = queryPlanner.createQueryPlan(session, preparedQuery, warningCollector);
            SubPlan fragmentedPlan = planFragmenter.fragmentQueryPlan(session, planAndMore.getPlan(), warningCollector);
            log.info(textDistributedPlan(fragmentedPlan, metadata.getFunctionManager(), session, true));
            TableWriteInfo tableWriteInfo = getTableWriteInfo(session, fragmentedPlan);

            JavaSparkContext javaSparkContext = new JavaSparkContext(sparkContext);
            CollectionAccumulator<SerializedTaskInfo> taskInfoCollector = new CollectionAccumulator<>();
            taskInfoCollector.register(sparkContext, new Some<>("taskInfoCollector"), false);

            queryStateTimer.endAnalysis();

            return new PrestoSparkQueryExecution(
                    javaSparkContext,
                    session,
                    queryMonitor,
                    taskInfoCollector,
                    prestoSparkTaskExecutorFactory,
                    executorFactoryProvider,
                    queryStateTimer,
                    warningCollector,
                    sql,
                    planAndMore,
                    fragmentedPlan,
                    sparkQueueName,
                    taskInfoJsonCodec,
                    sparkTaskDescriptorJsonCodec,
                    queryInfoJsonCodec,
                    rddFactory,
                    tableWriteInfo,
                    transactionManager,
                    new PagesSerde(blockEncodingManager, Optional.empty(), Optional.empty(), Optional.empty()),
                    executionExceptionFactory,
                    queryInfoOutputPath);
        }
        catch (RuntimeException executionFailure) {
            queryStateTimer.beginFinishing();
            try {
                rollback(session, transactionManager);
            }
            catch (RuntimeException rollbackFailure) {
                log.error(rollbackFailure, "Encountered error when performing rollback");
            }
            queryStateTimer.endQuery();

            Optional<ExecutionFailureInfo> failureInfo = Optional.empty();
            if (executionFailure instanceof PrestoSparkExecutionException) {
                failureInfo = executionExceptionFactory.extractExecutionFailureInfo((PrestoSparkExecutionException) executionFailure);
                verify(failureInfo.isPresent());
            }
            if (!failureInfo.isPresent()) {
                failureInfo = Optional.of(toFailure(executionFailure));
            }

            try {
                QueryInfo queryInfo = createQueryInfo(
                        session,
                        sql,
                        FAILED,
                        Optional.ofNullable(planAndMore),
                        sparkQueueName,
                        failureInfo,
                        queryStateTimer,
                        Optional.empty(),
                        warningCollector);
                queryMonitor.queryCompletedEvent(queryInfo);
                queryInfoOutputPath.ifPresent(path -> writeQueryInfo(path, queryInfo, queryInfoJsonCodec));
            }
            catch (RuntimeException eventFailure) {
                log.error(eventFailure, "Error publishing query immediate failure event");
            }

            throw failureInfo.get().toFailure();
        }
    }

    private TableWriteInfo getTableWriteInfo(Session session, SubPlan plan)
    {
        StreamingPlanSection streamingPlanSection = extractStreamingSections(plan);
        StreamingSubPlan streamingSubPlan = streamingPlanSection.getPlan();
        TableWriteInfo tableWriteInfo = createTableWriteInfo(streamingSubPlan, metadata, session);
        if (tableWriteInfo.getWriterTarget().isPresent()) {
            checkPageSinkCommitIsSupported(session, tableWriteInfo.getWriterTarget().get());
        }
        return tableWriteInfo;
    }

    private void checkPageSinkCommitIsSupported(Session session, ExecutionWriterTarget writerTarget)
    {
        ConnectorId connectorId;
        if (writerTarget instanceof ExecutionWriterTarget.DeleteHandle) {
            throw new PrestoException(NOT_SUPPORTED, "delete queries are not supported by presto on spark");
        }
        else if (writerTarget instanceof ExecutionWriterTarget.CreateHandle) {
            connectorId = ((ExecutionWriterTarget.CreateHandle) writerTarget).getHandle().getConnectorId();
        }
        else if (writerTarget instanceof ExecutionWriterTarget.InsertHandle) {
            connectorId = ((ExecutionWriterTarget.InsertHandle) writerTarget).getHandle().getConnectorId();
        }
        else {
            throw new IllegalArgumentException("unexpected writer target type: " + writerTarget.getClass());
        }
        verify(connectorId != null, "connectorId is null");
        Set<ConnectorCapabilities> connectorCapabilities = metadata.getConnectorCapabilities(session, connectorId);
        if (!connectorCapabilities.contains(SUPPORTS_PAGE_SINK_COMMIT)) {
            throw new PrestoException(NOT_SUPPORTED, "catalog does not support page sink commit: " + connectorId);
        }
    }

    private static void commit(Session session, TransactionManager transactionManager)
    {
        getFutureValue(transactionManager.asyncCommit(getTransactionInfo(session, transactionManager).getTransactionId()));
    }

    private static void rollback(Session session, TransactionManager transactionManager)
    {
        getFutureValue(transactionManager.asyncAbort(getTransactionInfo(session, transactionManager).getTransactionId()));
    }

    private static TransactionInfo getTransactionInfo(Session session, TransactionManager transactionManager)
    {
        Optional<TransactionInfo> transaction = session.getTransactionId()
                .flatMap(transactionManager::getOptionalTransactionInfo);
        checkState(transaction.isPresent(), "transaction is not present");
        checkState(transaction.get().isAutoCommitContext(), "transaction doesn't have auto commit context enabled");
        return transaction.get();
    }

    private static QueryInfo createQueryInfo(
            Session session,
            String query,
            QueryState queryState,
            Optional<PlanAndMore> planAndMore,
            Optional<String> sparkQueueName,
            Optional<ExecutionFailureInfo> failureInfo,
            QueryStateTimer queryStateTimer,
            Optional<StageInfo> rootStage,
            WarningCollector warningCollector)
    {
        checkArgument(failureInfo.isPresent() || queryState != FAILED, "unexpected query state: %s", queryState);

        int peakRunningTasks = 0;
        long peakUserMemoryReservationInBytes = 0;
        long peakTotalMemoryReservationInBytes = 0;
        long peakTaskUserMemoryInBytes = 0;
        long peakTaskTotalMemoryInBytes = 0;

        for (StageInfo stageInfo : getAllStages(rootStage)) {
            StageExecutionInfo stageExecutionInfo = stageInfo.getLatestAttemptExecutionInfo();
            for (TaskInfo taskInfo : stageExecutionInfo.getTasks()) {
                // there's no way to know how many tasks were running in parallel in Spark
                // for now let's assume that all the tasks were running in parallel
                peakRunningTasks++;
                long taskPeakUserMemoryInBytes = taskInfo.getStats().getUserMemoryReservation().toBytes();
                long taskPeakTotalMemoryInBytes = taskInfo.getStats().getPeakTotalMemoryInBytes();
                peakUserMemoryReservationInBytes += taskPeakUserMemoryInBytes;
                peakTotalMemoryReservationInBytes += taskPeakTotalMemoryInBytes;
                peakTaskUserMemoryInBytes = max(peakTaskUserMemoryInBytes, taskPeakUserMemoryInBytes);
                peakTaskTotalMemoryInBytes = max(peakTaskTotalMemoryInBytes, taskPeakTotalMemoryInBytes);
            }
        }

        QueryStats queryStats = QueryStats.create(
                queryStateTimer,
                rootStage,
                peakRunningTasks,
                succinctBytes(peakUserMemoryReservationInBytes),
                succinctBytes(peakTotalMemoryReservationInBytes),
                succinctBytes(peakTaskUserMemoryInBytes),
                succinctBytes(peakTaskTotalMemoryInBytes));

        return new QueryInfo(
                session.getQueryId(),
                session.toSessionRepresentation(),
                queryState,
                new MemoryPoolId("spark-memory-pool"),
                queryStats.isScheduled(),
                URI.create("http://fake.invalid/query/" + session.getQueryId()),
                planAndMore.map(PlanAndMore::getFieldNames).orElse(ImmutableList.of()),
                query,
                queryStats,
                Optional.empty(),
                Optional.empty(),
                ImmutableMap.of(),
                ImmutableSet.of(),
                ImmutableMap.of(),
                ImmutableMap.of(),
                ImmutableSet.of(),
                Optional.empty(),
                false,
                planAndMore.flatMap(PlanAndMore::getUpdateType).orElse(null),
                rootStage,
                failureInfo.orElse(null),
                failureInfo.map(ExecutionFailureInfo::getErrorCode).orElse(null),
                warningCollector.getWarnings(),
                planAndMore.map(PlanAndMore::getInputs).orElse(ImmutableSet.of()),
                planAndMore.flatMap(PlanAndMore::getOutput),
                true,
                sparkQueueName.map(ResourceGroupId::new),
                planAndMore.flatMap(PlanAndMore::getQueryType),
                Optional.empty(),
                Optional.empty());
    }

    private static StageInfo createStageInfo(QueryId queryId, SubPlan plan, List<TaskInfo> taskInfos)
    {
        ListMultimap<PlanFragmentId, TaskInfo> taskInfoMap = ArrayListMultimap.create();
        for (TaskInfo taskInfo : taskInfos) {
            PlanFragmentId fragmentId = new PlanFragmentId(taskInfo.getTaskId().getStageExecutionId().getStageId().getId());
            taskInfoMap.put(fragmentId, taskInfo);
        }
        return createStageInfo(queryId, plan, taskInfoMap);
    }

    private static StageInfo createStageInfo(QueryId queryId, SubPlan plan, ListMultimap<PlanFragmentId, TaskInfo> taskInfoMap)
    {
        PlanFragmentId planFragmentId = plan.getFragment().getId();
        StageId stageId = new StageId(queryId, planFragmentId.getId());
        List<TaskInfo> taskInfos = taskInfoMap.get(planFragmentId);
        long peakUserMemoryReservationInBytes = 0;
        for (TaskInfo taskInfo : taskInfos) {
            long taskPeakUserMemoryInBytes = taskInfo.getStats().getUserMemoryReservation().toBytes();
            peakUserMemoryReservationInBytes += taskPeakUserMemoryInBytes;
        }
        StageExecutionInfo stageExecutionInfo = StageExecutionInfo.create(
                new StageExecutionId(stageId, 0),
                // TODO: figure out a way to know what exactly stage has caused a failure
                StageExecutionState.FINISHED,
                Optional.empty(),
                taskInfos,
                DateTime.now(),
                new Distribution().snapshot(),
                succinctBytes(peakUserMemoryReservationInBytes),
                1,
                1);
        return new StageInfo(
                stageId,
                URI.create("http://fake.invalid/stage/" + stageId),
                Optional.of(plan.getFragment()),
                stageExecutionInfo,
                ImmutableList.of(),
                plan.getChildren().stream()
                        .map(child -> createStageInfo(queryId, child, taskInfoMap))
                        .collect(toImmutableList()),
                false);
    }

    private static void writeQueryInfo(Path queryInfoOutputPath, QueryInfo queryInfo, JsonCodec<QueryInfo> queryInfoJsonCodec)
    {
        try {
            Files.write(queryInfoOutputPath, queryInfoJsonCodec.toJsonBytes(queryInfo));
        }
        catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    public static class PrestoSparkQueryExecution
            implements IPrestoSparkQueryExecution
    {
        private final JavaSparkContext sparkContext;
        private final Session session;
        private final QueryMonitor queryMonitor;
        private final CollectionAccumulator<SerializedTaskInfo> taskInfoCollector;
        // used to create tasks on the Driver
        private final PrestoSparkTaskExecutorFactory taskExecutorFactory;
        // used to create tasks on executor, serializable
        private final PrestoSparkTaskExecutorFactoryProvider taskExecutorFactoryProvider;
        private final QueryStateTimer queryStateTimer;
        private final WarningCollector warningCollector;
        private final String query;
        private final PlanAndMore planAndMore;
        private final SubPlan fragmentedPlan;
        private final Optional<String> sparkQueueName;

        private final JsonCodec<TaskInfo> taskInfoJsonCodec;
        private final JsonCodec<PrestoSparkTaskDescriptor> sparkTaskDescriptorJsonCodec;
        private final JsonCodec<QueryInfo> queryInfoJsonCodec;
        private final PrestoSparkRddFactory rddFactory;
        private final TableWriteInfo tableWriteInfo;
        private final TransactionManager transactionManager;
        private final PagesSerde pagesSerde;
        private final PrestoSparkExecutionExceptionFactory executionExceptionFactory;
        private final Optional<Path> queryInfoOutputPath;

        private PrestoSparkQueryExecution(
                JavaSparkContext sparkContext,
                Session session,
                QueryMonitor queryMonitor,
                CollectionAccumulator<SerializedTaskInfo> taskInfoCollector,
                PrestoSparkTaskExecutorFactory taskExecutorFactory,
                PrestoSparkTaskExecutorFactoryProvider taskExecutorFactoryProvider,
                QueryStateTimer queryStateTimer,
                WarningCollector warningCollector,
                String query,
                PlanAndMore planAndMore,
                SubPlan fragmentedPlan,
                Optional<String> sparkQueueName,
                JsonCodec<TaskInfo> taskInfoJsonCodec,
                JsonCodec<PrestoSparkTaskDescriptor> sparkTaskDescriptorJsonCodec,
                JsonCodec<QueryInfo> queryInfoJsonCodec,
                PrestoSparkRddFactory rddFactory,
                TableWriteInfo tableWriteInfo,
                TransactionManager transactionManager,
                PagesSerde pagesSerde,
                PrestoSparkExecutionExceptionFactory executionExceptionFactory,
                Optional<Path> queryInfoOutputPath)
        {
            this.sparkContext = requireNonNull(sparkContext, "sparkContext is null");
            this.session = requireNonNull(session, "session is null");
            this.queryMonitor = requireNonNull(queryMonitor, "queryMonitor is null");
            this.taskInfoCollector = requireNonNull(taskInfoCollector, "taskInfoCollector is null");
            this.taskExecutorFactory = requireNonNull(taskExecutorFactory, "taskExecutorFactory is null");
            this.taskExecutorFactoryProvider = requireNonNull(taskExecutorFactoryProvider, "taskExecutorFactoryProvider is null");
            this.queryStateTimer = requireNonNull(queryStateTimer, "queryStateTimer is null");
            this.warningCollector = requireNonNull(warningCollector, "warningCollector is null");
            this.query = requireNonNull(query, "query is null");
            this.planAndMore = requireNonNull(planAndMore, "planAndMore is null");
            this.fragmentedPlan = requireNonNull(fragmentedPlan, "fragmentedPlan is null");
            this.sparkQueueName = requireNonNull(sparkQueueName, "sparkQueueName is null");

            this.taskInfoJsonCodec = requireNonNull(taskInfoJsonCodec, "taskInfoJsonCodec is null");
            this.sparkTaskDescriptorJsonCodec = requireNonNull(sparkTaskDescriptorJsonCodec, "sparkTaskDescriptorJsonCodec is null");
            this.queryInfoJsonCodec = requireNonNull(queryInfoJsonCodec, "queryInfoJsonCodec is null");
            this.rddFactory = requireNonNull(rddFactory, "rddFactory is null");
            this.tableWriteInfo = requireNonNull(tableWriteInfo, "tableWriteInfo is null");
            this.transactionManager = requireNonNull(transactionManager, "transactionManager is null");
            this.pagesSerde = requireNonNull(pagesSerde, "pagesSerde is null");
            this.executionExceptionFactory = requireNonNull(executionExceptionFactory, "executionExceptionFactory is null");
            this.queryInfoOutputPath = requireNonNull(queryInfoOutputPath, "queryInfoOutputPath is null");
        }

        @Override
        public List<List<Object>> execute()
        {
            queryStateTimer.beginRunning();

            List<Tuple2<MutablePartitionId, PrestoSparkSerializedPage>> rddResults;
            try {
                rddResults = doExecute(fragmentedPlan);
                queryStateTimer.beginFinishing();
                commit(session, transactionManager);
                queryStateTimer.endQuery();
            }
            catch (Exception executionFailure) {
                queryStateTimer.beginFinishing();
                try {
                    rollback(session, transactionManager);
                }
                catch (RuntimeException rollbackFailure) {
                    log.error(rollbackFailure, "Encountered error when performing rollback");
                }

                Optional<ExecutionFailureInfo> failureInfo = Optional.empty();
                if (executionFailure instanceof SparkException) {
                    failureInfo = executionExceptionFactory.extractExecutionFailureInfo((SparkException) executionFailure);
                }
                if (!failureInfo.isPresent() && executionFailure instanceof PrestoSparkExecutionException) {
                    failureInfo = executionExceptionFactory.extractExecutionFailureInfo((PrestoSparkExecutionException) executionFailure);
                }
                if (!failureInfo.isPresent()) {
                    failureInfo = Optional.of(toFailure(executionFailure));
                }

                queryStateTimer.endQuery();

                try {
                    queryCompletedEvent(failureInfo);
                }
                catch (RuntimeException eventFailure) {
                    log.error(eventFailure, "Error publishing query completed event");
                }

                throw failureInfo.get().toFailure();
            }

            // successfully finished
            try {
                queryCompletedEvent(Optional.empty());
            }
            catch (RuntimeException eventFailure) {
                log.error(eventFailure, "Error publishing query completed event");
            }

            ConnectorSession connectorSession = session.toConnectorSession();
            List<Type> types = fragmentedPlan.getFragment().getTypes();
            ImmutableList.Builder<List<Object>> result = ImmutableList.builder();
            for (Tuple2<MutablePartitionId, PrestoSparkSerializedPage> tuple : rddResults) {
                Page page = pagesSerde.deserialize(toSerializedPage(tuple._2));
                checkArgument(page.getChannelCount() == types.size(), "expected %s channels, got %s", types.size(), page.getChannelCount());
                for (int position = 0; position < page.getPositionCount(); position++) {
                    List<Object> columns = new ArrayList<>();
                    for (int channel = 0; channel < page.getChannelCount(); channel++) {
                        columns.add(types.get(channel).getObjectValue(connectorSession.getSqlFunctionProperties(), page.getBlock(channel), position));
                    }
                    result.add(unmodifiableList(columns));
                }
            }
            return result.build();
        }

        public List<Type> getOutputTypes()
        {
            return fragmentedPlan.getFragment().getTypes();
        }

        public Optional<String> getUpdateType()
        {
            return planAndMore.getUpdateType();
        }

        private List<Tuple2<MutablePartitionId, PrestoSparkSerializedPage>> doExecute(SubPlan root)
                throws SparkException
        {
            PlanFragment rootFragment = root.getFragment();

            if (rootFragment.getPartitioning().equals(COORDINATOR_DISTRIBUTION)) {
                PrestoSparkTaskDescriptor taskDescriptor = new PrestoSparkTaskDescriptor(
                        session.toSessionRepresentation(),
                        session.getIdentity().getExtraCredentials(),
                        rootFragment,
                        tableWriteInfo);
                SerializedPrestoSparkTaskDescriptor serializedTaskDescriptor = new SerializedPrestoSparkTaskDescriptor(sparkTaskDescriptorJsonCodec.toJsonBytes(taskDescriptor));

                Map<PlanFragmentId, RddAndMore<PrestoSparkSerializedPage>> inputRdds = root.getChildren().stream()
                        .collect(toImmutableMap(child -> child.getFragment().getId(), child -> createRdd(child, PrestoSparkSerializedPage.class)));

                Map<String, Future<List<Tuple2<MutablePartitionId, PrestoSparkSerializedPage>>>> inputFutures = inputRdds.entrySet().stream()
                        .collect(toImmutableMap(entry -> entry.getKey().toString(), entry -> entry.getValue().getRdd().collectAsync()));

                waitFor(inputFutures.values());

                Map<String, List<PrestoSparkSerializedPage>> inputs = inputFutures.entrySet().stream()
                        .collect(toImmutableMap(
                                Map.Entry::getKey,
                                entry -> getUnchecked(entry.getValue()).stream().map(Tuple2::_2).collect(toImmutableList())));

                IPrestoSparkTaskExecutor<PrestoSparkSerializedPage> prestoSparkTaskExecutor = taskExecutorFactory.create(
                        0,
                        0,
                        serializedTaskDescriptor,
                        emptyScalaIterator(),
                        new PrestoSparkTaskInputs(ImmutableMap.of(), ImmutableMap.of(), inputs),
                        taskInfoCollector,
                        PrestoSparkSerializedPage.class);
                return collectScalaIterator(prestoSparkTaskExecutor);
            }

            RddAndMore<PrestoSparkSerializedPage> rootRdd = createRdd(root, PrestoSparkSerializedPage.class);
            return rootRdd.collectAndDestroyDependencies();
        }

        private <T extends PrestoSparkTaskOutput> RddAndMore<T> createRdd(SubPlan subPlan, Class<T> outputType)
        {
            ImmutableMap.Builder<PlanFragmentId, JavaPairRDD<MutablePartitionId, PrestoSparkMutableRow>> rddInputs = ImmutableMap.builder();
            ImmutableMap.Builder<PlanFragmentId, Broadcast<List<PrestoSparkSerializedPage>>> broadcastInputs = ImmutableMap.builder();
            ImmutableList.Builder<Broadcast<?>> broadcastDependencies = ImmutableList.builder();

            for (SubPlan child : subPlan.getChildren()) {
                PlanFragment childFragment = child.getFragment();
                if (childFragment.getPartitioningScheme().getPartitioning().getHandle().equals(FIXED_BROADCAST_DISTRIBUTION)) {
                    RddAndMore<PrestoSparkSerializedPage> childRdd = createRdd(child, PrestoSparkSerializedPage.class);
                    List<PrestoSparkSerializedPage> broadcastPages = childRdd.collectAndDestroyDependencies().stream()
                            .map(Tuple2::_2)
                            .collect(toList());
                    Broadcast<List<PrestoSparkSerializedPage>> broadcast = sparkContext.broadcast(broadcastPages);
                    broadcastInputs.put(childFragment.getId(), broadcast);
                    broadcastDependencies.add(broadcast);
                }
                else {
                    RddAndMore<PrestoSparkMutableRow> childRdd = createRdd(child, PrestoSparkMutableRow.class);
                    rddInputs.put(childFragment.getId(), childRdd.getRdd());
                    broadcastDependencies.addAll(childRdd.getBroadcastDependencies());
                }
            }
            JavaPairRDD<MutablePartitionId, T> rdd = rddFactory.createSparkRdd(
                    sparkContext,
                    session,
                    subPlan.getFragment(),
                    rddInputs.build(),
                    broadcastInputs.build(),
                    taskExecutorFactoryProvider,
                    taskInfoCollector,
                    tableWriteInfo,
                    outputType);
            return new RddAndMore<>(rdd, broadcastDependencies.build());
        }

        private void queryCompletedEvent(Optional<ExecutionFailureInfo> failureInfo)
        {
            List<SerializedTaskInfo> serializedTaskInfos = taskInfoCollector.value();
            List<TaskInfo> taskInfos = serializedTaskInfos.stream()
                    .map(SerializedTaskInfo::getBytes)
                    .map(PrestoSparkUtils::decompress)
                    .map(taskInfoJsonCodec::fromJson)
                    .collect(toImmutableList());
            StageInfo stageInfo = createStageInfo(session.getQueryId(), fragmentedPlan, taskInfos);
            QueryState queryState = failureInfo.isPresent() ? FAILED : FINISHED;

            QueryInfo queryInfo = createQueryInfo(
                    session,
                    query,
                    queryState,
                    Optional.of(planAndMore),
                    sparkQueueName,
                    failureInfo,
                    queryStateTimer,
                    Optional.of(stageInfo),
                    warningCollector);

            queryMonitor.queryCompletedEvent(queryInfo);
            queryInfoOutputPath.ifPresent(path -> writeQueryInfo(path, queryInfo, queryInfoJsonCodec));
        }

        private static <T> void waitFor(Collection<Future<T>> futures)
                throws SparkException
        {
            try {
                for (Future<?> future : futures) {
                    future.get();
                }
            }
            catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new RuntimeException(e);
            }
            catch (ExecutionException e) {
                propagateIfPossible(e.getCause(), SparkException.class);
                propagateIfPossible(e.getCause(), RuntimeException.class);

                // this should never happen
                throw new UncheckedExecutionException(e.getCause());
            }
            finally {
                for (Future<?> future : futures) {
                    if (!future.isDone()) {
                        future.cancel(true);
                    }
                }
            }
        }
    }

    private static class RddAndMore<T extends PrestoSparkTaskOutput>
    {
        private final JavaPairRDD<MutablePartitionId, T> rdd;
        private final List<Broadcast<?>> broadcastDependencies;

        private boolean collected;

        private RddAndMore(JavaPairRDD<MutablePartitionId, T> rdd, List<Broadcast<?>> broadcastDependencies)
        {
            this.rdd = requireNonNull(rdd, "rdd is null");
            this.broadcastDependencies = ImmutableList.copyOf(requireNonNull(broadcastDependencies, "broadcastDependencies is null"));
        }

        public List<Tuple2<MutablePartitionId, T>> collectAndDestroyDependencies()
        {
            checkState(!collected, "already collected");
            collected = true;
            List<Tuple2<MutablePartitionId, T>> result = rdd.collect();
            broadcastDependencies.forEach(Broadcast::destroy);
            return result;
        }

        public JavaPairRDD<MutablePartitionId, T> getRdd()
        {
            return rdd;
        }

        public List<Broadcast<?>> getBroadcastDependencies()
        {
            return broadcastDependencies;
        }
    }
}


text:
from __future__ import unicode_literals

from django.db import migrations

class Migration(migrations.Migration):

    dependencies = [
        ('news', '0003_auto_20170109_1633'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='newspost',
            options={'permissions': (('view_news', 'view news'),), 'verbose_name': 'News post', 'verbose_name_plural': 'News posts'},
        ),
    ]


text:
<?php
defined('BASEPATH') OR exit('No direct script access allowed');

/*
|--------------------------------------------------------------------------
| Modules DB Connection
|
| Popolamento array:
|  key: nome modulo
|  value: nome connessione
| (Se non presente la chiave del modulo nell'array, si assume come nome
|  del database il nome del modulo stesso)
|--------------------------------------------------------------------------
 */
$config['modules_db_connections'] = array();

/*
|--------------------------------------------------------------------------
| Associazione controller a nome tabella fisica
|
| Popolamento array:
|  key: nome controller
|  value: array:
|           table_name: nome tabella fisica
|           model_alias: nome alias model
| (Se non presente la chiave del controller nell'array, si va di CoC)
|--------------------------------------------------------------------------
 */
/*$config['controllers_assoc'] = array(
    'Soggetto' => array(
        'table_name' => 'soggetto',
        'model_alias' => 'soggetto'
    )
);*/

$config['controllers_assoc'] = array();

/*
|--------------------------------------------------------------------------
| Parametri paginazione query
|
| Se il model non è presente nella mappa, assume il valore di default
|--------------------------------------------------------------------------
 */
$config['default_query_limit'] = 10;
$config['models_query_limit'] = array(
    'frontends' => 999
);


text:
using System;
using Newtonsoft.Json;

namespace Microsoft.Deployment.Common.Model.Informatica
{
    public class InformaticaSchedule : InformaticaObject
    {
        [JsonProperty("orgId", NullValueHandling = NullValueHandling.Ignore)]
        public string OrgId;

        [JsonProperty("startTime", NullValueHandling = NullValueHandling.Ignore)]
        public DateTime StartTime = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 1, 0, 0, DateTimeKind.Utc);

        [JsonProperty("interval", NullValueHandling = NullValueHandling.Ignore)]
        public string Interval;

        [JsonProperty("frequency")]
        public int Frequency = 1;

        [JsonProperty("mon")]
        public bool Mon = true;

        [JsonProperty("tue")]
        public bool Tue = true;

        [JsonProperty("wed")]
        public bool Wed = true;

        [JsonProperty("thu")]
        public bool Thu = true;

        [JsonProperty("fri")]
        public bool Fri = true;

        [JsonProperty("sat")]
        public bool Sat = true;

        [JsonProperty("sun")]
        public bool Sun = true;

        [JsonProperty("weekDay")]
        public const bool WeekDay = false;

        [JsonProperty("dayOfMonth")]
        public const int DayOfMonth = 0;

        [JsonProperty("timeZoneId", NullValueHandling = NullValueHandling.Ignore)]
        public const string TimeZoneId = "GMT";

        public InformaticaSchedule()
        {
            Type = "schedule";
        }

        public InformaticaSchedule(DateTime start, int frequency) : this()
        {
            this.Frequency = frequency;

            StartTime = start.ToUniversalTime();
        }
    }
}