text:
"""
This module contains functions and classes to train regressors and save them into SQL database.
"""

import logging
from os import path
from glob import glob
import pickle
from datetime import datetime
import codecs

import numpy as np
import pandas as pd

import sklearn
from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.metrics import explained_variance_score, r2_score, mean_squared_error, mean_absolute_error
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import Pipeline

from api_etl.settings import (
    __S3_BUCKETS__, __TRAINING_FEATURE_NAMES__, __ACCEPTED_LINES__,
    __TRAINING_SET_FOLDER_PATH__, __TRAINING_SET_FOLDER_NAME__, __DATA_PATH__)
from api_etl.utils_misc import S3Bucket
from api_etl.utils_rdb import rdb_provider
from api_etl.data_models import Predictor

logger = logging.getLogger(__name__)
pd.options.mode.chained_assignment = None

import matplotlib
matplotlib.use("TkAgg")
import matplotlib.pyplot as plt
import seaborn as sns

class RegressorTrainer:
    """
    This class allows you to easily load datasets and train regressors for a given line, and save it in a database.
    """

    def __init__(self, start_date="20170101", end_date="20170701", line="C",
        tempo=30, auto=False, **kwargs):

        assert line in __ACCEPTED_LINES__
        self.line = line
        self.scaler = None
        self.regressor = None
        self.pipeline = None
        self.features = __TRAINING_FEATURE_NAMES__
        self.start_date = None
        self.end_date = None

        self.dfm = None  # whole training set before filtering
        self.sel = None  # selection

        self._fitted = False
        self._big_label_delay_mask = None
        self._big_last_delay_mask = None
        self.delay_threshold = None

        if auto:
            self.set_feature_cols()
            self.build_training_set(start_date=start_date, end_date=end_date, tempo=tempo, **kwargs)
            self.build_pipeline()
            self.train_pipeline()

    def set_feature_cols(self, *args):
        args = args or __TRAINING_FEATURE_NAMES__
        self.features = args

    def build_training_set(self, start_date="20170215", end_date="20170401",
                           tempo=30, **kwargs):
        """
        Either from folder, either from S3
        :param start_date:
        :param end_date:
        :param tempo:
        :return:
        """

        # ARGS PARSING
        assert isinstance(tempo, int)
        datetime.strptime(start_date, "%Y%m%d")
        self.start_date = start_date
        datetime.strptime(end_date, "%Y%m%d")
        self.end_date = end_date

        # Training sets are stored by day
        dti = pd.date_range(start_date, end_date, freq="D")
        days = dti.map(lambda x: x.strftime("%Y%m%d")).tolist()

        self._load_files_from_s3(tempo=tempo)
        dfm = self._load_files_from_folder(tempo=tempo, days=days)

        # delete duplicated columns
        dfm = dfm.loc[:, ~dfm.columns.duplicated()]
        self._raw_training_set = dfm

        # Line selection
        self.sel = dfm.copy()
        # Apply line selection
        self._filter_line(line=self.line)
        # Apply other filter
        self._apply_filter_to_selection(**kwargs)

        # Train and test
        self._split_train_test()

    def _load_files_from_s3(self, tempo):
        """
        1 - Download training sets from S3 in data folder.
        2 - Build from folder
        :return:
        """
        # Load relevant bucket
        bucket_provider = S3Bucket(__S3_BUCKETS__["training-sets"], create_if_absent=True)

        # Download files from S3
        logger.info("Get training sets from folder %s, and download them in %s" %
                    (__TRAINING_SET_FOLDER_NAME__ % tempo, __TRAINING_SET_FOLDER_PATH__ % tempo))
        bucket_provider.download_folder(
            remote_prefix=__TRAINING_SET_FOLDER_NAME__ % tempo,
            local_folder_root=__DATA_PATH__
        )

        return True

    def _load_files_from_folder(self, tempo, days):
        # Load training sets from folder
        __FULL_TRAINING_SETS_FOLDER_PATH__ = __TRAINING_SET_FOLDER_PATH__ % tempo

        # All files in folder
        training_set_pickles = glob(path.join(__FULL_TRAINING_SETS_FOLDER_PATH__, "*"))

        # Select days
        training_set_pickles = [p for p in training_set_pickles if path.splitext(path.basename(p))[0] in days]

        # Load and concatenate training sets
        dataframes = list(map(pd.read_pickle, training_set_pickles))
        return pd.concat(dataframes)

    def _filter_line(self, line):
        assert line in __ACCEPTED_LINES__
        self.line = line
        mask = (self.sel.index.get_level_values("Route_route_short_name_ix") == line)
        self.sel = self.sel[mask]

    def _apply_filter_to_selection(self, min_diff=1, max_diff=40, min_delay=-3000, max_delay=10000):
        sel = self.sel

        # By sequence: selection only prediction for 1 to 10 stations ahead
        cond1 = (sel.index.get_level_values("TS_sequence_diff_ix") >= min_diff)
        cond2 = (sel.index.get_level_values("TS_sequence_diff_ix") <= max_diff)
        mask = cond1 & cond2
        sel = sel[mask]

        # Per delay
        cond1 = (self.sel["label"] >= min_delay)
        cond2 = (self.sel["label"] <= max_delay)
        mask = cond1 & cond2
        sel = sel[mask]

        self.sel = sel

    def _split_train_test(self):

        self.X = self.sel[self.features]
        self.y_naive_pred = self.sel["P_naive_pred"]
        self.y = self.sel["label"]

        self.X_train, self.X_test, self.y_train, self.y_test, self.y_naive_pred_train, self.y_naive_pred_test = \
            train_test_split(
                self.X, self.y, self.y_naive_pred,
                test_size=0.30, random_state=1
            )

    def build_pipeline(self, scale=True, polynomial=False, regressor=LinearRegression):

        steps = []
        # Scaling preprocessing
        if scale:
            scaler = StandardScaler(copy=True).fit(self.X)
            steps.append(("scaler", scaler))

        # Polynomial preprocessing
        if polynomial:
            poly = PolynomialFeatures(2)
            steps.append(("polynomial", poly))

        steps.append(("regressor", regressor()))

        self.pipeline = Pipeline(steps=steps)

    def train_pipeline(self, delay_threshold=None, last_delay_threshold=None):
        # accepts only one of two
        assert isinstance(delay_threshold, int) or delay_threshold is None
        assert isinstance(last_delay_threshold, int) or last_delay_threshold is None
        assert not ((last_delay_threshold is not None) and (delay_threshold is not None))

        X_train = self.X_train
        y_train = self.y_train

        if delay_threshold is not None:
            # Per delay
            self._big_label_delay_mask = (self.sel["label"] >= delay_threshold)
            X_train = X_train[self._big_label_delay_mask]
            y_train = y_train[self._big_label_delay_mask]
            self.delay_threshold = delay_threshold

        if last_delay_threshold is not None:
            self._big_last_delay_mask = self.sel.loc[:,"TS_last_observed_delay"] >= last_delay_threshold
            X_train = X_train[self._big_last_delay_mask]
            y_train = y_train[self._big_last_delay_mask]

        self.pipeline.fit(X_train, y_train)
        self._fitted = True

    def score_pipeline(self):
        assert self._fitted

        y_pred = self.pipeline.predict(self.X_test)
        # SCORES COMPARISON
        # Score comparison between naive prediction, and custom predictions
        message = "\n\nPREDICTIONS ALL:"
        message += self.show_scores(name="Regressor", y_true=self.y_test, y_pred=y_pred)
        message += self.show_scores(name="Naive", y_true=self.y_test, y_pred=self.y_naive_pred_test)

        if self._big_label_delay_mask is not None:
            y_pred_bd = self.pipeline.predict(self.X_test[self._big_label_delay_mask])
            y_naive_pred_test_bd = self.y_naive_pred_test[self._big_label_delay_mask]
            y_pred_sd = self.pipeline.predict(self.X_test[~self._big_label_delay_mask])
            y_naive_pred_test_sd = self.y_naive_pred_test[~self._big_label_delay_mask]

            message += "\n\nPREDICTIONS BIG DELAYS:"
            message += self.show_scores(name="Regressor", y_true=self.y_test[self._big_label_delay_mask], y_pred=y_pred_bd)
            message += self.show_scores(name="Naive", y_true=self.y_test[self._big_label_delay_mask], y_pred=y_naive_pred_test_bd)

            message += "\n\nPREDICTIONS SMALL DELAYS:"
            message += self.show_scores(name="Regressor", y_true=self.y_test[~self._big_label_delay_mask], y_pred=y_pred_sd)
            message += self.show_scores(name="Naive", y_true=self.y_test[~self._big_label_delay_mask], y_pred=y_naive_pred_test_sd)

        if self._big_last_delay_mask is not None:
            y_pred_bd = self.pipeline.predict(self.X_test[self._big_last_delay_mask])
            y_naive_pred_test_bd = self.y_naive_pred_test[self._big_last_delay_mask]
            y_pred_sd = self.pipeline.predict(self.X_test[~self._big_last_delay_mask])
            y_naive_pred_test_sd = self.y_naive_pred_test[~self._big_last_delay_mask]

            message += "\n\nPREDICTIONS BIG DELAYS:"
            message += self.show_scores(name="Regressor", y_true=self.y_test[self._big_last_delay_mask], y_pred=y_pred_bd)
            message += self.show_scores(name="Naive", y_true=self.y_test[self._big_last_delay_mask], y_pred=y_naive_pred_test_bd)

            message += "\n\nPREDICTIONS SMALL DELAYS:"
            message += self.show_scores(name="Regressor", y_true=self.y_test[~self._big_last_delay_mask], y_pred=y_pred_sd)
            message += self.show_scores(name="Naive", y_true=self.y_test[~self._big_last_delay_mask], y_pred=y_naive_pred_test_sd)

        return message

    def show_scores(self, name, y_true, y_pred, all=False):
        message = "\n%s SCORES" % name.upper()
        if all:
            message += "\n%s R2 score: %s" % (name, r2_score(y_true=y_true, y_pred=y_pred))
            message += "\n%s Explained variance: %s" % (name, explained_variance_score(y_true, y_pred))
            message += "\n%s Mean square error: %s" % (name, mean_squared_error(y_true, y_pred))

        message += "\n%s Mean absolute error: %s" % (name, mean_absolute_error(y_true, y_pred))
        return message

    def save_in_database(self):
        assert self._fitted

        pickled_pipeline = codecs.encode(pickle.dumps(self.pipeline), "base64").decode()

        # init predictor
        predictor = Predictor(
            pipeline=pickle.dumps(self.pipeline),
            features=self.features,
            line=self.line,
            delay_threshold=self.delay_threshold,
            training_set_start=self.start_date,
            training_set_end=self.end_date,
            sklearn_version=sklearn.__version__,
            pipeline_steps=self.pipeline.steps.__str__(),
            score_description=self.score_pipeline()
        )

        # save in db
        session = rdb_provider.get_session()
        try:
            session.add(predictor)
            session.commit()
        except:
            session.rollback()
        session.close()

    def analyze_scores(self):
        y_pred = self.pipeline.predict(self.X_test)

        comparison_df = pd.DataFrame(
            data={"r": self.y_test.values, "p": y_pred},
            index=self.y_test.index)

        comparison_df["reg_abs_error"] = np.abs(comparison_df.r - comparison_df.p)
        comparison_df["reg_sqr_error"] = comparison_df["reg_abs_error"]**2
        comparison_df = pd.concat([self.X_test, comparison_df], axis=1)

        comparison_df["naive_abs_error"] = np.abs(comparison_df.r - comparison_df.TS_last_observed_delay)
        comparison_df["naive_sqr_error"] = comparison_df["naive_abs_error"]**2

        # PLOTS
        # per sequence diff
        comparison_df.groupby(level=5)[["reg_abs_error","naive_abs_error"]].mean().plot()
        plt.title("Regressor prediction mean square error, per sequence_diff")
        plt.show()

        # per label
        comparison_df.groupby("r")[["reg_abs_error","naive_abs_error"]].mean().plot()
        plt.title("Regressor prediction mean square error, per label delay")
        plt.show()

        # per last_observed_delay
        comparison_df.groupby("TS_last_observed_delay")[["reg_abs_error","naive_abs_error"]].mean().plot()
        plt.title("Regressor prediction mean square error, per label delay")
        plt.show()

        # per number_of_trains
        comparison_df.groupby("TS_rolling_trips_on_line")[["reg_abs_error","naive_abs_error"]].mean().plot()
        plt.title("Regressor prediction mean square error, per label delay")
        plt.show()

        comparison_df.groupby(level=[2, 5]).reg_abs_error.mean().unstack().T.plot()
        plt.title("Regressor prediction mean square error, per sequence_diff and mission code")
        plt.show()


text:
<!DOCTYPE html>

{% comment %}
Copyright 2016 Google Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
{% endcomment %}

    <meta charset='utf-8' />
    <title>Sub Alerts</title>

    <link rel="stylesheet" type="text/css" href="/static/font-effects.css">

redpajama_set_name: RedPajamaGithub


text:
package com.opengamma.integration.marketdata.manipulator.dsl;

import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

import org.threeten.bp.Instant;
import org.threeten.bp.LocalDateTime;
import org.threeten.bp.ZonedDateTime;
import org.threeten.bp.format.DateTimeFormatter;
import org.threeten.bp.format.DateTimeParseException;

import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Maps;
import com.opengamma.engine.function.FunctionParameters;
import com.opengamma.engine.function.SimpleFunctionParameters;
import com.opengamma.engine.function.StructureManipulationFunction;
import com.opengamma.engine.marketdata.manipulator.DistinctMarketDataSelector;
import com.opengamma.engine.marketdata.manipulator.ScenarioDefinition;
import com.opengamma.engine.marketdata.manipulator.function.StructureManipulator;
import com.opengamma.id.VersionCorrection;
import com.opengamma.util.ArgumentChecker;
import com.opengamma.util.OpenGammaClock;

/**
 * Encapsulates a set of transformations to apply to market data when a calculation cycle is run.
 */
public class Scenario {

  /** For parsing valuation time. */
  private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");

  /** Manipulators keyed by the selectors for the items they apply to. */
  private final ListMultimap<DistinctMarketDataSelector, StructureManipulator<?>> _manipulations = ArrayListMultimap.create();

  /** This scenario's name. */
  private final String _name;
  /** The simulation to which this scenario belongs, possibly null */
  private final Simulation _simulation;

  /** Calc configs to which this scenario will be applied, null will match any config. */
  private Set<String> _calcConfigNames;
  /** Valuation time of this scenario's calculation cycle. */
  private Instant _valuationTime;
  /** Version correction used by the resolver. */
  private VersionCorrection _resolverVersionCorrection;

  /**
   * Creates a new scenario with a calcuation configuration name of "Default", valuation time of {@code Instant.now()} and resolver version correction of
   * {@link VersionCorrection#LATEST}.
   *
   * @param name
   *          The scenario name, not null
   */
  public Scenario(final String name) {
    ArgumentChecker.notEmpty(name, "name");
    _name = name;
    _simulation = null;
  }

  /* package */ Scenario(final Simulation simulation, final String name) {
    ArgumentChecker.notEmpty(name, "name");
    ArgumentChecker.notNull(simulation, "simulation");
    _simulation = simulation;
    _name = name;
  }

  /**
   * @return A object for specifying which curves should be transformed
   */
  public YieldCurveSelector.Builder curve() {
    return new YieldCurveSelector.Builder(this);
  }

  /**
   * @return An object for specifying which market data points should be transformed
   */
  public PointSelector.Builder marketDataPoint() {
    return new PointSelector.Builder(this);
  }

  /**
   * @return An object for specifying which volatility surfaces should be transformed
   */
  public VolatilitySurfaceSelector.Builder surface() {
    return new VolatilitySurfaceSelector.Builder(this);
  }

  public SpotRateSelectorBuilder spotRate() {
    return new SpotRateSelectorBuilder(this);
  }

  /**
   * Updates this scenario to apply to the specified calculation configuration.
   *
   * @param configNames
   *          The calculation configuration name
   * @return The modified scenario
   */
  public Scenario calculationConfigurations(final String... configNames) {
    ArgumentChecker.notEmpty(configNames, "configName");
    _calcConfigNames = ImmutableSet.copyOf(configNames);
    return this;
  }

  /**
   * Updates this scenario to use the specified valuation time.
   *
   * @param valuationTime
   *          The valuation time
   * @return The modified scenario
   */
  public Scenario valuationTime(final Instant valuationTime) {
    ArgumentChecker.notNull(valuationTime, "valuationTime");
    _valuationTime = valuationTime;
    return this;
  }

  /**
   * Updates this scenario to use the specified valuation time.
   *
   * @param valuationTime
   *          The valuation time
   * @return The modified scenario
   */
  public Scenario valuationTime(final String valuationTime) {
    try {
      final LocalDateTime localTime = LocalDateTime.parse(valuationTime, DATE_FORMATTER);
      _valuationTime = ZonedDateTime.of(localTime, OpenGammaClock.getZone()).toInstant();
    } catch (final DateTimeParseException e) {
      throw new IllegalArgumentException("Valuation time isn't in a valid format. Expected format " + "'yyyy-MM-dd HH:mm', value: " + valuationTime);
    }
    return this;
  }

  /**
   * Updates this scenario to use the specified valuation time.
   *
   * @param valuationTime
   *          The valuation time
   * @return The modified scenario
   */
  public Scenario valuationTime(final ZonedDateTime valuationTime) {
    ArgumentChecker.notNull(valuationTime, "valuationTime");
    return valuationTime(valuationTime.toInstant());
  }

  /**
   * Updates this scenario to use the specified version correction in the resolver.
   *
   * @param resolverVersionCorrection
   *          The resolver version correction
   * @return The modified scenario
   */
  public Scenario resolverVersionCorrection(final VersionCorrection resolverVersionCorrection) {
    ArgumentChecker.notNull(resolverVersionCorrection, "resolverVersionCorrection");
    _resolverVersionCorrection = resolverVersionCorrection;
    return this;
  }

  /**
   * @return A {@link ScenarioDefinition} created from this scenario's selectors and manipulators
   */
  @SuppressWarnings("unchecked")
  public ScenarioDefinition createDefinition() {
    final Map<DistinctMarketDataSelector, FunctionParameters> params = Maps.newHashMapWithExpectedSize(_manipulations.size());
    for (final Map.Entry<DistinctMarketDataSelector, Collection<StructureManipulator<?>>> entry : _manipulations.asMap().entrySet()) {
      final DistinctMarketDataSelector selector = entry.getKey();
      // ListMultimap always has Lists as entries even if the signature doesn't say so
      final List<StructureManipulator<?>> manipulators = (List<StructureManipulator<?>>) entry.getValue();
      final CompositeStructureManipulator compositeManipulator = new CompositeStructureManipulator(manipulators);
      final SimpleFunctionParameters functionParameters = new SimpleFunctionParameters();
      functionParameters.setValue(StructureManipulationFunction.EXPECTED_PARAMETER_NAME, compositeManipulator);
      params.put(selector, functionParameters);
    }
    return new ScenarioDefinition(_name, params);
  }

  /* package */ void add(final DistinctMarketDataSelector selector, final StructureManipulator<?> manipulator) {
    _manipulations.put(selector, manipulator);
  }

  /* package */ Instant getValuationTime() {
    if (_valuationTime != null) {
      return _valuationTime;
    } else if (_simulation != null) {
      return _simulation.getValuationTime();
    } else {
      return null;
    }
  }

  /* package */ VersionCorrection getResolverVersionCorrection() {
    if (_resolverVersionCorrection != null) {
      return _resolverVersionCorrection;
    } else if (_simulation != null) {
      return _simulation.getResolverVersionCorrection();
    } else {
      return null;
    }
  }

  // TODO get from simulation if null
  /* package */ Set<String> getCalcConfigNames() {
    if (_calcConfigNames != null) {
      return _calcConfigNames;
    } else if (_simulation != null) {
      return _simulation.getCalcConfigNames();
    } else {
      return null;
    }
  }

  /**
   * @return The scenario name, not null
   */
  public String getName() {
    return _name;
  }

  @Override
  public int hashCode() {
    return Objects.hash(_manipulations, _name, _calcConfigNames, _valuationTime, _resolverVersionCorrection);
  }

  @Override
  public boolean equals(final Object obj) {
    if (this == obj) {
      return true;
    }
    if (obj == null || getClass() != obj.getClass()) {
      return false;
    }
    final Scenario other = (Scenario) obj;
    return Objects.equals(this._manipulations, other._manipulations) && Objects.equals(this._name, other._name)
        && Objects.equals(this._calcConfigNames, other._calcConfigNames) && Objects.equals(this._valuationTime, other._valuationTime)
        && Objects.equals(this._resolverVersionCorrection, other._resolverVersionCorrection);
  }

  @Override
  public String toString() {
    return "Scenario [" + "_name='" + _name + "'" + ", _calcConfigNames=" + _calcConfigNames + ", _valuationTime=" + _valuationTime
        + ", _resolverVersionCorrection=" + _resolverVersionCorrection + ", _manipulations=" + _manipulations + "]";
  }
}


text:
// @flow

import { doGetJSON } from '../base/util';
import { isInBreakoutRoom } from '../breakout-rooms/functions';

/**
 * Determines whether Salesforce is enabled for the current conference.
 *
 * @param {Function|Object} state - The redux store, the redux
 * {@code getState} function, or the redux state itself.
 * @returns {boolean}
 */
export const isSalesforceEnabled = (state: Function | Object) => {
    const { salesforceUrl } = state['features/base/config'];
    const isBreakoutRoom = isInBreakoutRoom(state);

    return Boolean(salesforceUrl) && !isBreakoutRoom;
};

/**
 * Fetches the Salesforce records that were most recently interacted with.
 *
 * @param {string} url - The endpoint for the session records.
 * @param {string} jwt - The JWT needed for authentication.
 * @returns {Promise<any>}
 */
export async function getRecentSessionRecords(
        url: string,
        jwt: string
) {
    return doGetJSON(`${url}/records/recents`, true, {
        headers: {
            'Authorization': `Bearer ${jwt}`
        }
    });
}

/**
 * Fetches the Salesforce records that match the search criteria.
 *
 * @param {string} url - The endpoint for the session records.
 * @param {string} jwt - The JWT needed for authentication.
 * @param {string} text - The search term for the session record to find.
 * @returns {Promise<any>}
 */
export async function searchSessionRecords(
        url: string,
        jwt: string,
        text: string
) {
    return doGetJSON(`${url}/records?text=${text}`, true, {
        headers: {
            'Authorization': `Bearer ${jwt}`
        }
    });
}

/**
* Fetches the Salesforce record details from the server.
*
* @param {string} url - The endpoint for the record details.
* @param {string} jwt - The JWT needed for authentication.
* @param {Object} item - The item for which details are being retrieved.
* @returns {Promise<any>}
*/
export async function getSessionRecordDetails(
        url: string,
        jwt: string,
        item: Object
) {
    const fullUrl = `${url}/records/${item.id}?type=${item.type}`;

    return doGetJSON(fullUrl, true, {
        headers: {
            'Authorization': `Bearer ${jwt}`
        }
    });
}

/**
* Executes the meeting linking.
*
* @param {string} url - The endpoint for meeting linking.
* @param {string} jwt - The JWT needed for authentication.
* @param {string} sessionId - The ID of the meeting session.
* @param {Object} body - The body of the request.
* @returns {Object}
*/
export async function executeLinkMeetingRequest(
        url: string,
        jwt: string,
        sessionId: String,
        body: Object
) {
    const fullUrl = `${url}/sessions/${sessionId}/records/${body.id}`;
    const res = await fetch(fullUrl, {
        method: 'PUT',
        headers: {
            'Content-Type': 'application/json',
            'Authorization': `Bearer ${jwt}`
        },
        body: JSON.stringify(body)
    });

    const json = await res.json();

    return res.ok ? json : Promise.reject(json);
}


text:
.. -*- coding: utf-8; mode: rst -*-

.. _VIDEO_GET_PTS:

=============
VIDEO_GET_PTS
=============

Name
----

VIDEO_GET_PTS

.. attention:: This ioctl is deprecated.

Synopsis
--------

.. c:function:: int ioctl(int fd, VIDEO_GET_PTS, __u64 *pts)
    :name: VIDEO_GET_PTS

Arguments
---------

.. flat-table::
    :header-rows:  0
    :stub-columns: 0

    -  .. row 1

       -  int fd

       -  File descriptor returned by a previous call to open().

    -  .. row 2

       -  int request

       -  Equals VIDEO_GET_PTS for this command.

    -  .. row 3

       -  __u64 \*pts

       -  Returns the 33-bit timestamp as defined in ITU T-REC-H.222.0 /
	  ISO/IEC 13818-1.

	  The PTS should belong to the currently played frame if possible,
	  but may also be a value close to it like the PTS of the last
	  decoded frame or the last PTS extracted by the PES parser.

Description
-----------

This ioctl is obsolete. Do not use in new drivers. For V4L2 decoders
this ioctl has been replaced by the ``V4L2_CID_MPEG_VIDEO_DEC_PTS``
control.

This ioctl call asks the Video Device to return the current PTS
timestamp.

Return Value
------------

On success 0 is returned, on error -1 and the ``errno`` variable is set
appropriately. The generic error codes are described at the
:ref:`Generic Error Codes <gen-errors>` chapter.


text:
using System;
using System.Diagnostics;
using System.Numerics;

namespace GigaFFT
{
    public class ComplexArray : IDisposable
    {
        private bool lightWeight;
        private Complex[][] smallArray = null;
        private ComplexArray2MeC largeArray = null;

        public ComplexArray(ulong totalCapacityComplexStructs)
        {
            Debug.Assert(totalCapacityComplexStructs > 0);
            this.lightWeight = totalCapacityComplexStructs <= 1024UL * 1024UL;
            if (this.lightWeight)
            {
                ulong stripeCount = (totalCapacityComplexStructs + 4096UL - 1UL) / 4096UL;
                Debug.Assert((stripeCount != 0) && ((stripeCount & (stripeCount - 1)) == 0), "stripeCount must be power of 2");
                this.smallArray = new Complex[stripeCount][];
                for (int stripe = 0; stripe < this.smallArray.Length; stripe++)
                {
                    this.smallArray[stripe] = new Complex[Math.Min(totalCapacityComplexStructs, 4096UL)];
                }
            }
            else
            {
                this.largeArray = new ComplexArray2MeC(totalCapacityComplexStructs);
            }
        }

        public ulong Length
        {
            get
            {
                return lightWeight ? (smallArray.Length == 1 ? (ulong)smallArray[0].Length : (ulong)smallArray.Length * 4096UL) : largeArray.Length;
            }
        }

        public Complex this[ulong index]
        {
            get
            {
                return lightWeight ? smallArray[index / 4096UL][index % 4096UL] : largeArray[index];
            }
            set
            {
                if (lightWeight)
                {
                    smallArray[index / 4096UL][index % 4096UL] = value;
                }
                else
                {
                    largeArray[index] = value;
                }
            }
        }

        public void Dispose()
        {
            Dispose(true);
            GC.SuppressFinalize(this);
        }

        protected virtual void Dispose(bool disposing)
        {
            if (smallArray != null || largeArray != null)
            {
                if (disposing)
                {
                    if (!lightWeight)
                    {
                        largeArray.Dispose();
                    }
                    smallArray = null;
                    largeArray = null;
                }
            }
        }

        ~ComplexArray()
        {
            Dispose(false);
        }
    }
}


text:
ENV["SPOT_MODE"] = "console"

require File.expand_path("../boot.rb", __FILE__)

require "spontaneous/capistrano"
require "bundler/capistrano"

default_run_options[:pty]   = true
# run capistrano in a login shell so that ruby version managers can do their thing
# otherwise it's just a constant problem trying to use the installed/configured
# version of ruby
default_run_options[:shell] = '/bin/bash -l'

ssh_options[:port]          = 22
ssh_options[:forward_agent] = true

set :ruby_version,          "2.3.0-p0"

set :domain,                "test.com"
set :cms,                   "edit.test.com"
set :application,           "test.com"
server                      "test.com", :app, :web, :db, :media
set :user,                  "test"
set :use_sudo,              false

set :scm,                   "git"
set :repository,            "git@test.com:spontaneous.git"

set :deploy_to,             "/home/#{fetch(:user)}/spontaneous"

namespace :deploy do
  task :restart, :roles => [:app, :web] do
    run "sv kill /home/#{fetch(:user)}/service/enabled/back"
  end
end


text:
using Abp.Domain.Repositories;
using Abp.Domain.Uow;
using Abp.Events.Bus;
using Abp.Events.Bus.Entities;
using Abp.NHibernate.Tests.Entities;
using Shouldly;
using System.Linq;
using System.Threading.Tasks;
using Xunit;

namespace Abp.NHibernate.Tests
{
    public class Basic_Repository_Tests : NHibernateTestBase
    {
        private readonly IRepository<Person> _personRepository;
        private readonly IRepository<Book> _booksRepository;
        private readonly IUnitOfWorkManager _unitOfWorkManager;

        public Basic_Repository_Tests()
        {
            _personRepository = Resolve<IRepository<Person>>();
            _booksRepository = Resolve<IRepository<Book>>();
            _unitOfWorkManager = Resolve<IUnitOfWorkManager>();

            UsingSession(session => session.Save(new Person() { Name = "emre" }));
            UsingSession(session => session.Save(new Book { Name = "Hitchhikers Guide to the Galaxy" }));
            UsingSession(session => session.Save(new Book { Name = "My First ABCs", IsDeleted = true }));
        }

        [Fact]
        public async Task Should_Get_Null_On_Not_Found()
        {
            var person = _personRepository.FirstOrDefault(-1);
            person.ShouldBeNull();

            person = await _personRepository.FirstOrDefaultAsync(-1);
            person.ShouldBeNull();
        }

        [Fact]
        public void Should_Insert_People()
        {
            _personRepository.Insert(new Person() { Name = "halil" });

            var insertedPerson = UsingSession(session => session.Query<Person>().FirstOrDefault(p => p.Name == "halil"));
            insertedPerson.ShouldNotBe(null);
            insertedPerson.IsTransient().ShouldBe(false);
            insertedPerson.Name.ShouldBe("halil");
        }

        [Fact]
        public async Task Should_Insert_People_Async()
        {
            await _personRepository.InsertAsync(new Person() { Name = "halil" });

            var insertedPerson = UsingSession(session => session.Query<Person>().FirstOrDefault(p => p.Name == "halil"));
            insertedPerson.ShouldNotBeNull();
            insertedPerson.IsTransient().ShouldBeFalse();
            insertedPerson.Name.ShouldBe("halil");
        }

        [Fact]
        public void Should_Filter_SoftDelete()
        {
            var books = _booksRepository.GetAllList();
            books.All(p => !p.IsDeleted).ShouldBeTrue();
        }

        [Fact]
        public async Task Should_Filter_SoftDelete_Async()
        {
            var books = await _booksRepository.GetAllListAsync();
            books.All(p => !p.IsDeleted).ShouldBeTrue();
        }

        [Fact]
        public void Should_Get_SoftDeleted_Entities_If_Filter_Is_Disabled()
        {
            using (_unitOfWorkManager.Begin())
            using (_unitOfWorkManager.Current.DisableFilter(AbpDataFilters.SoftDelete))
            {
                var books = _booksRepository.GetAllList();
                books.Any(x => x.IsDeleted).ShouldBe(true);
            }
        }

        [Fact]
        public void Update_With_Action_Test()
        {
            var userBefore = UsingSession(session => session.Query<Person>().Single(p => p.Name == "emre"));

            var updatedUser = _personRepository.Update(userBefore.Id, user => user.Name = "yunus");
            updatedUser.Id.ShouldBe(userBefore.Id);
            updatedUser.Name.ShouldBe("yunus");

            var userAfter = UsingSession(session => session.Get<Person>(userBefore.Id));
            userAfter.Name.ShouldBe("yunus");
        }
        [Fact]
        public async Task Update_With_Action_Test_Async()
        {
            var userBefore = UsingSession(session => session.Query<Person>().Single(p => p.Name == "emre"));

            var updatedUser =await _personRepository.UpdateAsync(userBefore.Id, user =>
            {
                user.Name = "yunus";
                return Task.FromResult(user);
            });
            updatedUser.Id.ShouldBe(userBefore.Id);
            updatedUser.Name.ShouldBe("yunus");

            var userAfter = UsingSession(session => session.Get<Person>(userBefore.Id));
            userAfter.Name.ShouldBe("yunus");
        }
        [Fact]
        public void Should_Trigger_Event_On_Insert()
        {
            var triggerCount = 0;

            Resolve<IEventBus>().Register<EntityCreatedEventData<Person>>(
                eventData =>
                {
                    eventData.Entity.Name.ShouldBe("halil");
                    eventData.Entity.IsTransient().ShouldBe(false);
                    triggerCount++;
                });

            _personRepository.Insert(new Person { Name = "halil" });

            triggerCount.ShouldBe(1);
        }

        [Fact]
        public async Task Should_Trigger_Event_On_Insert_Async()
        {
            var triggerCount = 0;

            Resolve<IEventBus>().Register<EntityCreatedEventData<Person>>(
                eventData =>
                {
                    eventData.Entity.Name.ShouldBe("halil");
                    eventData.Entity.IsTransient().ShouldBeFalse();
                    triggerCount++;
                });

            await _personRepository.InsertAsync(new Person { Name = "halil" });

            triggerCount.ShouldBe(1);
        }

        [Fact]
        public void Should_Trigger_Event_On_Update()
        {
            var triggerCount = 0;

            Resolve<IEventBus>().Register<EntityUpdatedEventData<Person>>(
                eventData =>
                {
                    eventData.Entity.Name.ShouldBe("emre2");
                    triggerCount++;
                });

            var emrePerson = _personRepository.Single(p => p.Name == "emre");
            emrePerson.Name = "emre2";
            _personRepository.Update(emrePerson);

            triggerCount.ShouldBe(1);
        }

        [Fact]
        public async Task Should_Trigger_Event_On_Update_Async()
        {
            var triggerCount = 0;

            Resolve<IEventBus>().Register<EntityUpdatedEventData<Person>>(
                eventData =>
                {
                    eventData.Entity.Name.ShouldBe("emre2");
                    triggerCount++;
                });

            var emrePerson = await _personRepository.SingleAsync(p => p.Name == "emre");
            emrePerson.Name = "emre2";
            await _personRepository.UpdateAsync(emrePerson);

            triggerCount.ShouldBe(1);
        }

        [Fact]
        public void Should_Trigger_Event_On_Delete()
        {
            var triggerCount = 0;
            Resolve<IEventBus>().Register<EntityDeletedEventData<Person>>(
                eventData =>
                {
                    eventData.Entity.Name.ShouldBe("emre");
                    triggerCount++;
                });

            var emrePerson = _personRepository.Single(p => p.Name == "emre");
            _personRepository.Delete(emrePerson.Id);

            triggerCount.ShouldBe(1);
            _personRepository.FirstOrDefault(p => p.Name == "emre").ShouldBe(null);
        }

        [Fact]
        public async Task Should_Trigger_Event_On_Delete_Async()
        {
            var triggerCount = 0;
            Resolve<IEventBus>().Register<EntityDeletedEventData<Person>>(
                eventData =>
                {
                    eventData.Entity.Name.ShouldBe("emre");
                    triggerCount++;
                });

            var emrePerson = await _personRepository.SingleAsync(p => p.Name == "emre");
            await _personRepository.DeleteAsync(emrePerson.Id);

            triggerCount.ShouldBe(1);
            var deletedPerson = await _personRepository.FirstOrDefaultAsync(p => p.Name == "emre");
            deletedPerson.ShouldBeNull();
        }
    }
}