text:
from __future__ import annotations

from typing import Any, Dict, Optional, TYPE_CHECKING

from flask_babel import gettext as _
from marshmallow import EXCLUDE, fields, post_load, Schema, validate
from marshmallow.validate import Length, Range
from marshmallow_enum import EnumField

from superset import app
from superset.common.chart_data import ChartDataResultFormat, ChartDataResultType
from superset.db_engine_specs.base import builtin_time_grains
from superset.utils import schema as utils
from superset.utils.core import (
    AnnotationType,
    FilterOperator,
    PostProcessingBoxplotWhiskerType,
    PostProcessingContributionOrientation,
    TimeRangeEndpoint,
)

if TYPE_CHECKING:
    from superset.common.query_context import QueryContext
    from superset.common.query_context_factory import QueryContextFactory

config = app.config

#
# RISON/JSON schemas for query parameters
#
get_delete_ids_schema = {"type": "array", "items": {"type": "integer"}}

width_height_schema = {
    "type": "array",
    "items": {"type": "integer"},
}
thumbnail_query_schema = {
    "type": "object",
    "properties": {"force": {"type": "boolean"}},
}
screenshot_query_schema = {
    "type": "object",
    "properties": {
        "force": {"type": "boolean"},
        "window_size": width_height_schema,
        "thumb_size": width_height_schema,
    },
}
get_export_ids_schema = {"type": "array", "items": {"type": "integer"}}

get_fav_star_ids_schema = {"type": "array", "items": {"type": "integer"}}

#
# Column schema descriptions
#
slice_name_description = "The name of the chart."
description_description = "A description of the chart propose."
viz_type_description = "The type of chart visualization used."
owners_description = (
    "Owner are users ids allowed to delete or change this chart. "
    "If left empty you will be one of the owners of the chart."
)
params_description = (
    "Parameters are generated dynamically when clicking the save "
    "or overwrite button in the explore view. "
    "This JSON object for power users who may want to alter specific parameters."
)
query_context_description = (
    "The query context represents the queries that need to run "
    "in order to generate the data the visualization, and in what "
    "format the data should be returned."
)
query_context_generation_description = (
    "The query context generation represents whether the query_context"
    "is user generated or not so that it does not update user modfied"
    "state."
)
cache_timeout_description = (
    "Duration (in seconds) of the caching timeout "
    "for this chart. Note this defaults to the datasource/table"
    " timeout if undefined."
)
datasource_id_description = (
    "The id of the dataset/datasource this new chart will use. "
    "A complete datasource identification needs `datasouce_id` "
    "and `datasource_type`."
)
datasource_uid_description = (
    "The uid of the dataset/datasource this new chart will use. "
    "A complete datasource identification needs `datasouce_uid` "
)
datasource_type_description = (
    "The type of dataset/datasource identified on `datasource_id`."
)
datasource_name_description = "The datasource name."
dashboards_description = "A list of dashboards to include this new chart to."
changed_on_description = "The ISO date that the chart was last changed."
slice_url_description = "The URL of the chart."
form_data_description = (
    "Form data from the Explore controls used to form the chart's data query."
)
description_markeddown_description = "Sanitized HTML version of the chart description."
owners_name_description = "Name of an owner of the chart."
certified_by_description = "Person or group that has certified this chart"
certification_details_description = "Details of the certification"

#
# OpenAPI method specification overrides
#
openapi_spec_methods_override = {
    "get": {"get": {"description": "Get a chart detail information."}},
    "get_list": {
        "get": {
            "description": "Get a list of charts, use Rison or JSON query "
            "parameters for filtering, sorting, pagination and "
            " for selecting specific columns and metadata.",
        }
    },
    "info": {
        "get": {
            "description": "Several metadata information about chart API endpoints.",
        }
    },
    "related": {
        "get": {
            "description": "Get a list of all possible owners for a chart. "
            "Use `owners` has the `column_name` parameter"
        }
    },
}

class ChartEntityResponseSchema(Schema):
    """
    Schema for a chart object
    """

    slice_id = fields.Integer()
    slice_name = fields.String(description=slice_name_description)
    cache_timeout = fields.Integer(description=cache_timeout_description)
    changed_on = fields.String(description=changed_on_description)
    modified = fields.String()
    description = fields.String(description=description_description)
    description_markeddown = fields.String(
        description=description_markeddown_description
    )
    form_data = fields.Dict(description=form_data_description)
    slice_url = fields.String(description=slice_url_description)
    certified_by = fields.String(description=certified_by_description)
    certification_details = fields.String(description=certification_details_description)

class ChartPostSchema(Schema):
    """
    Schema to add a new chart.
    """

    slice_name = fields.String(
        description=slice_name_description, required=True, validate=Length(1, 250)
    )
    description = fields.String(description=description_description, allow_none=True)
    viz_type = fields.String(
        description=viz_type_description,
        validate=Length(0, 250),
        example=["bar", "line_multi", "area", "table"],
    )
    owners = fields.List(fields.Integer(description=owners_description))
    params = fields.String(
        description=params_description, allow_none=True, validate=utils.validate_json
    )
    query_context = fields.String(
        description=query_context_description,
        allow_none=True,
        validate=utils.validate_json,
    )
    query_context_generation = fields.Boolean(
        description=query_context_generation_description, allow_none=True
    )
    cache_timeout = fields.Integer(
        description=cache_timeout_description, allow_none=True
    )
    datasource_id = fields.Integer(description=datasource_id_description, required=True)
    datasource_type = fields.String(
        description=datasource_type_description,
        validate=validate.OneOf(choices=("druid", "table", "view")),
        required=True,
    )
    datasource_name = fields.String(
        description=datasource_name_description, allow_none=True
    )
    dashboards = fields.List(fields.Integer(description=dashboards_description))
    certified_by = fields.String(description=certified_by_description, allow_none=True)
    certification_details = fields.String(
        description=certification_details_description, allow_none=True
    )

class ChartPutSchema(Schema):
    """
    Schema to update or patch a chart
    """

    slice_name = fields.String(
        description=slice_name_description, allow_none=True, validate=Length(0, 250)
    )
    description = fields.String(description=description_description, allow_none=True)
    viz_type = fields.String(
        description=viz_type_description,
        allow_none=True,
        validate=Length(0, 250),
        example=["bar", "line_multi", "area", "table"],
    )
    owners = fields.List(fields.Integer(description=owners_description))
    params = fields.String(description=params_description, allow_none=True)
    query_context = fields.String(
        description=query_context_description, allow_none=True
    )
    query_context_generation = fields.Boolean(
        description=query_context_generation_description, allow_none=True
    )
    cache_timeout = fields.Integer(
        description=cache_timeout_description, allow_none=True
    )
    datasource_id = fields.Integer(
        description=datasource_id_description, allow_none=True
    )
    datasource_type = fields.String(
        description=datasource_type_description,
        validate=validate.OneOf(choices=("druid", "table", "view")),
        allow_none=True,
    )
    dashboards = fields.List(fields.Integer(description=dashboards_description))
    certified_by = fields.String(description=certified_by_description, allow_none=True)
    certification_details = fields.String(
        description=certification_details_description, allow_none=True
    )

class ChartGetDatasourceObjectDataResponseSchema(Schema):
    datasource_id = fields.Integer(description="The datasource identifier")
    datasource_type = fields.Integer(description="The datasource type")

class ChartGetDatasourceObjectResponseSchema(Schema):
    label = fields.String(description="The name of the datasource")
    value = fields.Nested(ChartGetDatasourceObjectDataResponseSchema)

class ChartGetDatasourceResponseSchema(Schema):
    count = fields.Integer(description="The total number of datasources")
    result = fields.Nested(ChartGetDatasourceObjectResponseSchema)

class ChartCacheScreenshotResponseSchema(Schema):
    cache_key = fields.String(description="The cache key")
    chart_url = fields.String(description="The url to render the chart")
    image_url = fields.String(description="The url to fetch the screenshot")

class ChartDataColumnSchema(Schema):
    column_name = fields.String(
        description="The name of the target column", example="mycol",
    )
    type = fields.String(description="Type of target column", example="BIGINT")

class ChartDataAdhocMetricSchema(Schema):
    """
    Ad-hoc metrics are used to define metrics outside the datasource.
    """

    expressionType = fields.String(
        description="Simple or SQL metric",
        required=True,
        validate=validate.OneOf(choices=("SIMPLE", "SQL")),
        example="SQL",
    )
    aggregate = fields.String(
        description="Aggregation operator. Only required for simple expression types.",
        validate=validate.OneOf(
            choices=("AVG", "COUNT", "COUNT_DISTINCT", "MAX", "MIN", "SUM")
        ),
    )
    column = fields.Nested(ChartDataColumnSchema)
    sqlExpression = fields.String(
        description="The metric as defined by a SQL aggregate expression. "
        "Only required for SQL expression type.",
        example="SUM(weight * observations) / SUM(weight)",
    )
    label = fields.String(
        description="Label for the metric. Is automatically generated unless "
        "hasCustomLabel is true, in which case label must be defined.",
        example="Weighted observations",
    )
    hasCustomLabel = fields.Boolean(
        description="When false, the label will be automatically generated based on "
        "the aggregate expression. When true, a custom label has to be "
        "specified.",
        example=True,
    )
    optionName = fields.String(
        description="Unique identifier. Can be any string value, as long as all "
        "metrics have a unique identifier. If undefined, a random name "
        "will be generated.",
        example="metric_aec60732-fac0-4b17-b736-93f1a5c93e30",
    )
    timeGrain = fields.String(
        description="Optional time grain for temporal filters", example="PT1M",
    )
    isExtra = fields.Boolean(
        description="Indicates if the filter has been added by a filter component as "
        "opposed to being a part of the original query."
    )

class ChartDataAggregateConfigField(fields.Dict):
    def __init__(self) -> None:
        super().__init__(
            description="The keys are the name of the aggregate column to be created, "
            "and the values specify the details of how to apply the "
            "aggregation. If an operator requires additional options, "
            "these can be passed here to be unpacked in the operator call. The "
            "following numpy operators are supported: average, argmin, argmax, cumsum, "
            "cumprod, max, mean, median, nansum, nanmin, nanmax, nanmean, nanmedian, "
            "min, percentile, prod, product, std, sum, var. Any options required by "
            "the operator can be passed to the `options` object.\n"
            "\n"
            "In the example, a new column `first_quantile` is created based on values "
            "in the column `my_col` using the `percentile` operator with "
            "the `q=0.25` parameter.",
            example={
                "first_quantile": {
                    "operator": "percentile",
                    "column": "my_col",
                    "options": {"q": 0.25},
                }
            },
        )

class ChartDataPostProcessingOperationOptionsSchema(Schema):
    pass

class ChartDataAggregateOptionsSchema(ChartDataPostProcessingOperationOptionsSchema):
    """
    Aggregate operation config.
    """

    groupby = (
        fields.List(
            fields.String(
                allow_none=False, description="Columns by which to group by",
            ),
            minLength=1,
            required=True,
        ),
    )
    aggregates = ChartDataAggregateConfigField()

class ChartDataRollingOptionsSchema(ChartDataPostProcessingOperationOptionsSchema):
    """
    Rolling operation config.
    """

    columns = (
        fields.Dict(
            description="columns on which to perform rolling, mapping source column to "
            "target column. For instance, `{'y': 'y'}` will replace the "
            "column `y` with the rolling value in `y`, while `{'y': 'y2'}` "
            "will add a column `y2` based on rolling values calculated "
            "from `y`, leaving the original column `y` unchanged.",
            example={"weekly_rolling_sales": "sales"},
        ),
    )
    rolling_type = fields.String(
        description="Type of rolling window. Any numpy function will work.",
        validate=validate.OneOf(
            choices=(
                "average",
                "argmin",
                "argmax",
                "cumsum",
                "cumprod",
                "max",
                "mean",
                "median",
                "nansum",
                "nanmin",
                "nanmax",
                "nanmean",
                "nanmedian",
                "nanpercentile",
                "min",
                "percentile",
                "prod",
                "product",
                "std",
                "sum",
                "var",
            )
        ),
        required=True,
        example="percentile",
    )
    window = fields.Integer(
        description="Size of the rolling window in days.", required=True, example=7,
    )
    rolling_type_options = fields.Dict(
        desctiption="Optional options to pass to rolling method. Needed for "
        "e.g. quantile operation.",
        example={},
    )
    center = fields.Boolean(
        description="Should the label be at the center of the window. Default: `false`",
        example=False,
    )
    win_type = fields.String(
        description="Type of window function. See "
        "[SciPy window functions](https://docs.scipy.org/doc/scipy/reference"
        "/signal.windows.html#module-scipy.signal.windows) "
        "for more details. Some window functions require passing "
        "additional parameters to `rolling_type_options`. For instance, "
        "to use `gaussian`, the parameter `std` needs to be provided.",
        validate=validate.OneOf(
            choices=(
                "boxcar",
                "triang",
                "blackman",
                "hamming",
                "bartlett",
                "parzen",
                "bohman",
                "blackmanharris",
                "nuttall",
                "barthann",
                "kaiser",
                "gaussian",
                "general_gaussian",
                "slepian",
                "exponential",
            )
        ),
    )
    min_periods = fields.Integer(
        description="The minimum amount of periods required for a row to be included "
        "in the result set.",
        example=7,
    )

class ChartDataSelectOptionsSchema(ChartDataPostProcessingOperationOptionsSchema):
    """
    Sort operation config.
    """

    columns = fields.List(
        fields.String(),
        description="Columns which to select from the input data, in the desired "
        "order. If columns are renamed, the original column name should be "
        "referenced here.",
        example=["country", "gender", "age"],
    )
    exclude = fields.List(
        fields.String(),
        description="Columns to exclude from selection.",
        example=["my_temp_column"],
    )
    rename = fields.List(
        fields.Dict(),
        description="columns which to rename, mapping source column to target column. "
        "For instance, `{'y': 'y2'}` will rename the column `y` to `y2`.",
        example=[{"age": "average_age"}],
    )

class ChartDataSortOptionsSchema(ChartDataPostProcessingOperationOptionsSchema):
    """
    Sort operation config.
    """

    columns = fields.Dict(
        description="columns by by which to sort. The key specifies the column name, "
        "value specifies if sorting in ascending order.",
        example={"country": True, "gender": False},
        required=True,
    )
    aggregates = ChartDataAggregateConfigField()

class ChartDataContributionOptionsSchema(ChartDataPostProcessingOperationOptionsSchema):
    """
    Contribution operation config.
    """

    orientation = fields.String(
        description="Should cell values be calculated across the row or column.",
        required=True,
        validate=validate.OneOf(
            choices=[val.value for val in PostProcessingContributionOrientation]
        ),
        example="row",
    )

class ChartDataProphetOptionsSchema(ChartDataPostProcessingOperationOptionsSchema):
    """
    Prophet operation config.
    """

    time_grain = fields.String(
        description="Time grain used to specify time period increments in prediction. "
        "Supports [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) "
        "durations.",
        validate=validate.OneOf(
            choices=[
                i
                for i in {**builtin_time_grains, **config["TIME_GRAIN_ADDONS"]}.keys()
                if i
            ]
        ),
        example="P1D",
        required=True,
    )
    periods = fields.Integer(
        descrption="Time periods (in units of `time_grain`) to predict into the future",
        min=0,
        example=7,
        required=True,
    )
    confidence_interval = fields.Float(
        description="Width of predicted confidence interval",
        validate=[
            Range(
                min=0,
                max=1,
                min_inclusive=False,
                max_inclusive=False,
                error=_("`confidence_interval` must be between 0 and 1 (exclusive)"),
            )
        ],
        example=0.8,
        required=True,
    )
    yearly_seasonality = fields.Raw(
        # TODO: add correct union type once supported by Marshmallow
        description="Should yearly seasonality be applied. "
        "An integer value will specify Fourier order of seasonality, `None` will "
        "automatically detect seasonality.",
        example=False,
    )
    weekly_seasonality = fields.Raw(
        # TODO: add correct union type once supported by Marshmallow
        description="Should weekly seasonality be applied. "
        "An integer value will specify Fourier order of seasonality, `None` will "
        "automatically detect seasonality.",
        example=False,
    )
    monthly_seasonality = fields.Raw(
        # TODO: add correct union type once supported by Marshmallow
        description="Should monthly seasonality be applied. "
        "An integer value will specify Fourier order of seasonality, `None` will "
        "automatically detect seasonality.",
        example=False,
    )

class ChartDataBoxplotOptionsSchema(ChartDataPostProcessingOperationOptionsSchema):
    """
    Boxplot operation config.
    """

    groupby = fields.List(
        fields.String(description="Columns by which to group the query.",),
        allow_none=True,
    )

    metrics = fields.List(
        fields.Raw(),
        description="Aggregate expressions. Metrics can be passed as both "
        "references to datasource metrics (strings), or ad-hoc metrics"
        "which are defined only within the query object. See "
        "`ChartDataAdhocMetricSchema` for the structure of ad-hoc metrics.",
        allow_none=True,
    )

    whisker_type = fields.String(
        description="Whisker type. Any numpy function will work.",
        validate=validate.OneOf(
            choices=([val.value for val in PostProcessingBoxplotWhiskerType])
        ),
        required=True,
        example="tukey",
    )

    percentiles = fields.Tuple(
        (
            fields.Float(
                description="Lower percentile",
                validate=[
                    Range(
                        min=0,
                        max=100,
                        min_inclusive=False,
                        max_inclusive=False,
                        error=_(
                            "lower percentile must be greater than 0 and less "
                            "than 100. Must be lower than upper percentile."
                        ),
                    ),
                ],
            ),
            fields.Float(
                description="Upper percentile",
                validate=[
                    Range(
                        min=0,
                        max=100,
                        min_inclusive=False,
                        max_inclusive=False,
                        error=_(
                            "upper percentile must be greater than 0 and less "
                            "than 100. Must be higher than lower percentile."
                        ),
                    ),
                ],
            ),
        ),
        description="Upper and lower percentiles for percentile whisker type.",
        example=[1, 99],
    )

class ChartDataPivotOptionsSchema(ChartDataPostProcessingOperationOptionsSchema):
    """
    Pivot operation config.
    """

    index = (
        fields.List(
            fields.String(allow_none=False),
            description="Columns to group by on the table index (=rows)",
            minLength=1,
            required=True,
        ),
    )
    columns = fields.List(
        fields.String(allow_none=False),
        description="Columns to group by on the table columns",
    )
    metric_fill_value = fields.Number(
        description="Value to replace missing values with in aggregate calculations.",
    )
    column_fill_value = fields.String(
        description="Value to replace missing pivot columns names with."
    )
    drop_missing_columns = fields.Boolean(
        description="Do not include columns whose entries are all missing "
        "(default: `true`).",
    )
    marginal_distributions = fields.Boolean(
        description="Add totals for row/column. (default: `false`)",
    )
    marginal_distribution_name = fields.String(
        description="Name of marginal distribution row/column. (default: `All`)",
    )
    aggregates = ChartDataAggregateConfigField()

class ChartDataGeohashDecodeOptionsSchema(
    ChartDataPostProcessingOperationOptionsSchema
):
    """
    Geohash decode operation config.
    """

    geohash = fields.String(
        description="Name of source column containing geohash string", required=True,
    )
    latitude = fields.String(
        description="Name of target column for decoded latitude", required=True,
    )
    longitude = fields.String(
        description="Name of target column for decoded longitude", required=True,
    )

class ChartDataGeohashEncodeOptionsSchema(
    ChartDataPostProcessingOperationOptionsSchema
):
    """
    Geohash encode operation config.
    """

    latitude = fields.String(
        description="Name of source latitude column", required=True,
    )
    longitude = fields.String(
        description="Name of source longitude column", required=True,
    )
    geohash = fields.String(
        description="Name of target column for encoded geohash string", required=True,
    )

class ChartDataGeodeticParseOptionsSchema(
    ChartDataPostProcessingOperationOptionsSchema
):
    """
    Geodetic point string parsing operation config.
    """

    geodetic = fields.String(
        description="Name of source column containing geodetic point strings",
        required=True,
    )
    latitude = fields.String(
        description="Name of target column for decoded latitude", required=True,
    )
    longitude = fields.String(
        description="Name of target column for decoded longitude", required=True,
    )
    altitude = fields.String(
        description="Name of target column for decoded altitude. If omitted, "
        "altitude information in geodetic string is ignored.",
    )

class ChartDataPostProcessingOperationSchema(Schema):
    operation = fields.String(
        description="Post processing operation type",
        required=True,
        validate=validate.OneOf(
            choices=(
                "aggregate",
                "boxplot",
                "contribution",
                "cum",
                "geodetic_parse",
                "geohash_decode",
                "geohash_encode",
                "pivot",
                "prophet",
                "rolling",
                "select",
                "sort",
                "diff",
                "compare",
                "resample",
            )
        ),
        example="aggregate",
    )
    options = fields.Dict(
        description="Options specifying how to perform the operation. Please refer "
        "to the respective post processing operation option schemas. "
        "For example, `ChartDataPostProcessingOperationOptions` specifies "
        "the required options for the pivot operation.",
        example={
            "groupby": ["country", "gender"],
            "aggregates": {
                "age_q1": {
                    "operator": "percentile",
                    "column": "age",
                    "options": {"q": 0.25},
                },
                "age_mean": {"operator": "mean", "column": "age",},
            },
        },
    )

class ChartDataFilterSchema(Schema):
    col = fields.Raw(
        description="The column to filter by. Can be either a string (physical or "
        "saved expression) or an object (adhoc column)",
        required=True,
        example="country",
    )
    op = fields.String(  # pylint: disable=invalid-name
        description="The comparison operator.",
        validate=utils.OneOfCaseInsensitive(
            choices=[filter_op.value for filter_op in FilterOperator]
        ),
        required=True,
        example="IN",
    )
    val = fields.Raw(
        description="The value or values to compare against. Can be a string, "
        "integer, decimal or list, depending on the operator.",
        example=["China", "France", "Japan"],
    )
    grain = fields.String(
        description="Optional time grain for temporal filters", example="PT1M",
    )
    isExtra = fields.Boolean(
        description="Indicates if the filter has been added by a filter component as "
        "opposed to being a part of the original query."
    )

class ChartDataExtrasSchema(Schema):

    time_range_endpoints = fields.List(EnumField(TimeRangeEndpoint, by_value=True))
    relative_start = fields.String(
        description="Start time for relative time deltas. "
        'Default: `config["DEFAULT_RELATIVE_START_TIME"]`',
        validate=validate.OneOf(choices=("today", "now")),
    )
    relative_end = fields.String(
        description="End time for relative time deltas. "
        'Default: `config["DEFAULT_RELATIVE_START_TIME"]`',
        validate=validate.OneOf(choices=("today", "now")),
    )
    where = fields.String(
        description="WHERE clause to be added to queries using AND operator.",
    )
    having = fields.String(
        description="HAVING clause to be added to aggregate queries using "
        "AND operator.",
    )
    having_druid = fields.List(
        fields.Nested(ChartDataFilterSchema),
        description="HAVING filters to be added to legacy Druid datasource queries.",
    )
    time_grain_sqla = fields.String(
        description="To what level of granularity should the temporal column be "
        "aggregated. Supports "
        "[ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations) durations.",
        validate=validate.OneOf(
            choices=[
                i
                for i in {**builtin_time_grains, **config["TIME_GRAIN_ADDONS"]}.keys()
                if i
            ]
        ),
        example="P1D",
        allow_none=True,
    )
    druid_time_origin = fields.String(
        description="Starting point for time grain counting on legacy Druid "
        "datasources. Used to change e.g. Monday/Sunday first-day-of-week.",
        allow_none=True,
    )

class AnnotationLayerSchema(Schema):
    annotationType = fields.String(
        description="Type of annotation layer",
        validate=validate.OneOf(choices=[ann.value for ann in AnnotationType]),
    )
    color = fields.String(description="Layer color", allow_none=True,)
    descriptionColumns = fields.List(
        fields.String(),
        description="Columns to use as the description. If none are provided, "
        "all will be shown.",
    )
    hideLine = fields.Boolean(
        description="Should line be hidden. Only applies to line annotations",
        allow_none=True,
    )
    intervalEndColumn = fields.String(
        description=(
            "Column containing end of interval. Only applies to interval layers"
        ),
        allow_none=True,
    )
    name = fields.String(description="Name of layer", required=True)
    opacity = fields.String(
        description="Opacity of layer",
        validate=validate.OneOf(
            choices=("", "opacityLow", "opacityMedium", "opacityHigh"),
        ),
        allow_none=True,
        required=False,
    )
    overrides = fields.Dict(
        keys=fields.String(
            desciption="Name of property to be overridden",
            validate=validate.OneOf(
                choices=("granularity", "time_grain_sqla", "time_range", "time_shift"),
            ),
        ),
        values=fields.Raw(allow_none=True),
        description="which properties should be overridable",
        allow_none=True,
    )
    show = fields.Boolean(description="Should the layer be shown", required=True)
    showLabel = fields.Boolean(
        description="Should the label always be shown", allow_none=True,
    )
    showMarkers = fields.Boolean(
        description="Should markers be shown. Only applies to line annotations.",
        required=True,
    )
    sourceType = fields.String(
        description="Type of source for annotation data",
        validate=validate.OneOf(choices=("", "line", "NATIVE", "table",)),
    )
    style = fields.String(
        description="Line style. Only applies to time-series annotations",
        validate=validate.OneOf(choices=("dashed", "dotted", "solid", "longDashed",)),
    )
    timeColumn = fields.String(
        description="Column with event date or interval start date", allow_none=True,
    )
    titleColumn = fields.String(description="Column with title", allow_none=True,)
    width = fields.Float(
        description="Width of annotation line",
        validate=[
            Range(
                min=0,
                min_inclusive=True,
                error=_("`width` must be greater or equal to 0"),
            )
        ],
    )
    value = fields.Raw(
        description="For formula annotations, this contains the formula. "
        "For other types, this is the primary key of the source object.",
        required=True,
    )

class ChartDataDatasourceSchema(Schema):
    description = "Chart datasource"
    id = fields.Integer(description="Datasource id", required=True,)
    type = fields.String(
        description="Datasource type",
        validate=validate.OneOf(choices=("druid", "table")),
    )

class ChartDataQueryObjectSchema(Schema):
    class Meta:  # pylint: disable=too-few-public-methods
        unknown = EXCLUDE

    datasource = fields.Nested(ChartDataDatasourceSchema, allow_none=True)
    result_type = EnumField(ChartDataResultType, by_value=True, allow_none=True)

    annotation_layers = fields.List(
        fields.Nested(AnnotationLayerSchema),
        description="Annotation layers to apply to chart",
        allow_none=True,
    )
    applied_time_extras = fields.Dict(
        description="A mapping of temporal extras that have been applied to the query",
        allow_none=True,
        example={"__time_range": "1 year ago : now"},
    )
    apply_fetch_values_predicate = fields.Boolean(
        description="Add fetch values predicate (where clause) to query "
        "if defined in datasource",
        allow_none=True,
    )
    filters = fields.List(fields.Nested(ChartDataFilterSchema), allow_none=True)
    granularity = fields.String(
        description="Name of temporal column used for time filtering. For legacy Druid "
        "datasources this defines the time grain.",
        allow_none=True,
    )
    granularity_sqla = fields.String(
        description="Name of temporal column used for time filtering for SQL "
        "datasources. This field is deprecated, use `granularity` "
        "instead.",
        allow_none=True,
        deprecated=True,
    )
    groupby = fields.List(
        fields.Raw(),
        description="Columns by which to group the query. "
        "This field is deprecated, use `columns` instead.",
        allow_none=True,
    )
    metrics = fields.List(
        fields.Raw(),
        description="Aggregate expressions. Metrics can be passed as both "
        "references to datasource metrics (strings), or ad-hoc metrics"
        "which are defined only within the query object. See "
        "`ChartDataAdhocMetricSchema` for the structure of ad-hoc metrics.",
        allow_none=True,
    )
    post_processing = fields.List(
        fields.Nested(ChartDataPostProcessingOperationSchema, allow_none=True),
        allow_none=True,
        description="Post processing operations to be applied to the result set. "
        "Operations are applied to the result set in sequential order.",
    )
    time_range = fields.String(
        description="A time rage, either expressed as a colon separated string "
        "`since : until` or human readable freeform. Valid formats for "
        "`since` and `until` are: \n"
        "- ISO 8601\n"
        "- X days/years/hours/day/year/weeks\n"
        "- X days/years/hours/day/year/weeks ago\n"
        "- X days/years/hours/day/year/weeks from now\n"
        "\n"
        "Additionally, the following freeform can be used:\n"
        "\n"
        "- Last day\n"
        "- Last week\n"
        "- Last month\n"
        "- Last quarter\n"
        "- Last year\n"
        "- No filter\n"
        "- Last X seconds/minutes/hours/days/weeks/months/years\n"
        "- Next X seconds/minutes/hours/days/weeks/months/years\n",
        example="Last week",
        allow_none=True,
    )
    time_shift = fields.String(
        description="A human-readable date/time string. "
        "Please refer to [parsdatetime](https://github.com/bear/parsedatetime) "
        "documentation for details on valid values.",
        allow_none=True,
    )
    is_timeseries = fields.Boolean(
        description="Is the `query_object` a timeseries.", allow_none=True,
    )
    series_columns = fields.List(
        fields.Raw(),
        description="Columns to use when limiting series count. "
        "All columns must be present in the `columns` property. "
        "Requires `series_limit` and `series_limit_metric` to be set.",
        allow_none=True,
    )
    series_limit = fields.Integer(
        description="Maximum number of series. "
        "Requires `series` and `series_limit_metric` to be set.",
        allow_none=True,
    )
    series_limit_metric = fields.Raw(
        description="Metric used to limit timeseries queries by. "
        "Requires `series` and `series_limit` to be set.",
        allow_none=True,
    )
    timeseries_limit = fields.Integer(
        description="Maximum row count for timeseries queries. "
        "This field is deprecated, use `series_limit` instead."
        "Default: `0`",
        allow_none=True,
    )
    timeseries_limit_metric = fields.Raw(
        description="Metric used to limit timeseries queries by. "
        "This field is deprecated, use `series_limit_metric` instead.",
        allow_none=True,
    )
    row_limit = fields.Integer(
        description='Maximum row count (0=disabled). Default: `config["ROW_LIMIT"]`',
        allow_none=True,
        validate=[
            Range(min=0, error=_("`row_limit` must be greater than or equal to 0"))
        ],
    )
    row_offset = fields.Integer(
        description="Number of rows to skip. Default: `0`",
        allow_none=True,
        validate=[
            Range(min=0, error=_("`row_offset` must be greater than or equal to 0"))
        ],
    )
    order_desc = fields.Boolean(
        description="Reverse order. Default: `false`", allow_none=True,
    )
    extras = fields.Nested(
        ChartDataExtrasSchema,
        description="Extra parameters to add to the query.",
        allow_none=True,
    )
    columns = fields.List(
        fields.Raw(),
        description="Columns which to select in the query.",
        allow_none=True,
    )
    orderby = fields.List(
        fields.Tuple(
            (
                fields.Raw(
                    validate=[
                        Length(min=1, error=_("orderby column must be populated"))
                    ],
                    allow_none=False,
                ),
                fields.Boolean(),
            )
        ),
        description="Expects a list of lists where the first element is the column "
        "name which to sort by, and the second element is a boolean.",
        allow_none=True,
        example=[("my_col_1", False), ("my_col_2", True)],
    )
    where = fields.String(
        description="WHERE clause to be added to queries using AND operator."
        "This field is deprecated and should be passed to `extras`.",
        allow_none=True,
        deprecated=True,
    )
    having = fields.String(
        description="HAVING clause to be added to aggregate queries using "
        "AND operator. This field is deprecated and should be passed "
        "to `extras`.",
        allow_none=True,
        deprecated=True,
    )
    having_filters = fields.List(
        fields.Nested(ChartDataFilterSchema),
        description="HAVING filters to be added to legacy Druid datasource queries. "
        "This field is deprecated and should be passed to `extras` "
        "as `having_druid`.",
        allow_none=True,
        deprecated=True,
    )
    druid_time_origin = fields.String(
        description="Starting point for time grain counting on legacy Druid "
        "datasources. Used to change e.g. Monday/Sunday first-day-of-week. "
        "This field is deprecated and should be passed to `extras` "
        "as `druid_time_origin`.",
        allow_none=True,
    )
    url_params = fields.Dict(
        description="Optional query parameters passed to a dashboard or Explore view",
        keys=fields.String(description="The query parameter"),
        values=fields.String(description="The value of the query parameter"),
        allow_none=True,
    )
    is_rowcount = fields.Boolean(
        description="Should the rowcount of the actual query be returned",
        allow_none=True,
    )
    time_offsets = fields.List(fields.String(), allow_none=True,)

class ChartDataQueryContextSchema(Schema):
    query_context_factory: Optional[QueryContextFactory] = None
    datasource = fields.Nested(ChartDataDatasourceSchema)
    queries = fields.List(fields.Nested(ChartDataQueryObjectSchema))
    force = fields.Boolean(
        description="Should the queries be forced to load from the source. "
        "Default: `false`",
    )

    result_type = EnumField(ChartDataResultType, by_value=True)
    result_format = EnumField(ChartDataResultFormat, by_value=True)

    form_data = fields.Raw(allow_none=True, required=False)

    # pylint: disable=unused-argument
    @post_load
    def make_query_context(self, data: Dict[str, Any], **kwargs: Any) -> QueryContext:
        query_context = self.get_query_context_factory().create(**data)
        return query_context

    def get_query_context_factory(self) -> QueryContextFactory:
        if self.query_context_factory is None:
            # pylint: disable=import-outside-toplevel
            from superset.common.query_context_factory import QueryContextFactory

            self.query_context_factory = QueryContextFactory()
        return self.query_context_factory

class AnnotationDataSchema(Schema):
    columns = fields.List(
        fields.String(),
        description="columns available in the annotation result",
        required=True,
    )
    records = fields.List(
        fields.Dict(keys=fields.String(),),
        description="records mapping the column name to it's value",
        required=True,
    )

class ChartDataResponseResult(Schema):
    annotation_data = fields.List(
        fields.Dict(
            keys=fields.String(description="Annotation layer name"),
            values=fields.String(),
        ),
        description="All requested annotation data",
        allow_none=True,
    )
    cache_key = fields.String(
        description="Unique cache key for query object", required=True, allow_none=True,
    )
    cached_dttm = fields.String(
        description="Cache timestamp", required=True, allow_none=True,
    )
    cache_timeout = fields.Integer(
        description="Cache timeout in following order: custom timeout, datasource "
        "timeout, default config timeout.",
        required=True,
        allow_none=True,
    )
    error = fields.String(description="Error", allow_none=True,)
    is_cached = fields.Boolean(
        description="Is the result cached", required=True, allow_none=None,
    )
    query = fields.String(
        description="The executed query statement", required=True, allow_none=False,
    )
    status = fields.String(
        description="Status of the query",
        validate=validate.OneOf(
            choices=(
                "stopped",
                "failed",
                "pending",
                "running",
                "scheduled",
                "success",
                "timed_out",
            )
        ),
        allow_none=False,
    )
    stacktrace = fields.String(
        desciption="Stacktrace if there was an error", allow_none=True,
    )
    rowcount = fields.Integer(
        description="Amount of rows in result set", allow_none=False,
    )
    data = fields.List(fields.Dict(), description="A list with results")
    colnames = fields.List(fields.String(), description="A list of column names")
    coltypes = fields.List(
        fields.Integer(), description="A list of generic data types of each column"
    )
    applied_filters = fields.List(
        fields.Dict(), description="A list with applied filters"
    )
    rejected_filters = fields.List(
        fields.Dict(), description="A list with rejected filters"
    )
    from_dttm = fields.Integer(
        desciption="Start timestamp of time range", required=False, allow_none=True
    )
    to_dttm = fields.Integer(
        desciption="End timestamp of time range", required=False, allow_none=True
    )

class ChartDataResponseSchema(Schema):
    result = fields.List(
        fields.Nested(ChartDataResponseResult),
        description="A list of results for each corresponding query in the request.",
    )

class ChartDataAsyncResponseSchema(Schema):
    channel_id = fields.String(
        description="Unique session async channel ID", allow_none=False,
    )
    job_id = fields.String(description="Unique async job ID", allow_none=False,)
    user_id = fields.String(description="Requesting user ID", allow_none=True,)
    status = fields.String(description="Status value for async job", allow_none=False,)
    result_url = fields.String(
        description="Unique result URL for fetching async query data", allow_none=False,
    )

class ChartFavStarResponseResult(Schema):
    id = fields.Integer(description="The Chart id")
    value = fields.Boolean(description="The FaveStar value")

class GetFavStarIdsSchema(Schema):
    result = fields.List(
        fields.Nested(ChartFavStarResponseResult),
        description="A list of results for each corresponding chart in the request",
    )

class ImportV1ChartSchema(Schema):
    slice_name = fields.String(required=True)
    viz_type = fields.String(required=True)
    params = fields.Dict()
    query_context = fields.String(allow_none=True, validate=utils.validate_json)
    cache_timeout = fields.Integer(allow_none=True)
    uuid = fields.UUID(required=True)
    version = fields.String(required=True)
    dataset_uuid = fields.UUID(required=True)

CHART_SCHEMAS = (
    ChartDataQueryContextSchema,
    ChartDataResponseSchema,
    ChartDataAsyncResponseSchema,
    # TODO: These should optimally be included in the QueryContext schema as an `anyOf`
    #  in ChartDataPostPricessingOperation.options, but since `anyOf` is not
    #  by Marshmallow<3, this is not currently possible.
    ChartDataAdhocMetricSchema,
    ChartDataAggregateOptionsSchema,
    ChartDataContributionOptionsSchema,
    ChartDataProphetOptionsSchema,
    ChartDataBoxplotOptionsSchema,
    ChartDataPivotOptionsSchema,
    ChartDataRollingOptionsSchema,
    ChartDataSelectOptionsSchema,
    ChartDataSortOptionsSchema,
    ChartDataGeohashDecodeOptionsSchema,
    ChartDataGeohashEncodeOptionsSchema,
    ChartDataGeodeticParseOptionsSchema,
    ChartEntityResponseSchema,
    ChartGetDatasourceResponseSchema,
    ChartCacheScreenshotResponseSchema,
    GetFavStarIdsSchema,
)


text:
/** Is the app running in dev mode? */
declare const __DEV__: boolean

/** The OAuth client id the app should use */
declare const __OAUTH_CLIENT_ID__: string | undefined

/** The OAuth secret the app should use. */
declare const __OAUTH_SECRET__: string | undefined

/** Is the app being built to run on Darwin? */
declare const __DARWIN__: boolean

/** Is the app being built to run on Win32? */
declare const __WIN32__: boolean

/** Is the app being built to run on Linux? */
declare const __LINUX__: boolean

/**
 * The commit id of the repository HEAD at build time.
 * Represented as a 40 character SHA-1 hexadecimal digest string.
 */
declare const __SHA__: string

/** The channel for which the release was created. */
declare const __RELEASE_CHANNEL__:
  | 'production'
  | 'beta'
  | 'test'
  | 'development'

/** The URL for Squirrel's updates. */
declare const __UPDATES_URL__: string

/**
 * The currently executing process kind, this is specific to desktop
 * and identifies the processes that we have.
 */
declare const __PROCESS_KIND__: 'main' | 'ui' | 'crash' | 'askpass'

/**
 * The DOMHighResTimeStamp type is a double and is used to store a time value.
 *
 * The value could be a discrete point in time or the difference in time between
 * two discrete points in time. The unit is milliseconds and should be accurate
 * to 5 µs (microseconds). However, if the browser is unable to provide a time
 * value accurate to 5 microseconds (due, for example, to hardware or software
 * constraints), the browser can represent the value as a time in milliseconds
 * accurate to a millisecond.
 *
 * See https://developer.mozilla.org/en-US/docs/Web/API/DOMHighResTimeStamp
 */
declare type DOMHighResTimeStamp = number

/**
 * The IdleDeadline interface is used as the data type of the input parameter to
 * idle callbacks established by calling Window.requestIdleCallback(). It offers
 * a method, timeRemaining(), which lets you determine how much longer the user
 * agent estimates it will remain idle and a property, didTimeout, which lets
 * you determine if your callback is executing because its timeout duration
 * expired.
 *
 * https://developer.mozilla.org/en-US/docs/Web/API/IdleDeadline
 */
interface IdleDeadline {
  readonly didTimeout: boolean
  readonly timeRemaining: () => DOMHighResTimeStamp
}

/**
 * Contains optional configuration parameters for the requestIdleCallback
 * function.
 *
 * See https://developer.mozilla.org/en-US/docs/Web/API/Window/requestIdleCallback
 */
interface IdleCallbackOptions {
  /**
   * If timeout is specified and has a positive value, and the callback has not
   * already been called by the time timeout milliseconds have passed, the
   * timeout will be called during the next idle period, even if doing so risks
   * causing a negative performance impact..
   */
  readonly timeout: number
}

/**
 * The window.requestIdleCallback() method queues a function to be called during
 * a browser's idle periods. This enables developers to perform background and
 * low priority work on the main event loop, without impacting latency-critical
 * events such as animation and input response. Functions are generally called
 * in first-in-first-out order; however, callbacks which have a timeout
 * specified may be called out-of-order if necessary in order to run them before
 * the timeout elapses.
 *
 * See https://developer.mozilla.org/en-US/docs/Web/API/Window/requestIdleCallback
 *
 * @param options Contains optional configuration parameters. Currently only one
 *                property is defined:
 *                  timeout:
 */
declare function requestIdleCallback(
  fn: (deadline: IdleDeadline) => void,
  options?: IdleCallbackOptions
): number

interface IDesktopLogger {
  /**
   * Writes a log message at the 'error' level.
   *
   * The error will be persisted to disk as long as the disk transport is
   * configured to pass along log messages at this level. For more details
   * about the on-disk transport, see log.ts in the main process.
   *
   * If used from a renderer the log message will also be appended to the
   * devtools console.
   *
   * @param message The text to write to the log file
   * @param error   An optional error instance that will be formatted to
   *                include the stack trace (if one is available) and
   *                then appended to the log message.
   */
  error(message: string, error?: Error): void

  /**
   * Writes a log message at the 'warn' level.
   *
   * The error will be persisted to disk as long as the disk transport is
   * configured to pass along log messages at this level. For more details
   * about the on-disk transport, see log.ts in the main process.
   *
   * If used from a renderer the log message will also be appended to the
   * devtools console.
   *
   * @param message The text to write to the log file
   * @param error   An optional error instance that will be formatted to
   *                include the stack trace (if one is available) and
   *                then appended to the log message.
   */
  warn(message: string, error?: Error): void

  /**
   * Writes a log message at the 'info' level.
   *
   * The error will be persisted to disk as long as the disk transport is
   * configured to pass along log messages at this level. For more details
   * about the on-disk transport, see log.ts in the main process.
   *
   * If used from a renderer the log message will also be appended to the
   * devtools console.
   *
   * @param message The text to write to the log file
   * @param error   An optional error instance that will be formatted to
   *                include the stack trace (if one is available) and
   *                then appended to the log message.
   */
  info(message: string, error?: Error): void

  /**
   * Writes a log message at the 'debug' level.
   *
   * The error will be persisted to disk as long as the disk transport is
   * configured to pass along log messages at this level. For more details
   * about the on-disk transport, see log.ts in the main process.
   *
   * If used from a renderer the log message will also be appended to the
   * devtools console.
   *
   * @param message The text to write to the log file
   * @param error   An optional error instance that will be formatted to
   *                include the stack trace (if one is available) and
   *                then appended to the log message.
   */
  debug(message: string, error?: Error): void
}

declare const log: IDesktopLogger
// these changes should be pushed into the Electron declarations

declare namespace NodeJS {
  // tslint:disable-next-line:interface-name
  interface Process extends EventEmitter {
    once(event: 'uncaughtException', listener: (error: Error) => void): this
    on(event: 'uncaughtException', listener: (error: Error) => void): this
    removeListener(event: 'exit', listener: Function): this
    once(event: 'exit', listener: Function): this
  }
}

declare namespace Electron {
  // tslint:disable-next-line:interface-name
  interface MenuItem {
    readonly accelerator?: Electron.Accelerator
    readonly submenu?: Electron.Menu
    readonly role?: string
    readonly type: 'normal' | 'separator' | 'submenu' | 'checkbox' | 'radio'
  }

  // tslint:disable-next-line:interface-name
  interface RequestOptions {
    readonly method: string
    readonly url: string
    readonly headers: any
  }

  type AppleActionOnDoubleClickPref = 'Maximize' | 'Minimize' | 'None'

  // tslint:disable-next-line:interface-name
  interface SystemPreferences {
    getUserDefault(
      key: 'AppleActionOnDoubleClick',
      type: 'string'
    ): AppleActionOnDoubleClickPref
  }
}

// https://wicg.github.io/ResizeObserver/#resizeobserverentry
interface IResizeObserverEntry {
  readonly target: HTMLElement
  readonly contentRect: ClientRect
}

declare class ResizeObserver {
  public constructor(cb: (entries: ReadonlyArray<IResizeObserverEntry>) => void)

  public disconnect(): void
  public observe(e: HTMLElement): void
}

redpajama_set_name: RedPajamaGithub


text:
<?php

/**
 * Reverse a string
 *
 * @phpstub
 *
 * @param string $string
 *
 * @return string Returns the reversed string.
 */
function strrev($string)
{
}


text:
__author__ = 'darshanhegde'
"""
Reads in all the chunks of paragraph vectors generated and puts them in a single
numpy aray. Finds the nearest neighbour sentences to given sentences.
"""

import os
import sys
import numpy as np

import pickle

def pickle_load(in_file_path):
    in_file = open(in_file_path, "rU")
    data = pickle.load(in_file)
    in_file.close()
    return data

def pickle_dump(data, out_file_path):
    out_file = open(out_file_path, "w")
    pickle.dump(data, out_file)
    out_file.close()

class SentIter:

    def __init__(self, review_sent_file_path):
        self.review_sent_file_path = review_sent_file_path

    def __iter__(self):
        review_sent_file = open(self.review_sent_file_path, "rU")
        for sentence in review_sent_file:
            yield sentence.strip().split()
        review_sent_file.close()

def collect_model_files(model_folder_path, model_file_name, model_size, sentence_file_path, out_file_path):
    #count number of sentences
    SI = SentIter(sentence_file_path)
    num_sents = 0
    for sentence in SI:
        num_sents += 1
    model = np.zeros((num_sents, model_size), dtype=np.float32)
    all_files = os.listdir(model_folder_path)
    model_files = filter(lambda f_name: f_name.find(model_file_name) != -1, all_files)
    model_files_aug = [(model_file, int(model_file.split("_")[-1])) for model_file in model_files]
    model_files_aug = sorted(model_files_aug, key=lambda x: x[-1])
    model_files = [model_file for model_file, idx in model_files_aug]
    for m_idx, model_file in enumerate(model_files):
        model_chunk = pickle_load(os.path.join(model_folder_path, model_file))
        if model_chunk.shape[0] == 1024:
            model[1024*m_idx:1024*(m_idx+1), :] = model_chunk
        else:
            model[1024*m_idx:, :] = model_chunk
    pickle_dump(model, out_file_path)

class ParagraphNearestNeaighbour:

    def __init__(self, model_file_path, sentence_file_path):
        self.model = pickle_load(model_file_path)
        self.SI = SentIter(sentence_file_path)
        self.sentences = []
        for sentence in self.SI:
            self.sentences.append(sentence)

    def find_nearest_neighbours(self, test_sentece_file, out_file_path, topn):
        norm = np.linalg.norm(self.model, axis=1)
        self.model = self.model / norm[:, np.newaxis]
        test_senteces = open(test_sentece_file, "rU")
        out_file = open(out_file_path, "w")
        for test_sentence in test_senteces:
            sent_idx, sentence = test_sentence.strip().split("\t")
            print " Given sentence: ", sentence
            out_file.write(" Given: " + sentence + "\n")
            sent_idx = int(sent_idx)-1
            sent_rep = self.model[sent_idx]
            dists = np.dot(self.model, sent_rep)
            best = np.argsort(dists)[::-1][:topn+1]
            results = [(self.sentences[sim], float(dists[sim])) for sim in best]
            for sentence, score in results[1:]:
                print " ".join(sentence), score
                out_file.write(" ".join(sentence) + str(score) + "\n")
            out_file.write("\n\n")
            print "\n"
        test_senteces.close()
        out_file.close()

def main():
    collect_model_files("../model_sub/", "paragraph_model", 96, "../data/paragraph_data.txt", "../model/paragraph_model.pkl")
    PVNN = ParagraphNearestNeaighbour("../model/paragraph_model.pkl", "../data/paragraph_data.txt")
    PVNN.find_nearest_neighbours("../data/nn_sentences.txt", "../results/nn_sentences_result.txt", 5)

if __name__ == '__main__':
    main()


text:
import view from './view/subCommentInput.js'

export { view };


text:
package ru.org.linux.section.stub;

import ru.org.linux.section.Section;
import ru.org.linux.section.SectionDao;
import ru.org.linux.topic.TopicPermissionService;

import java.util.ArrayList;
import java.util.List;

public class TestSectionDaoImpl implements SectionDao {

  @Override
  public List<Section> getAllSections() {
    List<Section> sectionList = new ArrayList<>();

    sectionList.add(new Section("Section 1", false, true, 1, false, "SECTION", TopicPermissionService.POSTSCORE_UNRESTRICTED));
    sectionList.add(new Section("Section 2", false, true, 2, false, "GROUP", TopicPermissionService.POSTSCORE_UNRESTRICTED));
    sectionList.add(new Section("Section 3", true, true, 3, false, "SECTION", TopicPermissionService.POSTSCORE_UNRESTRICTED));
    sectionList.add(new Section("Section 5", false, false, 5, true, "SECTION", TopicPermissionService.POSTSCORE_UNRESTRICTED));

    return sectionList;
  }

  @Override
  public String getAddInfo(int id) {
    return new StringBuilder().append("Extended info for Section ").append(id).toString();
  }
}


text:
package org.apache.kafka.streams;

import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.config.ConfigException;
import org.apache.kafka.common.config.TopicConfig;
import org.apache.kafka.common.serialization.Deserializer;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.common.serialization.Serializer;
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.streams.errors.StreamsException;
import org.apache.kafka.streams.processor.FailOnInvalidTimestamp;
import org.apache.kafka.streams.processor.TimestampExtractor;
import org.apache.kafka.streams.processor.internals.StreamsPartitionAssignor;
import org.hamcrest.CoreMatchers;
import org.junit.Before;
import org.junit.Test;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;

import static org.apache.kafka.common.requests.IsolationLevel.READ_COMMITTED;
import static org.apache.kafka.common.requests.IsolationLevel.READ_UNCOMMITTED;
import static org.apache.kafka.streams.StreamsConfig.EXACTLY_ONCE;
import static org.apache.kafka.streams.StreamsConfig.TOPOLOGY_OPTIMIZATION;
import static org.apache.kafka.streams.StreamsConfig.adminClientPrefix;
import static org.apache.kafka.streams.StreamsConfig.consumerPrefix;
import static org.apache.kafka.streams.StreamsConfig.producerPrefix;
import static org.apache.kafka.test.StreamsTestUtils.minimalStreamsConfig;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

public class StreamsConfigTest {

    private final Properties props = new Properties();
    private StreamsConfig streamsConfig;

    @Before
    public void setUp() {
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-config-test");
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
        props.put("key.deserializer.encoding", "UTF8");
        props.put("value.deserializer.encoding", "UTF-16");
        streamsConfig = new StreamsConfig(props);
    }

    @Test(expected = ConfigException.class)
    public void shouldThrowExceptionIfApplicationIdIsNotSet() {
        props.remove(StreamsConfig.APPLICATION_ID_CONFIG);
        new StreamsConfig(props);
    }

    @Test(expected = ConfigException.class)
    public void shouldThrowExceptionIfBootstrapServersIsNotSet() {
        props.remove(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG);
        new StreamsConfig(props);
    }

    @Test
    public void testGetProducerConfigs() {
        final String clientId = "client";
        final Map<String, Object> returnedProps = streamsConfig.getProducerConfigs(clientId);
        assertEquals(returnedProps.get(ProducerConfig.CLIENT_ID_CONFIG), clientId + "-producer");
        assertEquals(returnedProps.get(ProducerConfig.LINGER_MS_CONFIG), "100");
    }

    @Test
    public void testGetConsumerConfigs() {
        final String groupId = "example-application";
        final String clientId = "client";
        final Map<String, Object> returnedProps = streamsConfig.getMainConsumerConfigs(groupId, clientId);
        assertEquals(returnedProps.get(ConsumerConfig.CLIENT_ID_CONFIG), clientId + "-consumer");
        assertEquals(returnedProps.get(ConsumerConfig.GROUP_ID_CONFIG), groupId);
        assertEquals(returnedProps.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "1000");
    }

    @Test
    public void consumerConfigMustContainStreamPartitionAssignorConfig() {
        props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 42);
        props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
        props.put(StreamsConfig.WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG, 7L);
        props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, "dummy:host");
        props.put(StreamsConfig.RETRIES_CONFIG, 10);
        props.put(StreamsConfig.adminClientPrefix(StreamsConfig.RETRIES_CONFIG), 5);
        props.put(StreamsConfig.topicPrefix(TopicConfig.SEGMENT_BYTES_CONFIG), 100);
        final StreamsConfig streamsConfig = new StreamsConfig(props);

        final String groupId = "example-application";
        final String clientId = "client";
        final Map<String, Object> returnedProps = streamsConfig.getMainConsumerConfigs(groupId, clientId);

        assertEquals(42, returnedProps.get(StreamsConfig.REPLICATION_FACTOR_CONFIG));
        assertEquals(1, returnedProps.get(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG));
        assertEquals(StreamsPartitionAssignor.class.getName(), returnedProps.get(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG));
        assertEquals(7L, returnedProps.get(StreamsConfig.WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG));
        assertEquals("dummy:host", returnedProps.get(StreamsConfig.APPLICATION_SERVER_CONFIG));
        assertEquals(null, returnedProps.get(StreamsConfig.RETRIES_CONFIG));
        assertEquals(5, returnedProps.get(StreamsConfig.adminClientPrefix(StreamsConfig.RETRIES_CONFIG)));
        assertEquals(100, returnedProps.get(StreamsConfig.topicPrefix(TopicConfig.SEGMENT_BYTES_CONFIG)));
    }

    @Test
    public void consumerConfigMustUseAdminClientConfigForRetries() {
        props.put(StreamsConfig.adminClientPrefix(StreamsConfig.RETRIES_CONFIG), 20);
        props.put(StreamsConfig.RETRIES_CONFIG, 10);
        final StreamsConfig streamsConfig = new StreamsConfig(props);

        final String groupId = "example-application";
        final String clientId = "client";
        final Map<String, Object> returnedProps = streamsConfig.getMainConsumerConfigs(groupId, clientId);

        assertEquals(20, returnedProps.get(StreamsConfig.adminClientPrefix(StreamsConfig.RETRIES_CONFIG)));
    }

    @Test
    public void testGetMainConsumerConfigsWithMainConsumerOverridenPrefix() {
        props.put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "5");
        props.put(StreamsConfig.mainConsumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "50");
        final String groupId = "example-application";
        final String clientId = "client";
        final Map<String, Object> returnedProps = streamsConfig.getMainConsumerConfigs(groupId, clientId);
        assertEquals("50", returnedProps.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
    }

    @Test
    public void testGetRestoreConsumerConfigs() {
        final String clientId = "client";
        final Map<String, Object> returnedProps = streamsConfig.getRestoreConsumerConfigs(clientId);
        assertEquals(returnedProps.get(ConsumerConfig.CLIENT_ID_CONFIG), clientId + "-restore-consumer");
        assertNull(returnedProps.get(ConsumerConfig.GROUP_ID_CONFIG));
    }

    @Test
    public void defaultSerdeShouldBeConfigured() {
        final Map<String, Object> serializerConfigs = new HashMap<>();
        serializerConfigs.put("key.serializer.encoding", "UTF8");
        serializerConfigs.put("value.serializer.encoding", "UTF-16");
        final Serializer<String> serializer = Serdes.String().serializer();

        final String str = "my string for testing";
        final String topic = "my topic";

        serializer.configure(serializerConfigs, true);
        assertEquals("Should get the original string after serialization and deserialization with the configured encoding",
                str, streamsConfig.defaultKeySerde().deserializer().deserialize(topic, serializer.serialize(topic, str)));

        serializer.configure(serializerConfigs, false);
        assertEquals("Should get the original string after serialization and deserialization with the configured encoding",
                str, streamsConfig.defaultValueSerde().deserializer().deserialize(topic, serializer.serialize(topic, str)));
    }

    @Test
    public void shouldSupportMultipleBootstrapServers() {
        final List<String> expectedBootstrapServers = Arrays.asList("broker1:9092", "broker2:9092");
        final String bootstrapServersString = Utils.join(expectedBootstrapServers, ",");
        final Properties props = new Properties();
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "irrelevant");
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServersString);
        final StreamsConfig config = new StreamsConfig(props);

        final List<String> actualBootstrapServers = config.getList(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG);
        assertEquals(expectedBootstrapServers, actualBootstrapServers);
    }

    @Test
    public void shouldSupportPrefixedConsumerConfigs() {
        props.put(consumerPrefix(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), "earliest");
        props.put(consumerPrefix(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG), 1);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientId");
        assertEquals("earliest", consumerConfigs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
        assertEquals(1, consumerConfigs.get(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG));
    }

    @Test
    public void shouldSupportPrefixedRestoreConsumerConfigs() {
        props.put(consumerPrefix(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG), 1);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getRestoreConsumerConfigs("clientId");
        assertEquals(1, consumerConfigs.get(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG));
    }

    @Test
    public void shouldSupportPrefixedPropertiesThatAreNotPartOfConsumerConfig() {
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        props.put(consumerPrefix("interceptor.statsd.host"), "host");
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientId");
        assertEquals("host", consumerConfigs.get("interceptor.statsd.host"));
    }

    @Test
    public void shouldSupportPrefixedPropertiesThatAreNotPartOfRestoreConsumerConfig() {
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        props.put(consumerPrefix("interceptor.statsd.host"), "host");
        final Map<String, Object> consumerConfigs = streamsConfig.getRestoreConsumerConfigs("clientId");
        assertEquals("host", consumerConfigs.get("interceptor.statsd.host"));
    }

    @Test
    public void shouldSupportPrefixedPropertiesThatAreNotPartOfProducerConfig() {
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        props.put(producerPrefix("interceptor.statsd.host"), "host");
        final Map<String, Object> producerConfigs = streamsConfig.getProducerConfigs("clientId");
        assertEquals("host", producerConfigs.get("interceptor.statsd.host"));
    }

    @Test
    public void shouldSupportPrefixedProducerConfigs() {
        props.put(producerPrefix(ProducerConfig.BUFFER_MEMORY_CONFIG), 10);
        props.put(producerPrefix(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG), 1);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> configs = streamsConfig.getProducerConfigs("clientId");
        assertEquals(10, configs.get(ProducerConfig.BUFFER_MEMORY_CONFIG));
        assertEquals(1, configs.get(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG));
    }

    @Test
    public void shouldBeSupportNonPrefixedConsumerConfigs() {
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
        props.put(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG, 1);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientId");
        assertEquals("earliest", consumerConfigs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
        assertEquals(1, consumerConfigs.get(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG));
    }

    @Test
    public void shouldBeSupportNonPrefixedRestoreConsumerConfigs() {
        props.put(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG, 1);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getRestoreConsumerConfigs("groupId");
        assertEquals(1, consumerConfigs.get(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG));
    }

    @Test
    public void shouldSupportNonPrefixedProducerConfigs() {
        props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 10);
        props.put(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG, 1);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> configs = streamsConfig.getProducerConfigs("clientId");
        assertEquals(10, configs.get(ProducerConfig.BUFFER_MEMORY_CONFIG));
        assertEquals(1, configs.get(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG));
    }

    @Test
    public void shouldForwardCustomConfigsWithNoPrefixToAllClients() {
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        props.put("custom.property.host", "host");
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientId");
        final Map<String, Object> restoreConsumerConfigs = streamsConfig.getRestoreConsumerConfigs("clientId");
        final Map<String, Object> producerConfigs = streamsConfig.getProducerConfigs("clientId");
        final Map<String, Object> adminConfigs = streamsConfig.getAdminConfigs("clientId");
        assertEquals("host", consumerConfigs.get("custom.property.host"));
        assertEquals("host", restoreConsumerConfigs.get("custom.property.host"));
        assertEquals("host", producerConfigs.get("custom.property.host"));
        assertEquals("host", adminConfigs.get("custom.property.host"));
    }

    @Test
    public void shouldOverrideNonPrefixedCustomConfigsWithPrefixedConfigs() {
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        props.put("custom.property.host", "host0");
        props.put(consumerPrefix("custom.property.host"), "host1");
        props.put(producerPrefix("custom.property.host"), "host2");
        props.put(adminClientPrefix("custom.property.host"), "host3");
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientId");
        final Map<String, Object> restoreConsumerConfigs = streamsConfig.getRestoreConsumerConfigs("clientId");
        final Map<String, Object> producerConfigs = streamsConfig.getProducerConfigs("clientId");
        final Map<String, Object> adminConfigs = streamsConfig.getAdminConfigs("clientId");
        assertEquals("host1", consumerConfigs.get("custom.property.host"));
        assertEquals("host1", restoreConsumerConfigs.get("custom.property.host"));
        assertEquals("host2", producerConfigs.get("custom.property.host"));
        assertEquals("host3", adminConfigs.get("custom.property.host"));
    }

    @Test
    public void shouldSupportNonPrefixedAdminConfigs() {
        props.put(AdminClientConfig.RETRIES_CONFIG, 10);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> configs = streamsConfig.getAdminConfigs("clientId");
        assertEquals(10, configs.get(AdminClientConfig.RETRIES_CONFIG));
    }

    @Test(expected = StreamsException.class)
    public void shouldThrowStreamsExceptionIfKeySerdeConfigFails() {
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, MisconfiguredSerde.class);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        streamsConfig.defaultKeySerde();
    }

    @Test(expected = StreamsException.class)
    public void shouldThrowStreamsExceptionIfValueSerdeConfigFails() {
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, MisconfiguredSerde.class);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        streamsConfig.defaultValueSerde();
    }

    @Test
    public void shouldOverrideStreamsDefaultConsumerConfigs() {
        props.put(StreamsConfig.consumerPrefix(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), "latest");
        props.put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "10");
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientId");
        assertEquals("latest", consumerConfigs.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG));
        assertEquals("10", consumerConfigs.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
    }

    @Test
    public void shouldOverrideStreamsDefaultProducerConfigs() {
        props.put(StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG), "10000");
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> producerConfigs = streamsConfig.getProducerConfigs("clientId");
        assertEquals("10000", producerConfigs.get(ProducerConfig.LINGER_MS_CONFIG));
    }

    @Test
    public void shouldOverrideStreamsDefaultConsumerConifgsOnRestoreConsumer() {
        props.put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "10");
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getRestoreConsumerConfigs("clientId");
        assertEquals("10", consumerConfigs.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
    }

    @Test
    public void shouldResetToDefaultIfConsumerAutoCommitIsOverridden() {
        props.put(StreamsConfig.consumerPrefix(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), "true");
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("a", "b");
        assertEquals("false", consumerConfigs.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
    }

    @Test
    public void shouldResetToDefaultIfRestoreConsumerAutoCommitIsOverridden() {
        props.put(StreamsConfig.consumerPrefix(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), "true");
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getRestoreConsumerConfigs("client");
        assertEquals("false", consumerConfigs.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
    }

    @Test
    public void testGetRestoreConsumerConfigsWithRestoreConsumerOverridenPrefix() {
        props.put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "5");
        props.put(StreamsConfig.restoreConsumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "50");
        final Map<String, Object> returnedProps = streamsConfig.getRestoreConsumerConfigs("clientId");
        assertEquals("50", returnedProps.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
    }

    @Test
    public void testGetGlobalConsumerConfigs() {
        final String clientId = "client";
        final Map<String, Object> returnedProps = streamsConfig.getGlobalConsumerConfigs(clientId);
        assertEquals(returnedProps.get(ConsumerConfig.CLIENT_ID_CONFIG), clientId + "-global-consumer");
        assertNull(returnedProps.get(ConsumerConfig.GROUP_ID_CONFIG));
    }

    @Test
    public void shouldSupportPrefixedGlobalConsumerConfigs() {
        props.put(consumerPrefix(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG), 1);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getGlobalConsumerConfigs("clientId");
        assertEquals(1, consumerConfigs.get(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG));
    }

    @Test
    public void shouldSupportPrefixedPropertiesThatAreNotPartOfGlobalConsumerConfig() {
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        props.put(consumerPrefix("interceptor.statsd.host"), "host");
        final Map<String, Object> consumerConfigs = streamsConfig.getGlobalConsumerConfigs("clientId");
        assertEquals("host", consumerConfigs.get("interceptor.statsd.host"));
    }

    @Test
    public void shouldBeSupportNonPrefixedGlobalConsumerConfigs() {
        props.put(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG, 1);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getGlobalConsumerConfigs("groupId");
        assertEquals(1, consumerConfigs.get(ConsumerConfig.METRICS_NUM_SAMPLES_CONFIG));
    }

    @Test
    public void shouldResetToDefaultIfGlobalConsumerAutoCommitIsOverridden() {
        props.put(StreamsConfig.consumerPrefix(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG), "true");
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getGlobalConsumerConfigs("client");
        assertEquals("false", consumerConfigs.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
    }

    @Test
    public void testGetGlobalConsumerConfigsWithGlobalConsumerOverridenPrefix() {
        props.put(StreamsConfig.consumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "5");
        props.put(StreamsConfig.globalConsumerPrefix(ConsumerConfig.MAX_POLL_RECORDS_CONFIG), "50");
        final Map<String, Object> returnedProps = streamsConfig.getGlobalConsumerConfigs("clientId");
        assertEquals("50", returnedProps.get(ConsumerConfig.MAX_POLL_RECORDS_CONFIG));
    }

    @Test
    public void shouldSetInternalLeaveGroupOnCloseConfigToFalseInConsumer() {
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientId");
        assertThat(consumerConfigs.get("internal.leave.group.on.close"), CoreMatchers.<Object>equalTo(false));
    }

    @Test
    public void shouldAcceptAtLeastOnce() {
        // don't use `StreamsConfig.AT_LEAST_ONCE` to actually do a useful test
        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, "at_least_once");
        new StreamsConfig(props);
    }

    @Test
    public void shouldAcceptExactlyOnce() {
        // don't use `StreamsConfig.EXACLTY_ONCE` to actually do a useful test
        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, "exactly_once");
        new StreamsConfig(props);
    }

    @Test(expected = ConfigException.class)
    public void shouldThrowExceptionIfNotAtLestOnceOrExactlyOnce() {
        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, "bad_value");
        new StreamsConfig(props);
    }

    @Test
    public void shouldResetToDefaultIfConsumerIsolationLevelIsOverriddenIfEosEnabled() {
        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
        props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "anyValue");
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientId");
        assertThat(consumerConfigs.get(ConsumerConfig.ISOLATION_LEVEL_CONFIG), equalTo(READ_COMMITTED.name().toLowerCase(Locale.ROOT)));
    }

    @Test
    public void shouldAllowSettingConsumerIsolationLevelIfEosDisabled() {
        props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT));
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientrId");
        assertThat(consumerConfigs.get(ConsumerConfig.ISOLATION_LEVEL_CONFIG), equalTo(READ_UNCOMMITTED.name().toLowerCase(Locale.ROOT)));
    }

    @Test
    public void shouldResetToDefaultIfProducerEnableIdempotenceIsOverriddenIfEosEnabled() {
        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
        props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "anyValue");
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> producerConfigs = streamsConfig.getProducerConfigs("clientId");
        assertTrue((Boolean) producerConfigs.get(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG));
    }

    @Test
    public void shouldAllowSettingProducerEnableIdempotenceIfEosDisabled() {
        props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        final Map<String, Object> producerConfigs = streamsConfig.getProducerConfigs("clientId");
        assertThat(producerConfigs.get(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG), equalTo(false));
    }

    @Test
    public void shouldSetDifferentDefaultsIfEosEnabled() {
        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
        final StreamsConfig streamsConfig = new StreamsConfig(props);

        final Map<String, Object> consumerConfigs = streamsConfig.getMainConsumerConfigs("groupId", "clientId");
        final Map<String, Object> producerConfigs = streamsConfig.getProducerConfigs("clientId");

        assertThat(consumerConfigs.get(ConsumerConfig.ISOLATION_LEVEL_CONFIG), equalTo(READ_COMMITTED.name().toLowerCase(Locale.ROOT)));
        assertTrue((Boolean) producerConfigs.get(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG));
        assertThat(producerConfigs.get(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG), equalTo(Integer.MAX_VALUE));
        assertThat(streamsConfig.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG), equalTo(100L));
    }

    @Test
    public void shouldNotOverrideUserConfigRetriesIfExactlyOnceEnabled() {
        final int numberOfRetries = 42;
        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
        props.put(ProducerConfig.RETRIES_CONFIG, numberOfRetries);
        final StreamsConfig streamsConfig = new StreamsConfig(props);

        final Map<String, Object> producerConfigs = streamsConfig.getProducerConfigs("clientId");

        assertThat(producerConfigs.get(ProducerConfig.RETRIES_CONFIG), equalTo(numberOfRetries));
    }

    @Test
    public void shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceEnabled() {
        final long commitIntervalMs = 73L;
        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
        props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, commitIntervalMs);
        final StreamsConfig streamsConfig = new StreamsConfig(props);

        assertThat(streamsConfig.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG), equalTo(commitIntervalMs));
    }

    @Test
    public void shouldUseNewConfigsWhenPresent() {
        final Properties props = minimalStreamsConfig();
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Long().getClass());
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass());
        props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, MockTimestampExtractor.class);

        final StreamsConfig config = new StreamsConfig(props);
        assertTrue(config.defaultKeySerde() instanceof Serdes.LongSerde);
        assertTrue(config.defaultValueSerde() instanceof Serdes.LongSerde);
        assertTrue(config.defaultTimestampExtractor() instanceof MockTimestampExtractor);
    }

    @Test
    public void shouldUseCorrectDefaultsWhenNoneSpecified() {
        final StreamsConfig config = new StreamsConfig(minimalStreamsConfig());
        assertTrue(config.defaultKeySerde() instanceof Serdes.ByteArraySerde);
        assertTrue(config.defaultValueSerde() instanceof Serdes.ByteArraySerde);
        assertTrue(config.defaultTimestampExtractor() instanceof FailOnInvalidTimestamp);
    }

    @Test
    public void shouldSpecifyCorrectKeySerdeClassOnError() {
        final Properties props = minimalStreamsConfig();
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, MisconfiguredSerde.class);
        final StreamsConfig config = new StreamsConfig(props);
        try {
            config.defaultKeySerde();
            fail("Test should throw a StreamsException");
        } catch (final StreamsException e) {
            assertEquals("Failed to configure key serde class org.apache.kafka.streams.StreamsConfigTest$MisconfiguredSerde", e.getMessage());
        }
    }

    @SuppressWarnings("deprecation")
    @Test
    public void shouldSpecifyCorrectValueSerdeClassOnError() {
        final Properties props = minimalStreamsConfig();
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, MisconfiguredSerde.class);
        final StreamsConfig config = new StreamsConfig(props);
        try {
            config.defaultValueSerde();
            fail("Test should throw a StreamsException");
        } catch (final StreamsException e) {
            assertEquals("Failed to configure value serde class org.apache.kafka.streams.StreamsConfigTest$MisconfiguredSerde", e.getMessage());
        }
    }

    @Test
    public void shouldThrowExceptionIfMaxInflightRequestsGreatherThanFiveIfEosEnabled() {
        props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 7);
        props.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, EXACTLY_ONCE);
        final StreamsConfig streamsConfig = new StreamsConfig(props);
        try {
            streamsConfig.getProducerConfigs("clientId");
            fail("Should throw ConfigException when Eos is enabled and maxInFlight requests exceeds 5");
        } catch (final ConfigException e) {
            assertEquals("max.in.flight.requests.per.connection can't exceed 5 when using the idempotent producer", e.getMessage());
        }
    }

    @Test
    public void shouldSpecifyNoOptimizationWhenNotExplicitlyAddedToConfigs() {
        final String expectedOptimizeConfig = "none";
        final String actualOptimizedConifig = streamsConfig.getString(TOPOLOGY_OPTIMIZATION);
        assertEquals("Optimization should be \"none\"", expectedOptimizeConfig, actualOptimizedConifig);
    }

    @Test
    public void shouldSpecifyOptimizationWhenNotExplicitlyAddedToConfigs() {
        final String expectedOptimizeConfig = "all";
        props.put(TOPOLOGY_OPTIMIZATION, "all");
        final StreamsConfig config = new StreamsConfig(props);
        final String actualOptimizedConifig = config.getString(TOPOLOGY_OPTIMIZATION);
        assertEquals("Optimization should be \"all\"", expectedOptimizeConfig, actualOptimizedConifig);
    }

    @Test(expected = ConfigException.class)
    public void shouldThrowConfigExceptionWhenOptimizationConfigNotValueInRange() {
        props.put(TOPOLOGY_OPTIMIZATION, "maybe");
        new StreamsConfig(props);
    }

    static class MisconfiguredSerde implements Serde {
        @Override
        public void configure(final Map configs, final boolean isKey) {
            throw new RuntimeException("boom");
        }

        @Override
        public void close() {}

        @Override
        public Serializer serializer() {
            return null;
        }

        @Override
        public Deserializer deserializer() {
            return null;
        }
    }

    public static class MockTimestampExtractor implements TimestampExtractor {

        @Override
        public long extract(final ConsumerRecord<Object, Object> record, final long previousTimestamp) {
            return 0;
        }
    }
}


text:
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration supports_adding_forbidden="true">
  <property>
    <name>logsearch_use_external_solr</name>
    <value>false</value>
    <display-name>Use External Solr</display-name>
    <description>Use External Solr to store logs</description>
    <value-attributes>
      <type>value-list</type>
      <overridable>false</overridable>
      <entries>
        <entry>
          <value>true</value>
          ON
        </entry>
        <entry>
          <value>false</value>
          OFF
        </entry>
      </entries>
      <selection-cardinality>1</selection-cardinality>
    </value-attributes>
    <on-ambari-upgrade add="true"/>
  </property>
  <property>
    <name>logsearch_external_solr_zk_znode</name>
    <value></value>
    <display-name>External Solr Logs Znode</display-name>
    <description>Only needed if using custom solr cloud. E.g. /logsearch_solr</description>
    <on-ambari-upgrade add="true"/>
  </property>
  <property>
    <name>logsearch_external_solr_zk_quorum</name>
    <value></value>
    <display-name>External Solr Logs ZK Quorum</display-name>
    <description>Only needed if using custom solr cloud. E.g. zk1:2181,zk2:2182</description>
    <on-ambari-upgrade add="true"/>
  </property>
  <property>
    <name>logsearch_external_solr_ssl_enabled</name>
    <value>false</value>
    <display-name>Log Search External Solr SSL Enabled</display-name>
    <description>Use SSL to connect to the external solr</description>
    <value-attributes>
      <type>value-list</type>
      <overridable>false</overridable>
      <entries>
        <entry>
          <value>true</value>
          ON
        </entry>
        <entry>
          <value>false</value>
          OFF
        </entry>
      </entries>
      <selection-cardinality>1</selection-cardinality>
    </value-attributes>
    <on-ambari-upgrade add="true"/>
  </property>
  <property>
    <name>logsearch_external_solr_kerberos_enabled</name>
    <value>false</value>
    <display-name>Log Search External Solr Kerberos Enabled</display-name>
    <description>Use Kerberos to connect to the external solr</description>
    <value-attributes>
      <type>value-list</type>
      <overridable>false</overridable>
      <entries>
        <entry>
          <value>true</value>
          ON
        </entry>
        <entry>
          <value>false</value>
          OFF
        </entry>
      </entries>
      <selection-cardinality>1</selection-cardinality>
    </value-attributes>
    <on-ambari-upgrade add="true"/>
  </property>
</configuration>


text:
ACCEPTED

#### According to
The Catalogue of Life, 3rd January 2011

#### Published in
null

#### Original name
Sinosophiopsis bartholomewii Al-Shehbaz

### Remarks
null


text:
layout: post
title: first post in md
date: 2016-04-11
---

## Hello

this is from md


text:
<?xml version="1.0" encoding="UTF-8"?>
<actions>
        <action>
            <actionName>run</actionName>
            <packagings>
                <packaging>war</packaging>
                <packaging>ear</packaging>
                <packaging>ejb</packaging>
            </packagings>
            <goals>
                <goal>jetty:run</goal>
            </goals>
            <properties>
                <skipTests>true</skipTests>
            </properties>
        </action>
        <action>
            <actionName>debug</actionName>
            <packagings>
                <packaging>war</packaging>
                <packaging>ear</packaging>
                <packaging>ejb</packaging>
            </packagings>
            <goals>
                <goal>jetty:run</goal>
            </goals>
            <properties>
                <skipTests>true</skipTests>
                <jpda.listen>maven</jpda.listen>
            </properties>
        </action>
        <action>
            <actionName>CUSTOM-mvn jetty:run -Djpda.listen=maven</actionName>
            <displayName>mvn jetty:run -Djpda.listen=maven</displayName>
            <goals>
                <goal>jetty:run</goal>
            </goals>
            <properties>
                <jpda.listen>maven</jpda.listen>
            </properties>
        </action>
    </actions>


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>org.wildfly.swarm.config.jgroups (BOM:  * : All 2.6.0.Final API)</title>
<meta name="date" content="2019-12-05">
<link rel="stylesheet" type="text/css" href="../../../../../stylesheet.css" title="Style">

org.wildfly.swarm.config.jgroups

Interfaces

ChannelConsumer

ChannelSupplier

PropertyConsumer

PropertySupplier

ProtocolConsumer

ProtocolSupplier

StackConsumer

StackSupplier

Classes

Channel

Channel.ChannelResources

Property

Protocol

Protocol.ProtocolResources

Stack

Stack.StackResources


text:
<?php

namespace Dmishh\SettingsBundle\Tests;

use Doctrine\Common\Annotations\AnnotationRegistry;
use Doctrine\ORM\Configuration;
use PHPUnit\Framework\TestCase;

abstract class AbstractTest extends TestCase
{
    /**
     * @var \Doctrine\ORM\EntityManager
     */
    protected $em;

    /**
     * {@inheritdoc}
     */
    public function setUp(): void
    {
        $this->em = $this->createEntityManager();
        $this->generateSchema();
    }

    /**
     * {@inheritdoc}
     */
    protected function tearDown(): void
    {
        $this->em->close();
    }

    protected function createEntityManager()
    {
        $config = new Configuration();
        $config->setProxyDir(sys_get_temp_dir());
        $config->setProxyNamespace('EntityProxy');
        $config->setAutoGenerateProxyClasses(true);

        AnnotationRegistry::registerFile(
            __DIR__.
            '/../vendor/doctrine/orm/lib/Doctrine/ORM/Mapping/Driver/DoctrineAnnotations.php'
        );
        $driver = new \Doctrine\ORM\Mapping\Driver\AnnotationDriver(
            new \Doctrine\Common\Annotations\AnnotationReader(),
            [__DIR__.'/../Entity']
        );
        $config->setMetadataDriverImpl($driver);

        $conn = [
            'driver' => 'pdo_sqlite',
            'memory' => true,
        ];

        $em = \Doctrine\ORM\EntityManager::create($conn, $config);

        return $em;
    }

    protected function generateSchema()
    {
        $metadatas = $this->em->getMetadataFactory()->getAllMetadata();

        if (!empty($metadatas)) {
            $tool = new \Doctrine\ORM\Tools\SchemaTool($this->em);
            $tool->dropSchema($metadatas);
            $tool->createSchema($metadatas);
        }
    }
}