text:
from unittest import mock

import pytest
from ruamel.yaml import YAML

import great_expectations.exceptions as ge_exceptions
from great_expectations import DataContext
from great_expectations.core import IDDict
from great_expectations.core.batch import (
    BatchDefinition,
    BatchRequest,
    BatchRequestBase,
)
from great_expectations.data_context.util import instantiate_class_from_config
from great_expectations.datasource.data_connector import (
    ConfiguredAssetAzureDataConnector,
)
from great_expectations.execution_engine import PandasExecutionEngine

yaml = YAML()

@pytest.fixture
def expected_config_dict():
    """Used to validate `self_check()` and `test_yaml_config()` outputs."""
    config = {
        "class_name": "ConfiguredAssetAzureDataConnector",
        "data_asset_count": 1,
        "example_data_asset_names": [
            "alpha",
        ],
        "data_assets": {
            "alpha": {
                "example_data_references": [
                    "alpha-1.csv",
                    "alpha-2.csv",
                    "alpha-3.csv",
                ],
                "batch_definition_count": 3,
            },
        },
        "example_unmatched_data_references": [],
        "unmatched_data_reference_count": 0,
    }
    return config

@pytest.fixture
def expected_batch_definitions_unsorted():
    """
    Used to validate `get_batch_definition_list_from_batch_request()` outputs.
    Input and output should maintain the same order (henced "unsorted")
    """
    expected = [
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "alex", "timestamp": "20200809", "price": "1000"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "eugene", "timestamp": "20200809", "price": "1500"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "james", "timestamp": "20200811", "price": "1009"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "abe", "timestamp": "20200809", "price": "1040"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "will", "timestamp": "20200809", "price": "1002"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "james", "timestamp": "20200713", "price": "1567"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "eugene", "timestamp": "20201129", "price": "1900"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "will", "timestamp": "20200810", "price": "1001"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "james", "timestamp": "20200810", "price": "1003"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "alex", "timestamp": "20200819", "price": "1300"}
            ),
        ),
    ]
    return expected

@pytest.fixture
def expected_batch_definitions_sorted():
    """
    Used to validate `get_batch_definition_list_from_batch_request()` outputs.
    Input should be sorted based on some criteria, resulting in some change
    between input and output.
    """
    expected = [
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "abe", "timestamp": "20200809", "price": "1040"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "alex", "timestamp": "20200819", "price": "1300"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "alex", "timestamp": "20200809", "price": "1000"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "eugene", "timestamp": "20201129", "price": "1900"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "eugene", "timestamp": "20200809", "price": "1500"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "james", "timestamp": "20200811", "price": "1009"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "james", "timestamp": "20200810", "price": "1003"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "james", "timestamp": "20200713", "price": "1567"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "will", "timestamp": "20200810", "price": "1001"}
            ),
        ),
        BatchDefinition(
            datasource_name="test_environment",
            data_connector_name="general_azure_data_connector",
            data_asset_name="TestFiles",
            batch_identifiers=IDDict(
                {"name": "will", "timestamp": "20200809", "price": "1002"}
            ),
        ),
    ]
    return expected

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys",
    return_value=["alpha-1.csv", "alpha-2.csv", "alpha-3.csv"],
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_instantiation_with_account_url_and_credential(
    mock_azure_conn, mock_list_keys, expected_config_dict
):
    my_data_connector = ConfiguredAssetAzureDataConnector(
        name="my_data_connector",
        datasource_name="FAKE_DATASOURCE_NAME",
        execution_engine=PandasExecutionEngine(),
        default_regex={
            "pattern": "alpha-(.*)\\.csv",
            "group_names": ["index"],
        },
        container="my_container",
        name_starts_with="",
        assets={"alpha": {}},
        azure_options={
            "account_url": "my_account_url.blob.core.windows.net",
            "credential": "my_credential",
        },
    )
    assert my_data_connector.self_check() == expected_config_dict

    my_data_connector._refresh_data_references_cache()
    assert my_data_connector.get_data_reference_list_count() == 3
    assert my_data_connector.get_unmatched_data_references() == []

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys",
    return_value=["alpha-1.csv", "alpha-2.csv", "alpha-3.csv"],
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_instantiation_with_conn_str_and_credential(
    mock_azure_conn, mock_list_keys, expected_config_dict
):
    my_data_connector = ConfiguredAssetAzureDataConnector(
        name="my_data_connector",
        datasource_name="FAKE_DATASOURCE_NAME",
        execution_engine=PandasExecutionEngine(),
        default_regex={
            "pattern": "alpha-(.*)\\.csv",
            "group_names": ["index"],
        },
        container="my_container",
        name_starts_with="",
        assets={"alpha": {}},
        azure_options={  # Representative of format noted in official docs
            "conn_str": "DefaultEndpointsProtocol=https;AccountName=storagesample;AccountKey=my_account_key",
            "credential": "my_credential",
        },
    )

    assert my_data_connector.self_check() == expected_config_dict

    my_data_connector._refresh_data_references_cache()
    assert my_data_connector.get_data_reference_list_count() == 3
    assert my_data_connector.get_unmatched_data_references() == []

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_instantiation_with_valid_account_url_assigns_account_name(mock_azure_conn):
    my_data_connector = ConfiguredAssetAzureDataConnector(
        name="my_data_connector",
        datasource_name="FAKE_DATASOURCE_NAME",
        execution_engine=PandasExecutionEngine(),
        default_regex={
            "pattern": "alpha-(.*)\\.csv",
            "group_names": ["index"],
        },
        container="my_container",
        name_starts_with="",
        assets={"alpha": {}},
        azure_options={
            "account_url": "my_account_url.blob.core.windows.net",
            "credential": "my_credential",
        },
    )
    assert my_data_connector._account_name == "my_account_url"

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_instantiation_with_valid_conn_str_assigns_account_name(mock_azure_conn):
    my_data_connector = ConfiguredAssetAzureDataConnector(
        name="my_data_connector",
        datasource_name="FAKE_DATASOURCE_NAME",
        execution_engine=PandasExecutionEngine(),
        default_regex={
            "pattern": "alpha-(.*)\\.csv",
            "group_names": ["index"],
        },
        container="my_container",
        name_starts_with="",
        assets={"alpha": {}},
        azure_options={  # Representative of format noted in official docs
            "conn_str": "DefaultEndpointsProtocol=https;AccountName=storagesample;AccountKey=my_account_key",
            "credential": "my_credential",
        },
    )
    assert my_data_connector._account_name == "storagesample"

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_instantiation_with_multiple_auth_methods_raises_error(
    mock_azure_conn,
):
    # Raises error in DataContext's schema validation due to having both `account_url` and `conn_str`
    with pytest.raises(AssertionError):
        ConfiguredAssetAzureDataConnector(
            name="my_data_connector",
            datasource_name="FAKE_DATASOURCE_NAME",
            execution_engine=PandasExecutionEngine(),
            default_regex={
                "pattern": "alpha-(.*)\\.csv",
                "group_names": ["index"],
            },
            container="my_container",
            name_starts_with="",
            assets={"alpha": {}},
            azure_options={
                "account_url": "account.blob.core.windows.net",
                "conn_str": "DefaultEndpointsProtocol=https;AccountName=storagesample;AccountKey=my_account_key",
                "credential": "my_credential",
            },
        )

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_instantiation_with_improperly_formatted_auth_keys_in_azure_options_raises_error(
    mock_azure_conn,
):
    # Raises error in ConfiguredAssetAzureDataConnector's constructor due to `account_url` not conforming to the expected format
    # Format: <ACCOUNT>.blob.core.windows.net
    with pytest.raises(ImportError):
        ConfiguredAssetAzureDataConnector(
            name="my_data_connector",
            datasource_name="FAKE_DATASOURCE_NAME",
            execution_engine=PandasExecutionEngine(),
            default_regex={
                "pattern": "alpha-(.*)\\.csv",
                "group_names": ["index"],
            },
            container="my_container",
            name_starts_with="",
            assets={"alpha": {}},
            azure_options={"account_url": "not_a_valid_url"},
        )

    # Raises error in ConfiguredAssetAzureDataConnector's constructor due to `conn_str` not conforming to the expected format
    # Format: Must be a variable length, semicolon-delimited string containing "AccountName=<ACCOUNT>"
    with pytest.raises(ImportError):
        ConfiguredAssetAzureDataConnector(
            name="my_data_connector",
            datasource_name="FAKE_DATASOURCE_NAME",
            execution_engine=PandasExecutionEngine(),
            default_regex={
                "pattern": "alpha-(.*)\\.csv",
                "group_names": ["index"],
            },
            container="my_container",
            name_starts_with="",
            assets={"alpha": {}},
            azure_options={"conn_str": "not_a_valid_conn_str"},
        )

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys",
    return_value=["alpha-1.csv", "alpha-2.csv", "alpha-3.csv"],
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_instantiation_with_test_yaml_config(
    mock_azure_conn,
    mock_list_keys,
    mock_emit,
    empty_data_context_stats_enabled,
    expected_config_dict,
):
    context: DataContext = empty_data_context_stats_enabled

    report_object = context.test_yaml_config(
        """
        module_name: great_expectations.datasource.data_connector
        class_name: ConfiguredAssetAzureDataConnector
        datasource_name: FAKE_DATASOURCE
        name: TEST_DATA_CONNECTOR
        default_regex:
            pattern: alpha-(.*)\\.csv
            group_names:
                - index
        container: my_container
        name_starts_with: ""
        assets:
            alpha:
        azure_options:
            account_url: my_account_url.blob.core.windows.net
            credential: my_credential
    """,
        runtime_environment={
            "execution_engine": PandasExecutionEngine(),
        },
        return_mode="report_object",
    )

    assert report_object == expected_config_dict

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys",
    return_value=["alpha-1.csv", "alpha-2.csv", "alpha-3.csv"],
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_instantiation_with_test_yaml_config_emits_proper_payload(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    context: DataContext = empty_data_context_stats_enabled

    context.test_yaml_config(
        """
        module_name: great_expectations.datasource.data_connector
        class_name: ConfiguredAssetAzureDataConnector
        datasource_name: FAKE_DATASOURCE
        name: TEST_DATA_CONNECTOR
        default_regex:
            pattern: alpha-(.*)\\.csv
            group_names:
                - index
        container: my_container
        name_starts_with: ""
        assets:
            alpha:
        azure_options:
            account_url: my_account_url.blob.core.windows.net
            credential: my_credential
    """,
        runtime_environment={
            "execution_engine": PandasExecutionEngine(),
        },
        return_mode="report_object",
    )
    assert mock_emit.call_count == 1

    anonymized_name = mock_emit.call_args_list[0][0][0]["event_payload"][
        "anonymized_name"
    ]
    expected_call_args_list = [
        mock.call(
            {
                "event": "data_context.test_yaml_config",
                "event_payload": {
                    "anonymized_name": anonymized_name,
                    "parent_class": "ConfiguredAssetAzureDataConnector",
                },
                "success": True,
            }
        ),
    ]
    assert mock_emit.call_args_list == expected_call_args_list

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys",
    return_value=["alpha-1.csv", "alpha-2.csv", "alpha-3.csv"],
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_instantiation_from_a_config_with_nonmatching_regex_creates_unmatched_references(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    context: DataContext = empty_data_context_stats_enabled

    report_object = context.test_yaml_config(
        """
        module_name: great_expectations.datasource.data_connector
        class_name: ConfiguredAssetAzureDataConnector
        datasource_name: FAKE_DATASOURCE
        name: TEST_DATA_CONNECTOR
        default_regex:
            pattern: beta-(.*)\\.csv
            group_names:
                - index
        container: my_container
        name_starts_with: ""
        assets:
            alpha:
        azure_options:
            account_url: my_account_url.blob.core.windows.net
            credential: my_credential
    """,
        runtime_environment={
            "execution_engine": PandasExecutionEngine(),
        },
        return_mode="report_object",
    )

    assert report_object == {
        "class_name": "ConfiguredAssetAzureDataConnector",
        "data_asset_count": 1,
        "example_data_asset_names": [
            "alpha",
        ],
        "data_assets": {
            "alpha": {"example_data_references": [], "batch_definition_count": 0},
        },
        "example_unmatched_data_references": [
            "alpha-1.csv",
            "alpha-2.csv",
            "alpha-3.csv",
        ],
        "unmatched_data_reference_count": 3,
    }

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys",
    return_value=["alpha-1.csv", "alpha-2.csv", "alpha-3.csv"],
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_get_batch_definition_list_from_batch_request_with_nonexistent_datasource_name_raises_error(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    my_data_connector = ConfiguredAssetAzureDataConnector(
        name="my_data_connector",
        datasource_name="FAKE_DATASOURCE_NAME",
        execution_engine=PandasExecutionEngine(),
        default_regex={
            "pattern": "alpha-(.*)\\.csv",
            "group_names": ["index"],
        },
        container="my_container",
        name_starts_with="",
        assets={"alpha": {}},
        azure_options={
            "account_url": "my_account_url.blob.core.windows.net",
            "credential": "my_credential",
        },
    )

    # Raises error in `DataConnector._validate_batch_request()` due to `datasource_name` in BatchRequest not matching DataConnector `datasource_name`
    with pytest.raises(ValueError):
        my_data_connector.get_batch_definition_list_from_batch_request(
            BatchRequest(
                datasource_name="something",
                data_connector_name="my_data_connector",
                data_asset_name="something",
            )
        )

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_get_definition_list_from_batch_request_with_empty_args_raises_error(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    my_data_connector_yaml = yaml.load(
        """
           class_name: ConfiguredAssetAzureDataConnector
           datasource_name: test_environment
           container: my_container
           name_starts_with: ""
           assets:
               TestFiles:
           default_regex:
               pattern: (.+)_(.+)_(.+)\\.csv
               group_names:
                   - name
                   - timestamp
                   - price
           azure_options:
               account_url: my_account_url.blob.core.windows.net
               credential: my_credential
       """,
    )

    mock_list_keys.return_value = (
        [
            "alex_20200809_1000.csv",
            "eugene_20200809_1500.csv",
            "james_20200811_1009.csv",
            "abe_20200809_1040.csv",
            "will_20200809_1002.csv",
            "james_20200713_1567.csv",
            "eugene_20201129_1900.csv",
            "will_20200810_1001.csv",
            "james_20200810_1003.csv",
            "alex_20200819_1300.csv",
        ],
    )

    my_data_connector: ConfiguredAssetAzureDataConnector = (
        instantiate_class_from_config(
            config=my_data_connector_yaml,
            runtime_environment={
                "name": "general_azure_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
        )
    )

    # Raises error in `FilePathDataConnector.get_batch_definition_list_from_batch_request()` due to missing a `batch_request` arg
    with pytest.raises(TypeError):
        # noinspection PyArgumentList
        my_data_connector.get_batch_definition_list_from_batch_request()

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_get_definition_list_from_batch_request_with_unnamed_data_asset_name_raises_error(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    my_data_connector_yaml = yaml.load(
        """
           class_name: ConfiguredAssetAzureDataConnector
           datasource_name: test_environment
           container: my_container
           name_starts_with: ""
           assets:
               TestFiles:
           default_regex:
               pattern: (.+)_(.+)_(.+)\\.csv
               group_names:
                   - name
                   - timestamp
                   - price
           azure_options:
               account_url: my_account_url.blob.core.windows.net
               credential: my_credential
       """,
    )

    my_data_connector: ConfiguredAssetAzureDataConnector = (
        instantiate_class_from_config(
            config=my_data_connector_yaml,
            runtime_environment={
                "name": "general_azure_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
        )
    )

    # Raises error in `Batch._validate_init_parameters()` due to `data_asset_name` being `NoneType` and not the required `str`
    with pytest.raises(TypeError):
        my_data_connector.get_batch_definition_list_from_batch_request(
            BatchRequest(
                datasource_name="test_environment",
                data_connector_name="general_azure_data_connector",
                data_asset_name="",
            )
        )

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_return_all_batch_definitions_unsorted_without_named_data_asset_name(
    mock_azure_conn,
    mock_list_keys,
    mock_emit,
    empty_data_context_stats_enabled,
    expected_batch_definitions_unsorted,
):
    my_data_connector_yaml = yaml.load(
        """
           class_name: ConfiguredAssetAzureDataConnector
           datasource_name: test_environment
           container: my_container
           name_starts_with: ""
           assets:
               TestFiles:
           default_regex:
               pattern: (.+)_(.+)_(.+)\\.csv
               group_names:
                   - name
                   - timestamp
                   - price
           azure_options:
               account_url: my_account_url.blob.core.windows.net
               credential: my_credential
       """,
    )

    mock_list_keys.return_value = [
        "alex_20200809_1000.csv",
        "eugene_20200809_1500.csv",
        "james_20200811_1009.csv",
        "abe_20200809_1040.csv",
        "will_20200809_1002.csv",
        "james_20200713_1567.csv",
        "eugene_20201129_1900.csv",
        "will_20200810_1001.csv",
        "james_20200810_1003.csv",
        "alex_20200819_1300.csv",
    ]

    my_data_connector: ConfiguredAssetAzureDataConnector = (
        instantiate_class_from_config(
            config=my_data_connector_yaml,
            runtime_environment={
                "name": "general_azure_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
        )
    )

    # In an actual production environment, Azure Blob Storage will automatically sort these blobs by path (alphabetic order).
    # Source: https://docs.microsoft.com/en-us/rest/api/storageservices/List-Blobs?redirectedfrom=MSDN
    #
    # The expected behavior is that our `unsorted_batch_definition_list` will maintain the same order it parses through `list_azure_keys()` (hence "unsorted").
    # When using an actual `BlobServiceClient` (and not a mock), the output of `list_azure_keys` would be pre-sorted by nature of how the system orders blobs.
    # It is important to note that although this is a minor deviation, it is deemed to be immaterial as we still end up testing our desired behavior.

    unsorted_batch_definition_list = (
        my_data_connector._get_batch_definition_list_from_batch_request(
            BatchRequestBase(
                datasource_name="test_environment",
                data_connector_name="general_azure_data_connector",
                data_asset_name="",
            )
        )
    )
    assert unsorted_batch_definition_list == expected_batch_definitions_unsorted

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_return_all_batch_definitions_unsorted_with_named_data_asset_name(
    mock_azure_conn,
    mock_list_keys,
    mock_emit,
    empty_data_context_stats_enabled,
    expected_batch_definitions_unsorted,
):
    my_data_connector_yaml = yaml.load(
        """
           class_name: ConfiguredAssetAzureDataConnector
           datasource_name: test_environment
           container: my_container
           name_starts_with: ""
           assets:
               TestFiles:
           default_regex:
               pattern: (.+)_(.+)_(.+)\\.csv
               group_names:
                   - name
                   - timestamp
                   - price
           azure_options:
               account_url: my_account_url.blob.core.windows.net
               credential: my_credential
       """,
    )

    mock_list_keys.return_value = [
        "alex_20200809_1000.csv",
        "eugene_20200809_1500.csv",
        "james_20200811_1009.csv",
        "abe_20200809_1040.csv",
        "will_20200809_1002.csv",
        "james_20200713_1567.csv",
        "eugene_20201129_1900.csv",
        "will_20200810_1001.csv",
        "james_20200810_1003.csv",
        "alex_20200819_1300.csv",
    ]

    my_data_connector: ConfiguredAssetAzureDataConnector = (
        instantiate_class_from_config(
            config=my_data_connector_yaml,
            runtime_environment={
                "name": "general_azure_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
        )
    )

    # In an actual production environment, Azure Blob Storage will automatically sort these blobs by path (alphabetic order).
    # Source: https://docs.microsoft.com/en-us/rest/api/storageservices/List-Blobs?redirectedfrom=MSDN
    #
    # The expected behavior is that our `unsorted_batch_definition_list` will maintain the same order it parses through `list_azure_keys()` (hence "unsorted").
    # When using an actual `BlobServiceClient` (and not a mock), the output of `list_azure_keys` would be pre-sorted by nature of how the system orders blobs.
    # It is important to note that although this is a minor deviation, it is deemed to be immaterial as we still end up testing our desired behavior.

    unsorted_batch_definition_list = (
        my_data_connector.get_batch_definition_list_from_batch_request(
            BatchRequest(
                datasource_name="test_environment",
                data_connector_name="general_azure_data_connector",
                data_asset_name="TestFiles",
            )
        )
    )
    assert unsorted_batch_definition_list == expected_batch_definitions_unsorted

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_return_all_batch_definitions_basic_sorted(
    mock_azure_conn,
    mock_list_keys,
    mock_emit,
    empty_data_context_stats_enabled,
    expected_batch_definitions_sorted,
):
    my_data_connector_yaml = yaml.load(
        """
       class_name: ConfiguredAssetAzureDataConnector
       datasource_name: test_environment
       container: my_container
       name_starts_with: ""
       assets:
           TestFiles:
       default_regex:
           pattern: (.+)_(.+)_(.+)\\.csv
           group_names:
               - name
               - timestamp
               - price
       sorters:
           - orderby: asc
             class_name: LexicographicSorter
             name: name
           - datetime_format: "%Y%m%d"
             orderby: desc
             class_name: DateTimeSorter
             name: timestamp
           - orderby: desc
             class_name: NumericSorter
             name: price
       azure_options:
           account_url: my_account_url.blob.core.windows.net
           credential: my_credential
     """,
    )

    mock_list_keys.return_value = [
        "alex_20200809_1000.csv",
        "eugene_20200809_1500.csv",
        "james_20200811_1009.csv",
        "abe_20200809_1040.csv",
        "will_20200809_1002.csv",
        "james_20200713_1567.csv",
        "eugene_20201129_1900.csv",
        "will_20200810_1001.csv",
        "james_20200810_1003.csv",
        "alex_20200819_1300.csv",
    ]

    my_data_connector: ConfiguredAssetAzureDataConnector = (
        instantiate_class_from_config(
            config=my_data_connector_yaml,
            runtime_environment={
                "name": "general_azure_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
        )
    )

    self_check_report = my_data_connector.self_check()

    assert self_check_report["class_name"] == "ConfiguredAssetAzureDataConnector"
    assert self_check_report["data_asset_count"] == 1
    assert self_check_report["data_assets"]["TestFiles"]["batch_definition_count"] == 10
    assert self_check_report["unmatched_data_reference_count"] == 0

    sorted_batch_definition_list = (
        my_data_connector.get_batch_definition_list_from_batch_request(
            BatchRequest(
                datasource_name="test_environment",
                data_connector_name="general_azure_data_connector",
                data_asset_name="TestFiles",
            )
        )
    )
    assert sorted_batch_definition_list == expected_batch_definitions_sorted

@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
def test_return_all_batch_definitions_returns_specified_partition(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    my_data_connector_yaml = yaml.load(
        """
       class_name: ConfiguredAssetAzureDataConnector
       datasource_name: test_environment
       container: my_container
       name_starts_with: ""
       assets:
           TestFiles:
       default_regex:
           pattern: (.+)_(.+)_(.+)\\.csv
           group_names:
               - name
               - timestamp
               - price
       sorters:
           - orderby: asc
             class_name: LexicographicSorter
             name: name
           - datetime_format: "%Y%m%d"
             orderby: desc
             class_name: DateTimeSorter
             name: timestamp
           - orderby: desc
             class_name: NumericSorter
             name: price
       azure_options:
           account_url: my_account_url.blob.core.windows.net
           credential: my_credential
     """,
    )

    mock_list_keys.return_value = [
        "alex_20200809_1000.csv",
        "eugene_20200809_1500.csv",
        "james_20200811_1009.csv",
        "abe_20200809_1040.csv",
        "will_20200809_1002.csv",
        "james_20200713_1567.csv",
        "eugene_20201129_1900.csv",
        "will_20200810_1001.csv",
        "james_20200810_1003.csv",
        "alex_20200819_1300.csv",
    ]

    my_data_connector: ConfiguredAssetAzureDataConnector = (
        instantiate_class_from_config(
            config=my_data_connector_yaml,
            runtime_environment={
                "name": "general_azure_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
        )
    )

    self_check_report = my_data_connector.self_check()

    assert self_check_report["class_name"] == "ConfiguredAssetAzureDataConnector"
    assert self_check_report["data_asset_count"] == 1
    assert self_check_report["data_assets"]["TestFiles"]["batch_definition_count"] == 10
    assert self_check_report["unmatched_data_reference_count"] == 0

    my_batch_request: BatchRequest = BatchRequest(
        datasource_name="test_environment",
        data_connector_name="general_azure_data_connector",
        data_asset_name="TestFiles",
        data_connector_query=IDDict(
            **{
                "batch_filter_parameters": {
                    "name": "james",
                    "timestamp": "20200713",
                    "price": "1567",
                }
            }
        ),
    )

    my_batch_definition_list = (
        my_data_connector.get_batch_definition_list_from_batch_request(
            batch_request=my_batch_request
        )
    )

    assert len(my_batch_definition_list) == 1
    my_batch_definition = my_batch_definition_list[0]
    expected_batch_definition = BatchDefinition(
        datasource_name="test_environment",
        data_connector_name="general_azure_data_connector",
        data_asset_name="TestFiles",
        batch_identifiers=IDDict(
            **{
                "name": "james",
                "timestamp": "20200713",
                "price": "1567",
            }
        ),
    )
    assert my_batch_definition == expected_batch_definition

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys"
)
@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
def test_return_all_batch_definitions_sorted_without_data_connector_query(
    mock_azure_conn,
    mock_list_keys,
    mock_emit,
    empty_data_context_stats_enabled,
    expected_batch_definitions_sorted,
):
    my_data_connector_yaml = yaml.load(
        """
       class_name: ConfiguredAssetAzureDataConnector
       datasource_name: test_environment
       container: my_container
       name_starts_with: ""
       assets:
           TestFiles:
       default_regex:
           pattern: (.+)_(.+)_(.+)\\.csv
           group_names:
               - name
               - timestamp
               - price
       sorters:
           - orderby: asc
             class_name: LexicographicSorter
             name: name
           - datetime_format: "%Y%m%d"
             orderby: desc
             class_name: DateTimeSorter
             name: timestamp
           - orderby: desc
             class_name: NumericSorter
             name: price
       azure_options:
           account_url: my_account_url.blob.core.windows.net
           credential: my_credential
     """,
    )

    mock_list_keys.return_value = [
        "alex_20200809_1000.csv",
        "eugene_20200809_1500.csv",
        "james_20200811_1009.csv",
        "abe_20200809_1040.csv",
        "will_20200809_1002.csv",
        "james_20200713_1567.csv",
        "eugene_20201129_1900.csv",
        "will_20200810_1001.csv",
        "james_20200810_1003.csv",
        "alex_20200819_1300.csv",
    ]

    my_data_connector: ConfiguredAssetAzureDataConnector = (
        instantiate_class_from_config(
            config=my_data_connector_yaml,
            runtime_environment={
                "name": "general_azure_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
        )
    )

    self_check_report = my_data_connector.self_check()

    assert self_check_report["class_name"] == "ConfiguredAssetAzureDataConnector"
    assert self_check_report["data_asset_count"] == 1
    assert self_check_report["data_assets"]["TestFiles"]["batch_definition_count"] == 10
    assert self_check_report["unmatched_data_reference_count"] == 0

    sorted_batch_definition_list = (
        my_data_connector.get_batch_definition_list_from_batch_request(
            BatchRequest(
                datasource_name="test_environment",
                data_connector_name="general_azure_data_connector",
                data_asset_name="TestFiles",
            )
        )
    )
    assert sorted_batch_definition_list == expected_batch_definitions_sorted

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys"
)
@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
def test_return_all_batch_definitions_raises_error_due_to_sorter_that_does_not_match_group(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    my_data_connector_yaml = yaml.load(
        """
       class_name: ConfiguredAssetAzureDataConnector
       datasource_name: test_environment
       container: my_container
       assets:
           TestFiles:
               pattern: (.+)_(.+)_(.+)\\.csv
               group_names:
                   - name
                   - timestamp
                   - price
       default_regex:
           pattern: (.+)_.+_.+\\.csv
           group_names:
               - name
       sorters:
           - orderby: asc
             class_name: LexicographicSorter
             name: name
           - datetime_format: "%Y%m%d"
             orderby: desc
             class_name: DateTimeSorter
             name: timestamp
           - orderby: desc
             class_name: NumericSorter
             name: for_me_Me_Me

       azure_options:
           account_url: my_account_url.blob.core.windows.net
           credential: my_credential
   """,
    )

    mock_list_keys.return_value = [
        "alex_20200809_1000.csv",
        "eugene_20200809_1500.csv",
        "james_20200811_1009.csv",
        "abe_20200809_1040.csv",
        "will_20200809_1002.csv",
        "james_20200713_1567.csv",
        "eugene_20201129_1900.csv",
        "will_20200810_1001.csv",
        "james_20200810_1003.csv",
        "alex_20200819_1300.csv",
    ]

    # Raises error due to a sorter (for_me_Me_me) not matching a group_name in `FilePathDataConnector._validate_sorters_configuration()`
    with pytest.raises(ge_exceptions.DataConnectorError):
        instantiate_class_from_config(
            config=my_data_connector_yaml,
            runtime_environment={
                "name": "general_azure_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
        )

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys"
)
@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
def test_return_all_batch_definitions_too_many_sorters(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    my_data_connector_yaml = yaml.load(
        """
       class_name: ConfiguredAssetAzureDataConnector
       datasource_name: test_environment
       container: my_container
       name_starts_with: ""
       assets:
           TestFiles:
       default_regex:
           pattern: (.+)_.+_.+\\.csv
           group_names:
               - name
       sorters:
           - orderby: asc
             class_name: LexicographicSorter
             name: name
           - datetime_format: "%Y%m%d"
             orderby: desc
             class_name: DateTimeSorter
             name: timestamp
           - orderby: desc
             class_name: NumericSorter
             name: price

       azure_options:
           account_url: my_account_url.blob.core.windows.net
           credential: my_credential
   """,
    )

    mock_list_keys.return_value = [
        "alex_20200809_1000.csv",
        "eugene_20200809_1500.csv",
        "james_20200811_1009.csv",
        "abe_20200809_1040.csv",
        "will_20200809_1002.csv",
        "james_20200713_1567.csv",
        "eugene_20201129_1900.csv",
        "will_20200810_1001.csv",
        "james_20200810_1003.csv",
        "alex_20200819_1300.csv",
    ]

    # Raises error due to a non-existent sorter being specified in `FilePathDataConnector._validate_sorters_configuration()`
    with pytest.raises(ge_exceptions.DataConnectorError):
        instantiate_class_from_config(
            config=my_data_connector_yaml,
            runtime_environment={
                "name": "general_azure_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
        )

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys",
)
@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
def test_example_with_explicit_data_asset_names(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    yaml_string = """
class_name: ConfiguredAssetAzureDataConnector
datasource_name: FAKE_DATASOURCE_NAME
container: my_container
name_starts_with: my_base_directory/
default_regex:
   pattern: ^(.+)-(\\d{4})(\\d{2})\\.(csv|txt)$
   group_names:
       - data_asset_name
       - year_dir
       - month_dir
assets:
   alpha:
       name_starts_with: my_base_directory/alpha/files/go/here/
       pattern: ^(.+)-(\\d{4})(\\d{2})\\.csv$
   beta:
       name_starts_with: my_base_directory/beta_here/
       pattern: ^(.+)-(\\d{4})(\\d{2})\\.txt$
   gamma:
       pattern: ^(.+)-(\\d{4})(\\d{2})\\.csv$

azure_options:
   account_url: my_account_url.blob.core.windows.net
   credential: my_credential
   """
    config = yaml.load(yaml_string)

    mock_list_keys.return_value = [  # Initial return value during instantiation
        "my_base_directory/alpha/files/go/here/alpha-202001.csv",
        "my_base_directory/alpha/files/go/here/alpha-202002.csv",
        "my_base_directory/alpha/files/go/here/alpha-202003.csv",
        "my_base_directory/beta_here/beta-202001.txt",
        "my_base_directory/beta_here/beta-202002.txt",
        "my_base_directory/beta_here/beta-202003.txt",
        "my_base_directory/beta_here/beta-202004.txt",
        "my_base_directory/gamma-202001.csv",
        "my_base_directory/gamma-202002.csv",
        "my_base_directory/gamma-202003.csv",
        "my_base_directory/gamma-202004.csv",
        "my_base_directory/gamma-202005.csv",
    ]

    my_data_connector: ConfiguredAssetAzureDataConnector = (
        instantiate_class_from_config(
            config,
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
            runtime_environment={
                "name": "my_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
        )
    )

    # Since we are using mocks, we need to redefine the output of subsequent calls to `list_azure_keys()`
    # Our patched object provides the ability to define a "side_effect", an iterable containing return
    # values for subsequent calls. Since `_refresh_data_references_cache()` makes multiple calls to
    # this method (once per asset), we define our expected behavior below.
    #
    # Source: https://stackoverflow.com/questions/24897145/python-mock-multiple-return-values

    mock_list_keys.side_effect = [
        [  # Asset alpha
            "my_base_directory/alpha/files/go/here/alpha-202001.csv",
            "my_base_directory/alpha/files/go/here/alpha-202002.csv",
            "my_base_directory/alpha/files/go/here/alpha-202003.csv",
        ],
        [  # Asset beta
            "my_base_directory/beta_here/beta-202001.txt",
            "my_base_directory/beta_here/beta-202002.txt",
            "my_base_directory/beta_here/beta-202003.txt",
            "my_base_directory/beta_here/beta-202004.txt",
        ],
        [  # Asset gamma
            "my_base_directory/gamma-202001.csv",
            "my_base_directory/gamma-202002.csv",
            "my_base_directory/gamma-202003.csv",
            "my_base_directory/gamma-202004.csv",
            "my_base_directory/gamma-202005.csv",
        ],
    ]

    my_data_connector._refresh_data_references_cache()

    assert len(my_data_connector.get_unmatched_data_references()) == 0

    assert (
        len(
            my_data_connector.get_batch_definition_list_from_batch_request(
                batch_request=BatchRequest(
                    datasource_name="FAKE_DATASOURCE_NAME",
                    data_connector_name="my_data_connector",
                    data_asset_name="alpha",
                )
            )
        )
        == 3
    )

    assert (
        len(
            my_data_connector.get_batch_definition_list_from_batch_request(
                batch_request=BatchRequest(
                    datasource_name="FAKE_DATASOURCE_NAME",
                    data_connector_name="my_data_connector",
                    data_asset_name="beta",
                )
            )
        )
        == 4
    )

    assert (
        len(
            my_data_connector.get_batch_definition_list_from_batch_request(
                batch_request=BatchRequest(
                    datasource_name="FAKE_DATASOURCE_NAME",
                    data_connector_name="my_data_connector",
                    data_asset_name="gamma",
                )
            )
        )
        == 5
    )

@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.BlobServiceClient"
)
@mock.patch(
    "great_expectations.datasource.data_connector.configured_asset_azure_data_connector.list_azure_keys",
)
@mock.patch(
    "great_expectations.core.usage_statistics.usage_statistics.UsageStatisticsHandler.emit"
)
def test_get_full_file_path(
    mock_azure_conn, mock_list_keys, mock_emit, empty_data_context_stats_enabled
):
    yaml_string = """
class_name: ConfiguredAssetAzureDataConnector
datasource_name: FAKE_DATASOURCE_NAME
container: my_container
name_starts_with: my_base_directory/
default_regex:
   pattern: ^(.+)-(\\d{{4}})(\\d{{2}})\\.(csv|txt)$
   group_names:
       - data_asset_name
       - year_dir
       - month_dir
assets:
   alpha:
       prefix: my_base_directory/alpha/files/go/here/
       pattern: ^(.+)-(\\d{{4}})(\\d{{2}})\\.csv$
   beta:
       prefix: my_base_directory/beta_here/
       pattern: ^(.+)-(\\d{{4}})(\\d{{2}})\\.txt$
   gamma:
       pattern: ^(.+)-(\\d{{4}})(\\d{{2}})\\.csv$

azure_options:
   account_url: my_account_url.blob.core.windows.net
   credential: my_credential
   """
    config = yaml.load(yaml_string)

    mock_list_keys.return_value = [
        "my_base_directory/alpha/files/go/here/alpha-202001.csv",
        "my_base_directory/alpha/files/go/here/alpha-202002.csv",
        "my_base_directory/alpha/files/go/here/alpha-202003.csv",
        "my_base_directory/beta_here/beta-202001.txt",
        "my_base_directory/beta_here/beta-202002.txt",
        "my_base_directory/beta_here/beta-202003.txt",
        "my_base_directory/beta_here/beta-202004.txt",
        "my_base_directory/gamma-202001.csv",
        "my_base_directory/gamma-202002.csv",
        "my_base_directory/gamma-202003.csv",
        "my_base_directory/gamma-202004.csv",
        "my_base_directory/gamma-202005.csv",
    ]

    my_data_connector: ConfiguredAssetAzureDataConnector = (
        instantiate_class_from_config(
            config,
            config_defaults={
                "module_name": "great_expectations.datasource.data_connector"
            },
            runtime_environment={
                "name": "my_data_connector",
                "execution_engine": PandasExecutionEngine(),
            },
        )
    )

    assert (
        my_data_connector._get_full_file_path(
            "my_base_directory/alpha/files/go/here/alpha-202001.csv", "alpha"
        )
        == "my_account_url.blob.core.windows.net/my_container/my_base_directory/alpha/files/go/here/alpha-202001.csv"
    )
    assert (
        my_data_connector._get_full_file_path(
            "my_base_directory/beta_here/beta-202002.txt", "beta"
        )
        == "my_account_url.blob.core.windows.net/my_container/my_base_directory/beta_here/beta-202002.txt"
    )
    assert (
        my_data_connector._get_full_file_path(
            "my_base_directory/gamma-202005.csv", "gamma"
        )
        == "my_account_url.blob.core.windows.net/my_container/my_base_directory/gamma-202005.csv"
    )


text:
module VagrantPlugins
  module Ansible
    class Provisioner < Vagrant.plugin("2", :provisioner)
      def provision
        ssh = @machine.ssh_info

        # Connect with Vagrant user (unless --user or --private-key are
        # overidden by 'raw_arguments').
        #
        # TODO: multiple private key support
        options = %W[--private-key=#{ssh[:private_key_path][0]} --user=#{ssh[:username]}]

        # Joker! Not (yet) supported arguments can be passed this way.
        options << "#{config.raw_arguments}" if config.raw_arguments

        # Append Provisioner options (highest precedence):
        options << "--inventory-file=#{self.setup_inventory_file}"
        options << "--extra-vars=#{self.get_extra_vars_argument}" if config.extra_vars
        options << "--sudo" if config.sudo
        options << "--sudo-user=#{config.sudo_user}" if config.sudo_user
        options << "#{self.get_verbosity_argument}" if config.verbose
        options << "--ask-sudo-pass" if config.ask_sudo_pass
        options << "--tags=#{as_list_argument(config.tags)}" if config.tags
        options << "--skip-tags=#{as_list_argument(config.skip_tags)}" if config.skip_tags
        options << "--limit=#{as_list_argument(config.limit)}" if config.limit
        options << "--start-at-task=#{config.start_at_task}" if config.start_at_task

        # Assemble the full ansible-playbook command
        command = (%w(ansible-playbook) << options << config.playbook).flatten

        # Write stdout and stderr data, since it's the regular Ansible output
        command << {
          :env => {
            "ANSIBLE_FORCE_COLOR" => "true",
            "ANSIBLE_HOST_KEY_CHECKING" => "#{config.host_key_checking}",
            # Ensure Ansible output isn't buffered so that we receive ouput
            # on a task-by-task basis.
            "PYTHONUNBUFFERED" => 1
          },
          :notify => [:stdout, :stderr],
          :workdir => @machine.env.root_path.to_s
        }

        begin
          result = Vagrant::Util::Subprocess.execute(*command) do |type, data|
            if type == :stdout || type == :stderr
              @machine.env.ui.info(data, :new_line => false, :prefix => false)
            end
          end

          raise Vagrant::Errors::AnsibleFailed if result.exit_code != 0
        rescue Vagrant::Util::Subprocess::LaunchError
          raise Vagrant::Errors::AnsiblePlaybookAppNotFound
        end
      end

      protected

      # Auto-generate "safe" inventory file based on Vagrantfile,
      # unless inventory_path is explicitly provided
      def setup_inventory_file
        return config.inventory_path if config.inventory_path

        ssh = @machine.ssh_info

        generated_inventory_file =
          @machine.env.root_path.join("vagrant_ansible_inventory_#{machine.name}")

        generated_inventory_file.open('w') do |file|
          file.write("# Generated by Vagrant\n\n")
          file.write("#{machine.name} ansible_ssh_host=#{ssh[:host]} ansible_ssh_port=#{ssh[:port]}\n")

          # Write out groups information.  Only include current
          # machine and its groups to avoid Ansible errors on
          # provisioning.
          groups_of_groups = {}
          included_groups = []

          config.groups.each_pair do |gname, gmembers|
            if gname.end_with?(":children")
              groups_of_groups[gname] = gmembers
            elsif gmembers.include?("#{machine.name}")
              included_groups << gname
              file.write("\n[#{gname}]\n")
              file.write("#{machine.name}\n")
            end
          end

          groups_of_groups.each_pair do |gname, gmembers|
            unless (included_groups & gmembers).empty?
              file.write("\n[#{gname}]\n")
              gmembers.each do |gm|
                file.write("#{gm}\n") if included_groups.include?(gm)
              end
            end
          end
        end

        return generated_inventory_file.to_s
      end

      def get_extra_vars_argument
        if config.extra_vars.kind_of?(String) and config.extra_vars =~ /^@.+$/
          # A JSON or YAML file is referenced (requires Ansible 1.3+)
          return config.extra_vars
        else
          # Expected to be a Hash after config validation. (extra_vars as
          # JSON requires Ansible 1.2+, while YAML requires Ansible 1.3+)
          return config.extra_vars.to_json
        end
      end

      def get_verbosity_argument
        if config.verbose.to_s =~ /^v+$/
          # ansible-playbook accepts "silly" arguments like '-vvvvv' as '-vvvv' for now
          return "-#{config.verbose}"
        else
          # safe default, in case input strays
          return '-v'
        end
      end

      def as_list_argument(v)
        v.kind_of?(Array) ? v.join(',') : v
      end
    end
  end
end

redpajama_set_name: RedPajamaGithub


text:
<!DOCTYPE html>

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
    <title>Class XmpMappingEntryEditable
   | Picturepark.SDK.V1 API </title>
    <meta name="viewport" content="width=device-width">
    <meta name="title" content="Class XmpMappingEntryEditable
   | Picturepark.SDK.V1 API ">
    <meta name="generator" content="docfx 2.59.4.0">

    <link rel="shortcut icon" href="../favicon.ico">
    <link rel="stylesheet" href="../styles/docfx.vendor.css">
    <link rel="stylesheet" href="../styles/docfx.css">
    <link rel="stylesheet" href="../styles/main.css">
    <meta property="docfx:navrel" content="../toc.html">
    <meta property="docfx:tocrel" content="toc.html">

    <meta property="docfx:rel" content="../">

              <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar">
                Toggle navigation

              </button>

              <form class="navbar-form navbar-right" role="search" id="search">

                  <input type="text" class="form-control" id="search-query" placeholder="Search" autocomplete="off">

              </form>

Search Results for

          Show / Hide Table of Contents

Class XmpMappingEntryEditable

Editable properties of a XMP mapping entry

Inheritance

System.Object

XmpMappingEntryEditable

XmpMappingEntry

XmpMappingEntryCreateRequest

Inherited Members

      System.Object.ToString()

      System.Object.Equals(System.Object)

      System.Object.Equals(System.Object, System.Object)

      System.Object.ReferenceEquals(System.Object, System.Object)

      System.Object.GetHashCode()

      System.Object.GetType()

      System.Object.MemberwiseClone()

Namespace: System.Dynamic.ExpandoObject

Syntax

public class XmpMappingEntryEditable

Properties

Configuration

Optional additional configuration for the mapping.

Declaration

public XmpMappingEntryConfigurationBase Configuration { get; set; }

Property Value

        XmpMappingEntryConfigurationBase

Direction

Direction of the mapping.

Declaration

public MappingDirection Direction { get; set; }

Property Value

        MappingDirection

MetadataPath

Path to target property in metadata.

Declaration

public string MetadataPath { get; set; }

Property Value

        System.String

Priority

Priority of the mapping.

Declaration

public int Priority { get; set; }

Property Value

        System.Int32

StopProcessing

Stop processing further mappings for the same field if a value was found.

Declaration

public bool StopProcessing { get; set; }

Property Value

        System.Boolean

XmpPath

Path to source property in XMP.

Declaration

public string XmpPath { get; set; }

Property Value

        System.String

Methods

FromJson(String)

Declaration

public static XmpMappingEntryEditable FromJson(string data)

Parameters

        System.String
        data

Returns

        XmpMappingEntryEditable

ToJson()

Declaration

public string ToJson()

Returns

        System.String

              Back to top

            Generated by DocFX


text:
package com.gitblit;

import java.util.Collection;
import java.util.List;

import com.gitblit.manager.IRuntimeManager;
import com.gitblit.models.TeamModel;
import com.gitblit.models.UserModel;

/**
 * Implementations of IUserService control all aspects of UserModel objects and
 * user authentication.
 *
 * @author James Moger
 *
 */
public interface IUserService {

	/**
	 * Setup the user service. This method allows custom implementations to
	 * retrieve settings from gitblit.properties or the web.xml file without
	 * relying on the GitBlit static singleton.
	 *
	 * @param runtimeManager
	 * @since 1.4.0
	 */
	void setup(IRuntimeManager runtimeManager);

	/**
	 * Returns the cookie value for the specified user.
	 *
	 * @param model
	 * @return cookie value
 	 * @since 0.5.0
	 */
	String getCookie(UserModel model);

	/**
	 * Retrieve a user object for the specified cookie.
	 *
	 * @param cookie
	 * @return a user object or null
 	 * @since 0.5.0
	 */
	UserModel getUserModel(char[] cookie);

	/**
	 * Retrieve the user object for the specified username.
	 *
	 * @param username
	 * @return a user object or null
 	 * @since 0.5.0
	 */
	UserModel getUserModel(String username);

	/**
	 * Updates/writes a complete user object.
	 *
	 * @param model
	 * @return true if update is successful
 	 * @since 0.5.0
	 */
	boolean updateUserModel(UserModel model);

	/**
	 * Updates/writes all specified user objects.
	 *
	 * @param models a list of user models
	 * @return true if update is successful
	 * @since 1.2.0
	 */
	boolean updateUserModels(Collection<UserModel> models);

	/**
	 * Adds/updates a user object keyed by username. This method allows for
	 * renaming a user.
	 *
	 * @param username
	 *            the old username
	 * @param model
	 *            the user object to use for username
	 * @return true if update is successful
 	 * @since 0.5.0
	 */
	boolean updateUserModel(String username, UserModel model);

	/**
	 * Deletes the user object from the user service.
	 *
	 * @param model
	 * @return true if successful
 	 * @since 0.5.0
	 */
	boolean deleteUserModel(UserModel model);

	/**
	 * Delete the user object with the specified username
	 *
	 * @param username
	 * @return true if successful
 	 * @since 0.5.0
	 */
	boolean deleteUser(String username);

	/**
	 * Returns the list of all users available to the login service.
	 *
	 * @return list of all usernames
 	 * @since 0.5.0
	 */
	List<String> getAllUsernames();

	/**
	 * Returns the list of all users available to the login service.
	 *
	 * @return list of all users
	 * @since 0.8.0
	 */
	List<UserModel> getAllUsers();

	/**
	 * Returns the list of all teams available to the login service.
	 *
	 * @return list of all teams
	 * @since 0.8.0
	 */
	List<String> getAllTeamNames();

	/**
	 * Returns the list of all teams available to the login service.
	 *
	 * @return list of all teams
	 * @since 0.8.0
	 */
	List<TeamModel> getAllTeams();

	/**
	 * Returns the list of all users who are allowed to bypass the access
	 * restriction placed on the specified repository.
	 *
	 * @param role
	 *            the repository name
	 * @return list of all usernames that can bypass the access restriction
	 * @since 0.8.0
	 */
	List<String> getTeamNamesForRepositoryRole(String role);

	/**
	 * Retrieve the team object for the specified team name.
	 *
	 * @param teamname
	 * @return a team object or null
	 * @since 0.8.0
	 */
	TeamModel getTeamModel(String teamname);

	/**
	 * Updates/writes a complete team object.
	 *
	 * @param model
	 * @return true if update is successful
	 * @since 0.8.0
	 */
	boolean updateTeamModel(TeamModel model);

	/**
	 * Updates/writes all specified team objects.
	 *
	 * @param models a list of team models
	 * @return true if update is successful
	 * @since 1.2.0
	 */
	boolean updateTeamModels(Collection<TeamModel> models);

	/**
	 * Updates/writes and replaces a complete team object keyed by teamname.
	 * This method allows for renaming a team.
	 *
	 * @param teamname
	 *            the old teamname
	 * @param model
	 *            the team object to use for teamname
	 * @return true if update is successful
	 * @since 0.8.0
	 */
	boolean updateTeamModel(String teamname, TeamModel model);

	/**
	 * Deletes the team object from the user service.
	 *
	 * @param model
	 * @return true if successful
	 * @since 0.8.0
	 */
	boolean deleteTeamModel(TeamModel model);

	/**
	 * Delete the team object with the specified teamname
	 *
	 * @param teamname
	 * @return true if successful
	 * @since 0.8.0
	 */
	boolean deleteTeam(String teamname);

	/**
	 * Returns the list of all users who are allowed to bypass the access
	 * restriction placed on the specified repository.
	 *
	 * @param role
	 *            the repository name
	 * @return list of all usernames that can bypass the access restriction
	 * @since 0.8.0
	 */
	List<String> getUsernamesForRepositoryRole(String role);

	/**
	 * Renames a repository role.
	 *
	 * @param oldRole
	 * @param newRole
	 * @return true if successful
 	 * @since 0.5.0
	 */
	boolean renameRepositoryRole(String oldRole, String newRole);

	/**
	 * Removes a repository role from all users.
	 *
	 * @param role
	 * @return true if successful
 	 * @since 0.5.0
	 */
	boolean deleteRepositoryRole(String role);

	/**
	 * @See java.lang.Object.toString();
	 * @return string representation of the login service
	 */
	@Override
	String toString();
}


text:
package jcommon.process;

import jcommon.process.api.ByteArrayPool;

import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicInteger;

/**
 */
public abstract class ProcessListener<TAttachment extends Object> implements IProcessListener<TAttachment> {
  private static ByteArrayPool buffer_pool = null;
  private static final Object buffer_pool_lock = new Object();
  private static final AtomicInteger running = new AtomicInteger(0);

  private void init() {
    synchronized (buffer_pool_lock) {
      if (running.getAndIncrement() == 0) {
        buffer_pool = new ByteArrayPool(2, 1024);
      }
    }
  }

  private void check() {
    synchronized (buffer_pool_lock) {
      if (buffer_pool == null) {
        init();
      }
    }
  }

  private void release() {
    synchronized (buffer_pool_lock) {
      if (running.decrementAndGet() == 0) {
        //buffer_pool.dispose();
        //buffer_pool = null;
      }
    }
  }

  @Override
  public final void started(IProcess process) throws Throwable {
    init();

    processStarted(process);
  }

  @Override
  public final void stopped(IProcess process, int exitCode) throws Throwable {
    //Thread.sleep(3000);
    release();

    synchronized (process) {
    processStopped(process, exitCode);
    }
  }

  @Override
  public final void stdin(IProcess process, ByteBuffer buffer, int bytesWritten, TAttachment attachment) throws Throwable {
    final ByteArrayPool.Buffer pool_buffer = buffer_pool.requestInstance();
    try {
      stdin(process, buffer, bytesWritten, pool_buffer.getBuffer(), pool_buffer.getBufferSize(), attachment);
    } finally {
      buffer_pool.returnToPool(pool_buffer);
    }
  }

  @Override
  public final void stdout(IProcess process, ByteBuffer buffer, int bytesRead) throws Throwable {
//    check();
//    final ByteArrayPool.Buffer pool_buffer = buffer_pool.requestInstance();
//    try {
//      stdout(process, buffer, bytesRead, pool_buffer.getBuffer(), pool_buffer.getBufferSize());
//    } finally {
//      buffer_pool.returnToPool(pool_buffer);
//    }
    synchronized (process) {
    stdout(process, buffer, bytesRead, new byte[bytesRead], bytesRead);
    }
  }

  @Override
  public final void stderr(IProcess process, ByteBuffer buffer, int bytesRead) throws Throwable {
    final ByteArrayPool.Buffer pool_buffer = buffer_pool.requestInstance();
    try {
      stderr(process, buffer, bytesRead, pool_buffer.getBuffer(), pool_buffer.getBufferSize());
    } finally {
      buffer_pool.returnToPool(pool_buffer);
    }
  }

  @Override
  public final void error(IProcess process, Throwable t) {
    processError(process, t);
  }

  protected void processStarted(IProcess process) throws Throwable { }
  protected void processStopped(IProcess process, int exitCode) throws Throwable { }
  protected void stdin(IProcess process, ByteBuffer buffer, int bytesWritten, byte[] availablePoolBuffer, int poolBufferSize, TAttachment attachment) throws Throwable { }
  protected void stdout(IProcess process, ByteBuffer buffer, int bytesRead, byte[] availablePoolBuffer, int poolBufferSize) throws Throwable { }
  protected void stderr(IProcess process, ByteBuffer buffer, int bytesRead, byte[] availablePoolBuffer, int poolBufferSize) throws Throwable { }
  protected void processError(IProcess process, Throwable t) { }
}


text:
from starcluster.clustersetup import ClusterSetup
from starcluster.logger import log

class IntogenInstaller(ClusterSetup):
	def run(self, nodes, master, user, user_shell, volumes):
		for node in nodes:
			log.info("Installing Intogen 2.4.1 on %s" % (node.alias))
			node.ssh.execute('mkdir -p /opt/software/intogen')
			node.ssh.execute('wget -c -P /opt/software/intogen/ https://bitbucket.org/intogen/mutations-analysis/get/master.tar.gz')
			node.ssh.execute('tar xzvf /opt/software/intogen/master.tar.gz -C /opt/software/intogen/')
			node.ssh.execute('apt-get -y install swig')
			node.ssh.execute('sed -i '11s|$ROOT_PATH/data|/data/database/intogen|g' /opt/software/intogen/intogen-mutations-analysis-7fbd0b4803be/setup')
			node.ssh.execute('cd /opt/software/intogen/intogen-mutations-analysis-7fbd0b4803be/ && ./setup')
			node.ssh.execute('rm -r /data') # this will be linkin in on_boot.py to database drive

			log.info("Creating Intogen Module")
			node.ssh.execute('mkdir -p /usr/local/Modules/applications/intogen/;touch /usr/local/Modules/applications/intogen/2.4.1')
			node.ssh.execute('echo "#%Module" >> /usr/local/Modules/applications/intogen/2.4.1')
			node.ssh.execute('echo "set root /opt/software/intogen/intogen-mutations-analysis-7fbd0b4803be/" >> /usr/local/Modules/applications/intogen/2.4.1')
			node.ssh.execute('echo -e "prepend-path\tPATH\t\$root" >> /usr/local/Modules/applications/intogen/2.4.1')