# Copyright (c) 2013 Hortonworks, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import mock
import pkg_resources as pkg

from sahara.plugins import exceptions as ex
from sahara.plugins.hdp import clusterspec as cs
from sahara.plugins.hdp import hadoopserver
from sahara.plugins.hdp.versions.version_1_3_2 import services as s
from sahara.plugins import provisioning
from sahara.tests.unit import base as sahara_base
import sahara.tests.unit.plugins.hdp.hdp_test_base as base
from sahara.topology import topology_helper as th
from sahara import version


class TestCONF(object):
    def __init__(self, enable_data_locality, enable_hypervisor_awareness):
        self.enable_data_locality = enable_data_locality
        self.enable_hypervisor_awareness = enable_hypervisor_awareness


@mock.patch("sahara.utils.openstack.nova.get_instance_info",
            base.get_instance_info)
@mock.patch('sahara.plugins.hdp.versions.version_1_3_2.services.HdfsService.'
            '_get_swift_properties',
            return_value=[])
class ClusterSpecTest(sahara_base.SaharaTestCase):
    service_validators = {}

    def setUp(self):
        super(ClusterSpecTest, self).setUp()
        self.service_validators['HDFS'] = self._assert_hdfs
        self.service_validators['MAPREDUCE'] = self._assert_mr
        self.service_validators['GANGLIA'] = self._assert_ganglia
        self.service_validators['NAGIOS'] = self._assert_nagios
        self.service_validators['AMBARI'] = self._assert_ambari
        self.service_validators['PIG'] = self._assert_pig
        self.service_validators['HIVE'] = self._assert_hive
        self.service_validators['HCATALOG'] = self._assert_hcatalog
        self.service_validators['ZOOKEEPER'] = self._assert_zookeeper
        self.service_validators['WEBHCAT'] = self._assert_webhcat
        self.service_validators['OOZIE'] = self._assert_oozie
        self.service_validators['SQOOP'] = self._assert_sqoop
        self.service_validators['HBASE'] = self._assert_hbase

    # TODO(jspeidel): test host manifest
    def test_parse_default_with_cluster(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        server1 = base.TestServer('host1', 'test-master', '11111', 3,
                                  '111.11.1111', '222.11.1111')
        server2 = base.TestServer('host2', 'test-slave', '11111', 3,
                                  '222.22.2222', '333.22.2222')

        node_group1 = TestNodeGroup(
            'master', [server1], ["NAMENODE", "JOBTRACKER",
                                  "SECONDARY_NAMENODE", "GANGLIA_SERVER",
                                  "GANGLIA_MONITOR", "NAGIOS_SERVER",
                                  "AMBARI_SERVER", "AMBARI_AGENT"])
        node_group2 = TestNodeGroup('slave', [server2], ['TASKTRACKER',
                                                         'DATANODE'])
        cluster = base.TestCluster([node_group1, node_group2])

        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster_config.create_operational_config(cluster, [])

        self._assert_services(cluster_config.services)
        self._assert_configurations(cluster_config.configurations)

        node_groups = cluster_config.node_groups
        self.assertEqual(2, len(node_groups))
        self.assertIn('master', node_groups)
        self.assertIn('slave', node_groups)

        master_node_group = node_groups['master']
        self.assertEqual('master', master_node_group.name)
        self.assertEqual(9, len(master_node_group.components))
        self.assertIn('NAMENODE', master_node_group.components)
        self.assertIn('JOBTRACKER', master_node_group.components)
        self.assertIn('SECONDARY_NAMENODE', master_node_group.components)
        self.assertIn('GANGLIA_SERVER', master_node_group.components)
        self.assertIn('GANGLIA_MONITOR', master_node_group.components)
        self.assertIn('NAGIOS_SERVER', master_node_group.components)
        self.assertIn('AMBARI_SERVER', master_node_group.components)
        self.assertIn('AMBARI_AGENT', master_node_group.components)
        self.assertIn('HISTORYSERVER', master_node_group.components)

        slave_node_group = node_groups['slave']
        self.assertEqual('slave', slave_node_group.name)
        self.assertIn('TASKTRACKER', slave_node_group.components)

        return cluster_config

    def test_determine_component_hosts(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        server1 = base.TestServer('ambari_machine', 'master', '11111', 3,
                                  '111.11.1111', '222.11.1111')
        server2 = base.TestServer('host2', 'slave', '11111', 3, '222.22.2222',
                                  '333.22.2222')
        server3 = base.TestServer('host3', 'slave', '11111', 3, '222.22.2223',
                                  '333.22.2223')

        node_group1 = TestNodeGroup(
            'master', [server1], ["NAMENODE", "JOBTRACKER",
                                  "SECONDARY_NAMENODE", "GANGLIA_SERVER",
                                  "NAGIOS_SERVER", "AMBARI_SERVER"])

        node_group2 = TestNodeGroup(
            'slave', [server2], ["DATANODE", "TASKTRACKER",
                                 "HDFS_CLIENT", "MAPREDUCE_CLIENT"])

        node_group3 = TestNodeGroup(
            'slave2', [server3], ["DATANODE", "TASKTRACKER",
                                  "HDFS_CLIENT", "MAPREDUCE_CLIENT"])

        cluster = base.TestCluster([node_group1, node_group2, node_group3])

        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster_config.create_operational_config(cluster, [])

        hosts = cluster_config.determine_component_hosts('AMBARI_SERVER')
        self.assertEqual(1, len(hosts))
        self.assertEqual('ambari_machine', hosts.pop().fqdn())

        hosts = cluster_config.determine_component_hosts('DATANODE')
        self.assertEqual(2, len(hosts))
        datanodes = set([server2.fqdn(), server3.fqdn()])
        host_fqdn = set([hosts.pop().fqdn(), hosts.pop().fqdn()])
        # test intersection is both servers
        self.assertEqual(datanodes, host_fqdn & datanodes)

    def test_finalize_configuration(self, patched):
        patched.return_value = [{'name': 'swift.prop1',
                                'value': 'swift_prop_value'},
                                {'name': 'swift.prop2',
                                'value': 'swift_prop_value2'}]
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        master_host = base.TestServer(
            'master.novalocal', 'master', '11111', 3,
            '111.11.1111', '222.11.1111')

        jt_host = base.TestServer(
            'jt_host.novalocal', 'jt', '11111', 3,
            '111.11.2222', '222.11.2222')

        nn_host = base.TestServer(
            'nn_host.novalocal', 'nn', '11111', 3,
            '111.11.3333', '222.11.3333')

        snn_host = base.TestServer(
            'snn_host.novalocal', 'jt', '11111', 3,
            '111.11.4444', '222.11.4444')

        hive_host = base.TestServer(
            'hive_host.novalocal', 'hive', '11111', 3,
            '111.11.5555', '222.11.5555')

        hive_ms_host = base.TestServer(
            'hive_ms_host.novalocal', 'hive_ms', '11111', 3,
            '111.11.6666', '222.11.6666')

        hive_mysql_host = base.TestServer(
            'hive_mysql_host.novalocal', 'hive_mysql', '11111', 3,
            '111.11.7777', '222.11.7777')

        hcat_host = base.TestServer(
            'hcat_host.novalocal', 'hcat', '11111', 3,
            '111.11.8888', '222.11.8888')

        zk1_host = base.TestServer(
            'zk1_host.novalocal', 'zk1', '11111', 3,
            '111.11.9999', '222.11.9999')

        zk2_host = base.TestServer(
            'zk2_host.novalocal', 'zk2', '11112', 3,
            '111.11.9990', '222.11.9990')

        oozie_host = base.TestServer(
            'oozie_host.novalocal', 'oozie', '11111', 3,
            '111.11.9999', '222.11.9999')

        slave_host = base.TestServer(
            'slave1.novalocal', 'slave', '11111', 3,
            '222.22.6666', '333.22.6666')

        master_ng = TestNodeGroup(
            'master', [master_host], ["GANGLIA_SERVER",
                                      "GANGLIA_MONITOR",
                                      "NAGIOIS_SERVER",
                                      "AMBARI_SERVER",
                                      "AMBARI_AGENT"])

        jt_ng = TestNodeGroup(
            'jt', [jt_host], ["JOBTRACKER", "GANGLIA_MONITOR",
                              "AMBARI_AGENT"])

        nn_ng = TestNodeGroup(
            'nn', [nn_host], ["NAMENODE", "GANGLIA_MONITOR",
                              "AMBARI_AGENT"])

        snn_ng = TestNodeGroup(
            'snn', [snn_host], ["SECONDARY_NAMENODE", "GANGLIA_MONITOR",
                                "AMBARI_AGENT"])

        hive_ng = TestNodeGroup(
            'hive', [hive_host], ["HIVE_SERVER", "GANGLIA_MONITOR",
                                  "AMBARI_AGENT"])

        hive_ms_ng = TestNodeGroup(
            'meta', [hive_ms_host], ["HIVE_METASTORE", "GANGLIA_MONITOR",
                                     "AMBARI_AGENT"])

        hive_mysql_ng = TestNodeGroup(
            'mysql', [hive_mysql_host], ["MYSQL_SERVER", "GANGLIA_MONITOR",
                                         "AMBARI_AGENT"])

        hcat_ng = TestNodeGroup(
            'hcat', [hcat_host], ["WEBHCAT_SERVER", "GANGLIA_MONITOR",
                                  "AMBARI_AGENT"])

        zk1_ng = TestNodeGroup(
            'zk1', [zk1_host], ["ZOOKEEPER_SERVER", "GANGLIA_MONITOR",
                                "AMBARI_AGENT"])

        zk2_ng = TestNodeGroup(
            'zk2', [zk2_host], ["ZOOKEEPER_SERVER", "GANGLIA_MONITOR",
                                "AMBARI_AGENT"])

        oozie_ng = TestNodeGroup(
            'oozie', [oozie_host], ["OOZIE_SERVER", "GANGLIA_MONITOR",
                                    "AMBARI_AGENT"])
        slave_ng = TestNodeGroup(
            'slave', [slave_host], ["DATANODE", "TASKTRACKER",
                                    "GANGLIA_MONITOR", "HDFS_CLIENT",
                                    "MAPREDUCE_CLIENT", "OOZIE_CLIENT",
                                    "AMBARI_AGENT"])

        user_input_config = TestUserInputConfig(
            'core-site', 'cluster', 'fs.default.name')
        user_input = provisioning.UserInput(
            user_input_config, 'hdfs://nn_dif_host.novalocal:8020')

        cluster = base.TestCluster([master_ng, jt_ng, nn_ng, snn_ng, hive_ng,
                                    hive_ms_ng, hive_mysql_ng,
                                    hcat_ng, zk1_ng, zk2_ng, oozie_ng,
                                    slave_ng])
        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster_config.create_operational_config(cluster, [user_input])
        config = cluster_config.configurations

        # for this value, validating that user inputs override configured
        # values, whether they are processed by runtime or not
        self.assertEqual(config['core-site']['fs.default.name'],
                         'hdfs://nn_dif_host.novalocal:8020')

        self.assertEqual(config['mapred-site']['mapred.job.tracker'],
                         'jt_host.novalocal:50300')
        self.assertEqual(config['mapred-site']
                         ['mapred.job.tracker.http.address'],
                         'jt_host.novalocal:50030')
        self.assertEqual(config['mapred-site']
                         ['mapreduce.history.server.http.address'],
                         'jt_host.novalocal:51111')
        self.assertEqual(config['hdfs-site']['dfs.http.address'],
                         'nn_host.novalocal:50070')
        self.assertEqual(config['hdfs-site']['dfs.secondary.http.address'],
                         'snn_host.novalocal:50090')
        self.assertEqual(config['hdfs-site']['dfs.https.address'],
                         'nn_host.novalocal:50470')
        self.assertEqual(config['global']['hive_hostname'],
                         'hive_host.novalocal')
        self.assertEqual(config['core-site']['hadoop.proxyuser.hive.hosts'],
                         'hive_host.novalocal')
        self.assertEqual(config['hive-site']
                         ['javax.jdo.option.ConnectionURL'],
                         'jdbc:mysql://hive_host.novalocal/hive?'
                         'createDatabaseIfNotExist=true')
        self.assertEqual(config['hive-site']['hive.metastore.uris'],
                         'thrift://hive_ms_host.novalocal:9083')
        self.assertTrue(
            'hive.metastore.uris=thrift://hive_ms_host.novalocal:9083' in
            config['webhcat-site']['templeton.hive.properties'])
        self.assertEqual(config['global']['hive_jdbc_connection_url'],
                         'jdbc:mysql://hive_mysql_host.novalocal/hive?'
                         'createDatabaseIfNotExist=true')
        self.assertEqual(config['core-site']['hadoop.proxyuser.hcat.hosts'],
                         'hcat_host.novalocal')
        self.assertEqual(set(
            config['webhcat-site']['templeton.zookeeper.hosts'].split(',')),
            set(['zk1_host.novalocal:2181', 'zk2_host.novalocal:2181']))

        self.assertEqual(config['oozie-site']['oozie.base.url'],
                         'http://oozie_host.novalocal:11000/oozie')
        self.assertEqual(config['global']['oozie_hostname'],
                         'oozie_host.novalocal')
        self.assertEqual(config['core-site']['hadoop.proxyuser.oozie.hosts'],
                         'oozie_host.novalocal,222.11.9999,111.11.9999')

        # test swift properties
        self.assertEqual('swift_prop_value',
                         config['core-site']['swift.prop1'])
        self.assertEqual('swift_prop_value2',
                         config['core-site']['swift.prop2'])

    def test__determine_deployed_services(self, nova_mock):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        master_host = base.TestServer(
            'master.novalocal', 'master', '11111', 3,
            '111.11.1111', '222.11.1111')

        jt_host = base.TestServer(
            'jt_host.novalocal', 'jt', '11111', 3,
            '111.11.2222', '222.11.2222')

        nn_host = base.TestServer(
            'nn_host.novalocal', 'nn', '11111', 3,
            '111.11.3333', '222.11.3333')

        snn_host = base.TestServer(
            'snn_host.novalocal', 'jt', '11111', 3,
            '111.11.4444', '222.11.4444')

        slave_host = base.TestServer(
            'slave1.novalocal', 'slave', '11111', 3,
            '222.22.6666', '333.22.6666')

        master_ng = TestNodeGroup(
            'master', [master_host],
            ['GANGLIA_SERVER',
             'GANGLIA_MONITOR', 'NAGIOS_SERVER',
             'AMBARI_SERVER', 'AMBARI_AGENT'])
        jt_ng = TestNodeGroup('jt', [jt_host], ["JOBTRACKER",
                              "GANGLIA_MONITOR", "AMBARI_AGENT"])
        nn_ng = TestNodeGroup('nn', [nn_host], ["NAMENODE",
                              "GANGLIA_MONITOR", "AMBARI_AGENT"])
        snn_ng = TestNodeGroup('snn', [snn_host], ["SECONDARY_NAMENODE",
                               "GANGLIA_MONITOR", "AMBARI_AGENT"])
        slave_ng = TestNodeGroup(
            'slave', [slave_host],
            ["DATANODE", "TASKTRACKER",
             "GANGLIA_MONITOR", "HDFS_CLIENT", "MAPREDUCE_CLIENT",
             "AMBARI_AGENT"])

        cluster = base.TestCluster([master_ng, jt_ng, nn_ng,
                                   snn_ng, slave_ng])
        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster_config.create_operational_config(cluster, [])
        services = cluster_config.services
        for service in services:
            if service.name in ['HDFS', 'MAPREDUCE', 'GANGLIA',
                                'AMBARI', 'NAGIOS']:
                self.assertTrue(service.deployed)
            else:
                self.assertFalse(service.deployed)

    def test_ambari_rpm_path(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')
        cluster_spec = cs.ClusterSpec(cluster_config_file)

        ambari_config = cluster_spec.configurations['ambari']
        rpm = ambari_config.get('rpm', None)
        self.assertEqual('http://s3.amazonaws.com/'
                         'public-repo-1.hortonworks.com/ambari/centos6/'
                         '1.x/updates/1.6.0/ambari.repo', rpm)

    def test_default_ambari_rpm_path(self, patched):
        self.assertEqual('http://s3.amazonaws.com/'
                         'public-repo-1.hortonworks.com/ambari/centos6/'
                         '1.x/updates/1.6.0/ambari.repo',
                         hadoopserver.AMBARI_RPM)

    def test_parse_default(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        cluster_config = cs.ClusterSpec(cluster_config_file)

        self._assert_services(cluster_config.services)
        self._assert_configurations(cluster_config.configurations)

        node_groups = cluster_config.node_groups
        self.assertEqual(2, len(node_groups))
        master_node_group = node_groups['master']
        self.assertEqual('master', master_node_group.name)
        self.assertIsNone(master_node_group.predicate)
        self.assertEqual('1', master_node_group.cardinality)
        self.assertEqual(6, len(master_node_group.components))
        self.assertIn('NAMENODE', master_node_group.components)
        self.assertIn('JOBTRACKER', master_node_group.components)
        self.assertIn('SECONDARY_NAMENODE', master_node_group.components)
        self.assertIn('GANGLIA_SERVER', master_node_group.components)
        self.assertIn('NAGIOS_SERVER', master_node_group.components)
        self.assertIn('AMBARI_SERVER', master_node_group.components)

        slave_node_group = node_groups['slave']
        self.assertEqual('slave', slave_node_group.name)
        self.assertIsNone(slave_node_group.predicate)
        self.assertEqual('1+', slave_node_group.cardinality)
        self.assertEqual(4, len(slave_node_group.components))
        self.assertIn('DATANODE', slave_node_group.components)
        self.assertIn('TASKTRACKER', slave_node_group.components)
        self.assertIn('HDFS_CLIENT', slave_node_group.components)
        self.assertIn('MAPREDUCE_CLIENT', slave_node_group.components)

        return cluster_config

    def test_ambari_rpm(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        cluster_config = cs.ClusterSpec(cluster_config_file)

        self._assert_configurations(cluster_config.configurations)
        ambari_config = cluster_config.configurations['ambari']
        self.assertIsNotNone('no rpm uri found',
                             ambari_config.get('rpm', None))

    def test_normalize(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster = cluster_config.normalize()

        configs = cluster.cluster_configs
        contains_dfs_datanode_http_address = False
        contains_mapred_jobtracker_taskScheduler = False
        contains_dfs_include = False

        for entry in configs:
            config = entry.config
            # assert some random configurations across targets
            if config.name == 'dfs.datanode.http.address':
                contains_dfs_datanode_http_address = True
                self.assertEqual('string', config.type)
                self.assertEqual('0.0.0.0:50075', config.default_value)
                self.assertEqual('HDFS', config.applicable_target)

            if config.name == 'mapred.jobtracker.taskScheduler':
                contains_mapred_jobtracker_taskScheduler = True
                self.assertEqual('string', config.type)
                self.assertEqual(
                    'org.apache.hadoop.mapred.CapacityTaskScheduler',
                    config.default_value)
                self.assertEqual('MAPREDUCE',
                                 config.applicable_target)

            if config.name == 'dfs_include':
                contains_dfs_include = True
                self.assertEqual('string', config.type)
                self.assertEqual('dfs.include', config.default_value)
                self.assertEqual('HDFS', config.applicable_target)

                #            print 'Config: name: {0}, type:{1},
                # default value:{2}, target:{3}, Value:{4}'.format(
                #                config.name, config.type,
                # config.default_value,
                #  config.applicable_target, entry.value)

        self.assertTrue(contains_dfs_datanode_http_address)
        self.assertTrue(contains_mapred_jobtracker_taskScheduler)
        self.assertTrue(contains_dfs_include)
        node_groups = cluster.node_groups
        self.assertEqual(2, len(node_groups))
        contains_master_group = False
        contains_slave_group = False
        for i in range(2):
            node_group = node_groups[i]
            components = node_group.node_processes
            if node_group.name == "master":
                contains_master_group = True
                self.assertEqual(6, len(components))
                self.assertIn('NAMENODE', components)
                self.assertIn('JOBTRACKER', components)
                self.assertIn('SECONDARY_NAMENODE', components)
                self.assertIn('GANGLIA_SERVER', components)
                self.assertIn('NAGIOS_SERVER', components)
                self.assertIn('AMBARI_SERVER', components)
                # TODO(jspeidel): node configs
                # TODO(jspeidel): vm_requirements
            elif node_group.name == 'slave':
                contains_slave_group = True
                self.assertEqual(4, len(components))
                self.assertIn('DATANODE', components)
                self.assertIn('TASKTRACKER', components)
                self.assertIn('HDFS_CLIENT', components)
                self.assertIn('MAPREDUCE_CLIENT', components)
                # TODO(jspeidel): node configs
                # TODO(jspeidel): vm requirements
            else:
                self.fail('Unexpected node group: {0}'.format(node_group.name))
        self.assertTrue(contains_master_group)
        self.assertTrue(contains_slave_group)

    def test_existing_config_item_in_top_level_within_blueprint(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        user_input_config = TestUserInputConfig(
            'global', 'general', 'fs_checkpoint_dir')
        user_input = provisioning.UserInput(user_input_config,
                                            '/some/new/path')

        server1 = base.TestServer('host1', 'test-master', '11111', 3,
                                  '111.11.1111', '222.11.1111')
        server2 = base.TestServer('host2', 'test-slave', '11111', 3,
                                  '222.22.2222', '333.22.2222')

        node_group1 = TestNodeGroup(
            'master', [server1], ["NAMENODE", "JOBTRACKER",
                                  "SECONDARY_NAMENODE", "GANGLIA_SERVER",
                                  "GANGLIA_MONITOR", "NAGIOS_SERVER",
                                  "AMBARI_SERVER", "AMBARI_AGENT"])
        node_group2 = TestNodeGroup(
            'slave', [server2], ["TASKTRACKER", "DATANODE",
                                 "AMBARI_AGENT", "GANGLIA_MONITOR"])

        cluster = base.TestCluster([node_group1, node_group2])
        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster_config.create_operational_config(cluster, [user_input])
        self.assertEqual('/some/new/path', cluster_config.configurations
                         ['global']['fs_checkpoint_dir'])

    def test_new_config_item_in_top_level_within_blueprint(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        user_input_config = TestUserInputConfig(
            'global', 'general', 'new_property')
        user_input = provisioning.UserInput(user_input_config, 'foo')

        server1 = base.TestServer('host1', 'test-master', '11111', 3,
                                  '111.11.1111', '222.11.1111')
        server2 = base.TestServer('host2', 'test-slave', '11111', 3,
                                  '222.22.2222', '333.22.2222')

        node_group1 = TestNodeGroup(
            'master', [server1],
            ["NAMENODE", "JOBTRACKER",
             "SECONDARY_NAMENODE", "GANGLIA_SERVER", "GANGLIA_MONITOR",
             "NAGIOS_SERVER", "AMBARI_SERVER", "AMBARI_AGENT"])
        node_group2 = TestNodeGroup(
            'slave', [server2],
            ["TASKTRACKER", "DATANODE", "AMBARI_AGENT",
             "GANGLIA_MONITOR"])

        cluster = base.TestCluster([node_group1, node_group2])
        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster_config.create_operational_config(cluster, [user_input])
        self.assertEqual(
            'foo', cluster_config.configurations['global']['new_property'])

    def test_topology_configuration_no_hypervisor(self, patched):
        s_conf = s.CONF
        th_conf = th.CONF
        try:
            s.CONF = TestCONF(True, False)
            th.CONF = TestCONF(True, False)
            cluster_config_file = pkg.resource_string(
                version.version_info.package,
                'plugins/hdp/versions/version_1_3_2/resources/'
                'default-cluster.template')

            server1 = base.TestServer('host1', 'test-master', '11111', 3,
                                      '111.11.1111', '222.11.1111')
            server2 = base.TestServer('host2', 'test-slave', '11111', 3,
                                      '222.22.2222', '333.22.2222')

            node_group1 = TestNodeGroup(
                'master', [server1], ["NAMENODE", "JOBTRACKER",
                                      "SECONDARY_NAMENODE", "GANGLIA_SERVER",
                                      "GANGLIA_MONITOR", "NAGIOS_SERVER",
                                      "AMBARI_SERVER", "AMBARI_AGENT"])
            node_group2 = TestNodeGroup(
                'slave', [server2], ["TASKTRACKER", "DATANODE", "AMBARI_AGENT",
                                     "GANGLIA_MONITOR"])

            cluster = base.TestCluster([node_group1, node_group2])
            cluster_config = cs.ClusterSpec(cluster_config_file)
            cluster_config.create_operational_config(cluster, [])
            # core-site
            self.assertEqual(
                'org.apache.hadoop.net.NetworkTopology',
                cluster_config.configurations['core-site']
                ['net.topology.impl'])
            self.assertEqual(
                'true',
                cluster_config.configurations['core-site']
                ['net.topology.nodegroup.aware'])
            self.assertEqual(
                'org.apache.hadoop.hdfs.server.namenode.'
                'BlockPlacementPolicyWithNodeGroup',
                cluster_config.configurations['core-site']
                ['dfs.block.replicator.classname'])
            self.assertEqual(
                'true',
                cluster_config.configurations['core-site']
                ['fs.swift.service.sahara.location-aware'])
            self.assertEqual(
                'org.apache.hadoop.net.ScriptBasedMapping',
                cluster_config.configurations['core-site']
                ['topology.node.switch.mapping.impl'])
            self.assertEqual(
                '/etc/hadoop/conf/topology.sh',
                cluster_config.configurations['core-site']
                ['topology.script.file.name'])

            # mapred-site
            self.assertEqual(
                'true',
                cluster_config.configurations['mapred-site']
                ['mapred.jobtracker.nodegroup.aware'])
            self.assertEqual(
                '3',
                cluster_config.configurations['mapred-site']
                ['mapred.task.cache.levels'])
            self.assertEqual(
                'org.apache.hadoop.mapred.JobSchedulableWithNodeGroup',
                cluster_config.configurations['mapred-site']
                ['mapred.jobtracker.jobSchedulable'])
        finally:
            s.CONF = s_conf
            th.CONF = th_conf

    def test_topology_configuration_with_hypervisor(self, patched):
        s_conf = s.CONF
        try:
            s.CONF = TestCONF(True, True)
            cluster_config_file = pkg.resource_string(
                version.version_info.package,
                'plugins/hdp/versions/version_1_3_2/resources/'
                'default-cluster.template')

            server1 = base.TestServer('host1', 'test-master', '11111', 3,
                                      '111.11.1111', '222.11.1111')
            server2 = base.TestServer('host2', 'test-slave', '11111', 3,
                                      '222.22.2222', '333.22.2222')

            node_group1 = TestNodeGroup(
                'master', [server1], ["NAMENODE", "JOBTRACKER",
                                      "SECONDARY_NAMENODE", "GANGLIA_SERVER",
                                      "GANGLIA_MONITOR", "NAGIOS_SERVER",
                                      "AMBARI_SERVER", "AMBARI_AGENT"])
            node_group2 = TestNodeGroup(
                'slave', [server2], ["TASKTRACKER", "DATANODE", "AMBARI_AGENT",
                                     "GANGLIA_MONITOR"])

            cluster = base.TestCluster([node_group1, node_group2])
            cluster_config = cs.ClusterSpec(cluster_config_file)
            cluster_config.create_operational_config(cluster, [])
            # core-site
            self.assertEqual(
                'org.apache.hadoop.net.NetworkTopologyWithNodeGroup',
                cluster_config.configurations['core-site']
                ['net.topology.impl'])
        finally:
            s.CONF = s_conf

    def test_update_ambari_admin_user(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        user_input_config = TestUserInputConfig('ambari-stack', 'AMBARI',
                                                'ambari.admin.user')
        user_input = provisioning.UserInput(user_input_config, 'new-user')

        server1 = base.TestServer('host1', 'test-master', '11111', 3,
                                  '111.11.1111', '222.11.1111')
        server2 = base.TestServer('host2', 'test-slave', '11111', 3,
                                  '222.22.2222', '333.22.2222')

        node_group1 = TestNodeGroup(
            'master', [server1],
            ["NAMENODE", "JOBTRACKER",
             "SECONDARY_NAMENODE", "GANGLIA_SERVER", "GANGLIA_MONITOR",
             "NAGIOS_SERVER", "AMBARI_SERVER", "AMBARI_AGENT"])
        node_group2 = TestNodeGroup(
            'slave', [server2],
            ["TASKTRACKER", "DATANODE",
             "AMBARI_AGENT", "GANGLIA_MONITOR"])

        cluster = base.TestCluster([node_group1, node_group2])
        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster_config.create_operational_config(cluster, [user_input])
        ambari_service = next(service for service in cluster_config.services
                              if service.name == 'AMBARI')
        users = ambari_service.users
        self.assertEqual(1, len(users))
        self.assertEqual('new-user', users[0].name)

    def test_update_ambari_admin_password(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        user_input_config = TestUserInputConfig('ambari-stack', 'AMBARI',
                                                'ambari.admin.password')
        user_input = provisioning.UserInput(user_input_config, 'new-pwd')

        server1 = base.TestServer('host1', 'test-master', '11111', 3,
                                  '111.11.1111', '222.11.1111')
        server2 = base.TestServer('host2', 'test-slave', '11111', 3,
                                  '222.22.2222', '333.22.2222')

        node_group1 = TestNodeGroup(
            'master', [server1],
            ["NAMENODE", "JOBTRACKER",
             "SECONDARY_NAMENODE", "GANGLIA_SERVER", "GANGLIA_MONITOR",
             "NAGIOS_SERVER", "AMBARI_SERVER", "AMBARI_AGENT"])
        node_group2 = TestNodeGroup(
            'slave', [server2],
            ["TASKTRACKER", "DATANODE",
             "AMBARI_AGENT", "GANGLIA_MONITOR"])

        cluster = base.TestCluster([node_group1, node_group2])
        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster_config.create_operational_config(cluster, [user_input])
        ambari_service = next(service for service in cluster_config.services
                              if service.name == 'AMBARI')
        users = ambari_service.users
        self.assertEqual(1, len(users))
        self.assertEqual('new-pwd', users[0].password)

    def test_update_ambari_admin_user_and_password(self, patched):
        cluster_config_file = pkg.resource_string(
            version.version_info.package,
            'plugins/hdp/versions/version_1_3_2/resources/'
            'default-cluster.template')

        user_user_input_config = TestUserInputConfig('ambari-stack', 'AMBARI',
                                                     'ambari.admin.user')
        pwd_user_input_config = TestUserInputConfig('ambari-stack', 'AMBARI',
                                                    'ambari.admin.password')
        user_user_input = provisioning.UserInput(user_user_input_config,
                                                 'new-admin_user')
        pwd_user_input = provisioning.UserInput(pwd_user_input_config,
                                                'new-admin_pwd')

        server1 = base.TestServer('host1', 'test-master', '11111', 3,
                                  '111.11.1111', '222.11.1111')
        server2 = base.TestServer('host2', 'test-slave', '11111', 3,
                                  '222.22.2222', '333.22.2222')

        node_group1 = TestNodeGroup(
            'one', [server1], ["NAMENODE", "JOBTRACKER",
                               "SECONDARY_NAMENODE", "GANGLIA_SERVER",
                               "GANGLIA_MONITOR", "NAGIOS_SERVER",
                               "AMBARI_SERVER", "AMBARI_AGENT"])
        node_group2 = TestNodeGroup(
            'two', [server2], ["TASKTRACKER", "DATANODE",
                               "AMBARI_AGENT", "GANGLIA_MONITOR"])

        cluster = base.TestCluster([node_group1, node_group2])
        cluster_config = cs.ClusterSpec(cluster_config_file)
        cluster_config.create_operational_config(
            cluster, [user_user_input, pwd_user_input])
        ambari_service = next(service for service in cluster_config.services
                              if service.name == 'AMBARI')
        users = ambari_service.users
        self.assertEqual(1, len(users))
        self.assertEqual('new-admin_user', users[0].name)
        self.assertEqual('new-admin_pwd', users[0].password)

    def test_validate_missing_hdfs(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["TASKTRACKER", "MAPREDUCE_CLIENT"])

        node_group2 = TestNodeGroup(
            'master', [server2], ["JOBTRACKER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing hdfs service
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.RequiredServiceMissingException:
            # expected
            pass

    def test_validate_missing_mr(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["NAMENODE"])

        node_group2 = TestNodeGroup(
            'master', [server2], ["DATANODE"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing mr service
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.RequiredServiceMissingException:
            # expected
            pass

    def test_validate_missing_ambari(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["NAMENODE", "JOBTRACKER"])

        node_group2 = TestNodeGroup(
            'master', [server2], ["DATANODE", "TASKTRACKER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing ambari service
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.RequiredServiceMissingException:
            # expected
            pass

    # TODO(jspeidel): move validate_* to test_services when validate
    # is called independently of cluspterspec
    def test_validate_hdfs(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER",
                                "HDFS_CLIENT", "MAPREDUCE_CLIENT"], 1)

        node_group2 = TestNodeGroup(
            'master', [server2], ["JOBTRACKER", "AMBARI_SERVER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing NN
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "AMBARI_SERVER"])
        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should validate successfully now
        cluster_config.create_operational_config(cluster, [])

        # should cause validation exception due to 2 NN
        node_group3 = TestNodeGroup(
            'master2', [server2], ["NAMENODE"])
        cluster = base.TestCluster([node_group, node_group2, node_group3])
        cluster_config = base.create_clusterspec()
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

    def test_validate_mr(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER",
                                "HDFS_CLIENT", "MAPREDUCE_CLIENT"])
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "AMBARI_SERVER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing JT
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "AMBARI_SERVER"])
        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should validate successfully now
        cluster_config.create_operational_config(cluster, [])

        # should cause validation exception due to 2 JT
        node_group3 = TestNodeGroup(
            'master', [server2], ["JOBTRACKER"])
        cluster = base.TestCluster([node_group, node_group2, node_group3])
        cluster_config = base.create_clusterspec()
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

        # should cause validation exception due to 2 NN
        node_group3 = TestNodeGroup(
            'master', [server2], ["NAMENODE"])
        cluster = base.TestCluster([node_group, node_group2, node_group3])
        cluster_config = base.create_clusterspec()
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

        # should fail due to no TT
        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "HDFS_CLIENT",
                                "MAPREDUCE_CLIENT"])
        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing JT
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

    def test_validate_hive(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER",
                                "HIVE_CLIENT"])
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "AMBARI_SERVER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing hive_server
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "HIVE_SERVER", "AMBARI_SERVER"])
        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should validate successfully now
        cluster_config.create_operational_config(cluster, [])

        # should cause validation exception due to 2 HIVE_SERVER
        node_group3 = TestNodeGroup(
            'master', [server2], ["HIVE_SERVER"])
        cluster = base.TestCluster([node_group, node_group2, node_group3])
        cluster_config = base.create_clusterspec()
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

    def test_validate_zk(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')
        server3 = base.TestServer('host3', 'master', '11113', 3,
                                  '111.11.1113', '222.22.2224')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER",
                                "ZOOKEEPER_CLIENT"])
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "AMBARI_SERVER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing ZOOKEEPER_SERVER
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "ZOOKEEPER_SERVER", "AMBARI_SERVER"])
        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should validate successfully now
        cluster_config.create_operational_config(cluster, [])

        # should allow multiple ZOOKEEPER_SERVER processes
        node_group3 = TestNodeGroup(
            'zkserver', [server3], ["ZOOKEEPER_SERVER"])
        cluster = base.TestCluster([node_group, node_group2, node_group3])
        cluster_config = base.create_clusterspec()
        cluster_config.create_operational_config(cluster, [])

    def test_validate_oozie(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER",
                                "OOZIE_CLIENT"])
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "AMBARI_SERVER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing OOZIE_SERVER
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "OOZIE_SERVER", "AMBARI_SERVER"])
        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should validate successfully now
        cluster_config.create_operational_config(cluster, [])

        # should cause validation exception due to 2 OOZIE_SERVER
        node_group3 = TestNodeGroup(
            'master', [server2], ["OOZIE_SERVER"])
        cluster = base.TestCluster([node_group, node_group2, node_group3])
        cluster_config = base.create_clusterspec()
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

    def test_validate_ganglia(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER",
                                "GANGLIA_MONITOR"])
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "AMBARI_SERVER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing GANGLIA_SERVER
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "GANGLIA_SERVER", "AMBARI_SERVER"])
        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should validate successfully now
        cluster_config.create_operational_config(cluster, [])

        # should cause validation exception due to 2 GANGLIA_SERVER
        node_group3 = TestNodeGroup(
            'master2', [server2], ["GANGLIA_SERVER"])
        cluster = base.TestCluster([node_group, node_group2, node_group3])
        cluster_config = base.create_clusterspec()
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

    def test_validate_ambari(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER",
                                "AMBARI_AGENT"])
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should fail due to missing AMBARI_SERVER
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "AMBARI_SERVER"])
        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # should validate successfully now
        cluster_config.create_operational_config(cluster, [])

        # should cause validation exception due to 2 AMBARI_SERVER
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "AMBARI_SERVER"])
        node_group3 = TestNodeGroup(
            'master', [server2], ["AMBARI_SERVER"])
        cluster = base.TestCluster([node_group, node_group2, node_group3])
        cluster_config = base.create_clusterspec()
        try:
            cluster_config.create_operational_config(cluster, [])
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

    def test_validate_scaling_existing_ng(self, patched):
        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER"])
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER",
                                  "AMBARI_SERVER"])

        cluster = base.TestCluster([node_group, node_group2])
        cluster_config = base.create_clusterspec()
        # sanity check that original config validates
        cluster_config.create_operational_config(cluster, [])

        cluster_config = base.create_clusterspec()
        scaled_groups = {'master': 2}
        # should fail due to 2 JT
        try:
            cluster_config.create_operational_config(
                cluster, [], scaled_groups)
            self.fail('Validation should have thrown an exception')
        except ex.InvalidComponentCountException:
            # expected
            pass

    def test_scale(self, patched):

        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER",
                                "AMBARI_AGENT"])
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER", "AMBARI_SERVER"])

        cluster = base.TestCluster([node_group, node_group2])

        cluster_config = base.create_clusterspec()
        # sanity check that original config validates
        cluster_config.create_operational_config(cluster, [])

        slave_ng = cluster_config.node_groups['slave']
        self.assertEqual(1, slave_ng.count)

        cluster_config.scale({'slave': 2})

        self.assertEqual(2, slave_ng.count)

    def test_get_deployed_configurations(self, patched):

        server = base.TestServer('host1', 'slave', '11111', 3,
                                 '111.11.1111', '222.22.2222')
        server2 = base.TestServer('host2', 'master', '11112', 3,
                                  '111.11.1112', '222.22.2223')

        node_group = TestNodeGroup(
            'slave', [server], ["DATANODE", "TASKTRACKER"])
        node_group2 = TestNodeGroup(
            'master', [server2], ["NAMENODE", "JOBTRACKER", "AMBARI_SERVER"])

        cluster = base.TestCluster([node_group, node_group2])

        cluster_config = base.create_clusterspec()
        # sanity check that original config validates
        cluster_config.create_operational_config(cluster, [])
        configs = cluster_config.get_deployed_configurations()
        expected_configs = set(['mapred-site', 'ambari', 'hdfs-site',