"""Compatibility tools between Python 2 and Python 3 I/O interfaces.
"""

from __future__ import print_function
from __future__ import unicode_literals

import io
from io import SEEK_SET, SEEK_CUR

from .mode import Mode

class RawWrapper(io.IOBase):
    """Convert a Python 2 style file-like object in to a IO object.
    """

    def __init__(self, f, mode=None, name=None):
        self._f = f
        self.mode = mode or getattr(f, 'mode', None)
        self.name = name
        super(RawWrapper, self).__init__()

    def close(self):
        if not self.closed:
            # Close self first since it will
            # flush itself, so we can't close
            # self._f before that
            super(RawWrapper, self).close()
            self._f.close()

    def fileno(self):
        return self._f.fileno()

    def flush(self):
        return self._f.flush()

    def isatty(self):
        return self._f.isatty()

    def seek(self, offset, whence=SEEK_SET):
        return self._f.seek(offset, whence)

    def readable(self):
        return getattr(
            self._f,
            'readable',
            lambda: Mode(self.mode).reading
        )()

    def writable(self):
        return getattr(
            self._f,
            'writable',
            lambda: Mode(self.mode).writing
        )()

    def seekable(self):
        try:
            return self._f.seekable()
        except AttributeError:
            try:
                self.seek(0, SEEK_CUR)
            except IOError:
                return False
            else:
                return True

    def tell(self):
        return self._f.tell()

    def truncate(self, size=None):
        return self._f.truncate(size)

    def write(self, data):
        count = self._f.write(data)
        return len(data) if count is None else count

    def read(self, n=-1):
        if n == -1:
            return self.readall()
        return self._f.read(n)

    def read1(self, n=-1):
        if hasattr(self._f, 'read1'):
            return self._f.read1(n)
        return self.read(n)

    def readall(self):
        return self._f.read()

    def readinto(self, b):
        try:
            return self._f.readinto(b)
        except AttributeError:
            data = self._f.read(len(b))
            bytes_read = len(data)
            b[:len(data)] = data
            return bytes_read

    def readinto1(self, b):
        try:
            return self._f.readinto1(b)
        except AttributeError:
            data = self._f.read1(len(b))
            bytes_read = len(data)
            b[:len(data)] = data
            return bytes_read

    def readline(self, limit=-1):
        return self._f.readline(limit)

    def readlines(self, hint=-1):
        return self._f.readlines(hint)

    def writelines(self, sequence):
        return self._f.writelines(sequence)

    def __iter__(self):
        return iter(self._f)

def make_stream(name,
                bin_file,
                mode='r',
                buffering=-1,
                encoding=None,
                errors=None,
                newline='',
                line_buffering=False,
                **kwargs):
    """Take a Python 2.x binary file and return an IO Stream.
    """
    reading = 'r' in mode
    writing = 'w' in mode
    appending = 'a' in mode
    binary = 'b' in mode
    if '+' in mode:
        reading = True
        writing = True

    encoding = None if binary else (encoding or 'utf-8')

    io_object = RawWrapper(bin_file, mode=mode, name=name)
    if buffering >= 0:
        if reading and writing:
            io_object = io.BufferedRandom(
                io_object,
                buffering or io.DEFAULT_BUFFER_SIZE
            )
        elif reading:
            io_object = io.BufferedReader(
                io_object,
                buffering or io.DEFAULT_BUFFER_SIZE
            )
        elif writing or appending:
            io_object = io.BufferedWriter(
                io_object,
                buffering or io.DEFAULT_BUFFER_SIZE
            )

    if not binary:
        io_object = io.TextIOWrapper(
            io_object,
            encoding=encoding,
            errors=errors,
            newline=newline,
            line_buffering=line_buffering,
        )

    return io_object

def line_iterator(readable_file, size=None):
    """Iterate over the lines of a file.

    Implementation reads each char individually, which is not very
    efficient.

    Yields:
        str: a single line in the file.

    """
    read = readable_file.read
    line = []
    byte = b'1'
    if size is None or size < 0:
        while byte:
            byte = read(1)
            line.append(byte)
            if byte in b'\n':
                yield b''.join(line)
                del line[:]

    else:
        while byte and size:
            byte = read(1)
            size -= len(byte)
            line.append(byte)
            if byte in b'\n' or not size:
                yield b''.join(line)
                del line[:]


"""
=====================================
Get the coordinates of the student ID

Author: Zheng Zhihuang

Date: 2019.7.12
=====================================

A script for Python3.

Use template matching to locate the XueHao region.
使用模版匹配的方法定位学号区域。

"""

# Use matchTemplate function of opencv
import cv2 as cv

# Standard scientific Python imports
import numpy as np

def xuehao(imagePath):
    """Fetches the position of XueHao region from a paper image.

    Use prepared template to get the first match position in image.
    The result is precise, but it may not apply to some kind of
    papers with different fonts because the template is changeless
    and the size of region is fixed (260*83pixel) too.

    Args:
        imagePath: The file path of image.

    Returns:
        An array with four integer elements includes the coordinates of
        the upper left corner and the lower right corner of the xuehao
        region. For example:

        [1, 2, 3, 4]

    Raises:
        IOError: An error occurred accessing the imagePath object.

    """
    imgRgb = cv.imread(imagePath)
    imgGray = cv.cvtColor(imgRgb, cv.COLOR_BGR2GRAY)
    template = cv.imread('xuehao_templates/xuehao.jpg', 0)
    w, h = template.shape[::-1]
    res = cv.matchTemplate(imgGray, template, cv.TM_CCOEFF_NORMED)
    threshold = 0.8
    loc = np.where(res >= threshold)
    idPosition = []
    for pt in zip(*loc[::-1]):
        idPosition = [pt[0]+w, pt[1]-h/2, pt[0]+w+260, pt[1]+h]
        break
    return [int(x) for x in idPosition]


from django.shortcuts import render, redirect
from . models import Produto
from . forms import ProdutoForm
from . forms import PesquisaForm

def home (request):
     return render(request, 'home.html')

def produto_list (request):
    produtos=Produto.objects.all()
    return render(request, 'list.html', {'produtos': produtos})

def produto_show(request, produto_id):
    produto=Produto.objects.get(pk=produto_id)
    return render(request, 'show.html', {'produto':produto})

def produto_form(request):
    if (request.method == 'POST'):
        form = ProdutoForm(request.POST)
        if form.is_valid():
           form.save()
           return redirect ('/primeira/produtos/')
        else:
            return render(request, 'form.html', {'form':form})

    else:
        form = ProdutoForm()
        return render(request, 'form.html', {'form': form})

def excluir_produto(request, produto_id):
     produto=Produto.objects.get(pk=produto_id)
     produto.delete()
     return redirect ('/primeira/produtos/')

def editar_produto (request, produto_id):
     if (request.method == 'POST'):
          produto=Produto.objects.get(pk=produto_id)
          form = ProdutoForm(request.POST, instance = produto)
          if form.is_valid():

              form.save()
              return redirect ('/primeira/produtos/')
          else:
              return render (request, 'editar.html', {'form':form, 'produto_id': produto_id})

     else:
         produto=Produto.objects.get(pk=produto_id)
         form = ProdutoForm(instance = produto)
         return render (request, 'editar.html', {'form':form, 'produto_id': produto_id})

def pesquisa_form(request):
    if (request.method == 'POST'):
        form = PesquisaForm(request.POST)
        if form.is_valid():
           form.save()
           return redirect ('/primeira/produtos/')
        else:
            return render(request, 'list.html', {'form':form})

    else:
        form = PesquisaForm()
        return render(request, 'list.html', {'form': form})
#def pesquisa (request):

# Create your views here.


# -*- coding:utf-8 -*-

from spider.driver.base.field import Fieldlist,Field,FieldName
from spider.driver.base.tabsetup import TabSetup
from spider.driver.base.page import Page,NextPageCssSelectorSetup,PageFunc
from spider.driver.base.listcssselector import ListCssSelector
from spider.driver.base.mongodb import Mongodb
from spider.driver.travel.core.traveldriver import TravelDriver
import time
from pyquery import PyQuery
import json
import re
import random

def get_shop_tag(self, _str):
    p = PyQuery(_str)
    tag_list = []
    for i in list(p('span').items())[1:]:
        tag_list.append(i.text())
    return json.dumps(tag_list, ensure_ascii=False)

def get_shop_rate(self, _str):
    return str(float((int(_str)/10)))

fl_shop1 = Fieldlist(
    Field(fieldname=FieldName.SHOP_PRICE, css_selector='div.hotel-info-ctn > div.hotel-remark > div.price > p > strong'),
    Field(fieldname=FieldName.SHOP_NAME, css_selector='div.hotel-info-ctn > div.hotel-info-main > h2 > a.hotel-name-link'),
    Field(fieldname=FieldName.SHOP_RATE, css_selector='div.hotel-info-ctn > div.hotel-remark > div.remark > div > div > span', attr='class', regex=r'[^\d]*', filter_func=get_shop_rate),
    # Field(fieldname=FieldName.SHOP_TAG, css_selector='div.hotel-info-ctn > div.hotel-info-main > p.hotel-tags', attr='innerHTML', filter_func=get_shop_tag, pause_time=3),
    Field(fieldname=FieldName.SHOP_URL,css_selector='',attr='href',is_info=True),
    Field(fieldname=FieldName.SHOP_IMG,css_selector='',attr='src',is_info=True),
    Field(fieldname=FieldName.SHOP_ADDRESS,css_selector='',is_info=True),
    Field(fieldname=FieldName.SHOP_GRADE,css_selector='',is_info=True),
    Field(fieldname=FieldName.SHOP_COMMENT_NUM,css_selector='',is_info=True),
    Field(fieldname=FieldName.SHOP_FEATURE,css_selector='',is_info=True)

)

def get_shop_room_all(self, _str):
    p = PyQuery(_str)
    sale_dict = {}
    room_list = []
    for i in p('div.hotel-rooms > div.hotel-rooms-list > div.hotel-rooms-list-cont > ul > li').items():
        room = {'room_name': i('div.title-info.clearfix.dph-col.dph-col1 > div.title > h3').text()}
        for j in i('div.h-item-more.h-hide').text().split('\n'):
            room.update((lambda x: {x[0].strip(): x[1].strip()} if len(x) == 2 else {})(j.split(':')))
        item_list = []
        for j in i('div.roomlist > div').items():
            item_list.append(j.text().split('\n'))
        room.setdefault('room_list', item_list)
        room_list.append(room)
    sale_dict.setdefault('房型预定', room_list)
    package_list = []
    for i in p('div > ul.group-deal > li').items():
        info_list = i.text().split('￥')
        if len(info_list) == 2:
            package_list.append({info_list[0].strip(): info_list[1].strip()})
    sale_dict.setdefault('套餐预定', package_list)
    return json.dumps(sale_dict, ensure_ascii=False)

def get_shop_intro(self, _str):
    p = PyQuery(_str)
    info_dict = {}
    for i in p('ul.list-info > li').items():
        info = i.text().split('\n')
        if len(info) == 2:
            items = list(i('div > span').items())
            item_list = []
            if len(items) >= 2:
                for j in items:
                    item_list.append(j.text())
                info[1] = item_list
            info_dict.setdefault(info[0], info[1])
    return json.dumps(info_dict, ensure_ascii=False)

def get_shop_statistics(self, _str):
    p = PyQuery(_str)
    tag_star_dict = {}
    tags = {}
    for i in p('div.tags > ul > li').items():
        tags.setdefault(re.sub('[^\u4e00-\u9fa5]*', '', i.text()), re.sub('[^\d]*', '', i.text()))
    tag_star_dict.setdefault('tags', tags)
    stars = {}
    for i in p('#comment > div > h2 > span > a').items():
        star = i.text().split('星')
        if len(star) == 2:
            stars.setdefault(star[0], re.sub(r'[^\d]*', '', star[1]))
    tag_star_dict.setdefault('star', stars)
    return json.dumps(tag_star_dict, ensure_ascii=False)

fl_shop2 = Fieldlist(
    Field(fieldname=FieldName.SHOP_GRADE, css_selector='#poi-detail > div.container > div.base-info > div.main-detail.clearfix > div.main-detail-right > div.hotel-appraise > div.hotel-scope > span', pause_time=5, is_focus=True),
    Field(fieldname=FieldName.SHOP_PHONE, css_selector='#poi-detail > div.container > div.base-info > div.main-detail.clearfix > div.main-detail-left > div.main-detail-left-top.clearfix > div.hotel-detail-info > div > div.call-info > div > span.call-number', is_focus=True),
    Field(fieldname=FieldName.SHOP_ADDRESS, css_selector='#poi-detail > div.container > div.base-info > div.main-detail.clearfix > div.main-detail-left > div.main-detail-left-top.clearfix > div.hotel-detail-price > div.hotel-address-box.clearfix > span.hotel-address', is_focus=True),
    Field(fieldname=FieldName.SHOP_ROOM_RECOMMEND_ALL, css_selector='#deal', attr='innerHTML', filter_func=get_shop_room_all, is_focus=True),
    Field(fieldname=FieldName.SHOP_INTRO, css_selector='#poi-detail > div.container > div.sub-content.clearfix > div.main > div> div.hotel-info', attr='innerHTML', filter_func=get_shop_intro, is_focus=True),
    Field(fieldname=FieldName.SHOP_STATISTICS, css_selector='#poi-detail > div.container > div.sub-content.clearfix > div.main > div.user-comment-info', attr='innerHTML', filter_func=get_shop_statistics, is_focus=True),
    Field(fieldname=FieldName.SHOP_COMMENT_NUM, css_selector='#comment > div > h2 > a > span.count', regex=r'[^\d]*'),
)

page_shop_1 = Page(name='大众点评酒店店铺列表页面', fieldlist=fl_shop1, listcssselector=ListCssSelector(list_css_selector='#poi-list > div.content-wrap > div > div.list-wrapper > div.content > ul > li',item_start=11,item_end=12), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection))

page_shop_2 = Page(name='大众点评酒店店铺详情页面', fieldlist=fl_shop2, tabsetup=TabSetup(click_css_selector='div.hotel-info-ctn > div.hotel-info-main > h2 > a.hotel-name-link'),mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.shop_collection), is_save=True)

def get_rate(self, _str):
    return str(int(re.sub('[^\d]*','',_str))/10)

def get_comment_rate_tag(self, _str):
    p = PyQuery(_str)
    tag_list = []
    for i in p('span.item').items():
        tag_list.append(i.text().strip())
    return json.dumps(tag_list, ensure_ascii=False)

def get_comment_content(self, _str):
    return PyQuery(_str).text().replace('收起评论','')

fl_comment1 = Fieldlist(
    Field(fieldname=FieldName.SHOP_NAME, css_selector='#review-list > div.review-list-container > div.review-list-main > div.review-list-header > h1 > a', is_isolated=True),
    Field(fieldname=FieldName.COMMENT_USER_NAME, css_selector='div > div.dper-info > a'),
    Field(fieldname=FieldName.COMMENT_TIME, css_selector='div > div.misc-info.clearfix > span.time'),
    Field(fieldname=FieldName.COMMENT_USER_RATE, css_selector='div > div.dper-info > span', attr='class', filter_func=get_rate),
    Field(fieldname=FieldName.COMMENT_RATE, css_selector='div > div.review-rank > span.sml-rank-stars', attr='class', filter_func=get_rate),
    Field(fieldname=FieldName.COMMENT_RATE_TAG, css_selector='div > div.review-rank > span.score', attr='innerHTML', filter_func=get_comment_rate_tag),
    Field(fieldname=FieldName.COMMENT_CONTENT, css_selector='div > div.review-words.Hide', attr='innerHTML', filter_func=get_comment_content),
    Field(fieldname=FieldName.COMMENT_PIC_LIST, list_css_selector='div > div.review-pictures > ul', item_css_selector='li > a > img', attr='src', timeout=0),
)

page_comment_1 = Page(name='大众点评酒店评论列表', fieldlist=fl_comment1, listcssselector=ListCssSelector(list_css_selector='#review-list > div.review-list-container > div.review-list-main > div.reviews-wrapper > div.reviews-items > ul > li'), mongodb=Mongodb(db=TravelDriver.db, collection=TravelDriver.comments_collection), is_save=True)

class DianpingHotelSpider(TravelDriver):

    def more_comment(self):
        while(True):
            self.is_ready_by_proxy_ip()
            self.switch_window_by_index(index=-1)
            self.deal_with_failure_page()
            self.until_scroll_to_center_click_by_css_selector(css_selector='#comment > div > div.comment > div.more-comment > a.dp-link')
            time.sleep(2)
            self.switch_window_by_index(index=-1)  # 页面选择
            if '验证中心' in self.driver.title:
                self.info_log(data='关闭验证页面!!!')
                self.close_curr_page()
            else:
                break
        # time.sleep(1)
        # while(True):
        #     self.until_scroll_to_center_click_by_css_selector(css_selector='#review-list > div.review-list-container > div.review-list-main > div.reviews-wrapper > div.reviews-filter.clearfix > div.sort > a')
        #     time.sleep(1)
        #     self.is_ready_by_proxy_ip()
        #     self.until_scroll_to_center_click_by_css_selector(css_selector='#review-list > div.review-list-container > div.review-list-main > div.reviews-wrapper > div.reviews-filter.clearfix > div.sort-selection-list > a')
        #     time.sleep(1)
        #     self.switch_window_by_index(index=-1)
        #     if '验证' in self.driver.title:#如果是验证页面
        #         self.driver.back()
        #     else:
        #         break

    def get_shop_info(self):
        try:
            shop_data_list = self.from_page_get_data_list(page=page_shop_1)
            nextpagesetup = NextPageCssSelectorSetup(css_selector='#review-list > div.review-list-container > div.review-list-main > div.reviews-wrapper > div.bottom-area.clearfix > div > a.NextPage',page=page_comment_1, pause_time=2, pre_pagefunc=PageFunc(func=self.more_comment), after_pagefunc=PageFunc(func=self.close_curr_page))
            extra_pagefunc = PageFunc(func=self.get_newest_comment_data_by_css_selector, nextpagesetup=nextpagesetup, shop_name_css_selector='#poi-detail > div.container > div.base-info > div.main-detail.clearfix > div.main-detail-left > div.main-detail-left-top.clearfix > div.hotel-detail-info > div > h1', is_effective=False)
            self.from_page_add_data_to_data_list(page=page_shop_2, pre_page=page_shop_1, data_list=shop_data_list, extra_pagefunc=extra_pagefunc)
        except Exception as e:
            self.error_log(e=str(e))

    def get_shop_info_list(self):
        self.fast_get_page(url='https://www.baidu.com')
        time.sleep(2)
        self.until_scroll_to_center_send_text_by_css_selector(css_selector='#kw', text=self.data_region + self.data_website)
        self.until_scroll_to_center_send_enter_by_css_selector(css_selector='#kw')
        self.fast_click_first_item_page_by_partial_link_text(link_text=self.data_website)
        with open('./cookies/dianping_cookies.json', 'r', encoding='utf-8') as f:
            listCookies = json.loads(f.read())
        for cookie in listCookies:
            self.driver.add_cookie(cookie)
        self.close_curr_page()
        self.fast_click_first_item_page_by_partial_link_text(link_text=self.data_website)
        time.sleep(2)
        self.fast_click_first_item_page_by_partial_link_text(link_text='酒店')
        self.debug_log(data='暂停20秒......')
        time.sleep(5)
        self.until_click_no_next_page_by_css_selector(nextpagesetup=NextPageCssSelectorSetup(css_selector='#poi-list > div.content-wrap > div > div.page > a.next', main_pagefunc=PageFunc(func=self.get_shop_info), is_next=False))

    def run_spider(self):
        try:
            self.get_shop_info_list()
        except Exception as e:
            self.error_log(e=str(e))


from brl_gym.wrapper_envs.wrapper_doors import Expert, ExplicitBayesDoorsEnv
import numpy as np
if __name__ == "__main__":

    expert = Expert()

    all_rewards = []
    num_sensing = []
    for _ in range(100):
        env = ExplicitBayesDoorsEnv(reward_entropy=False)
        o = env.reset()

        rewards = []
        sensing = 0
        for t in range(300):
            action = expert.action(
                np.concatenate([o['obs'], o['zbel']]).reshape(1, -1)).ravel()

            # print(action)
            # if t < 100:
            action[2] = action[2] + np.random.normal()*0.1
            o, r, d, _ = env.step(action)
            # env.render()
            # print(np.around(o['zbel'], 2))
            rewards += [r]
            if action[2] > 0.0:
                sensing += 1

            if d:
                print(t)
                break

        undiscounted_sum = np.sum(rewards)
        num_sensing += [sensing]
        all_rewards += [undiscounted_sum]
        print('undiscounted sum', undiscounted_sum, "sensing", sensing)

    print("stat", np.mean(all_rewards), np.std(all_rewards) / np.sqrt(len(all_rewards)))
    print("sensing", np.mean(num_sensing), np.std(num_sensing) / np.sqrt(len(num_sensing)))
     # -108.5296 16.521
    import IPython; IPython.embed();


import random

import numpy as np

def find_index(ary, data):
    global count
    pos = -1
    start = 0
    end = len(ary)-1
    while end >= start:
        count = count + 1
        mid = (start+end)//2
        if data == ary[mid]:
            return mid
        elif data > ary[mid]:
            start = mid + 1
        else:
            end = mid + 1
    return pos

data_array = np.random.randint(0, 100000, 100000)
data_array.sort()
find_data = np.random.randint(100000)
count = 0

position = find_index(data_array, find_data)
if position == -1:
    print("찾는 data가 data_array에 없음")
    print(f"{count}회 검색함")
else:
    print(f"찾는 data가 {position}에 있음")
    print(f"{count}회 검색함")


from torchvision import models
import torch.nn as nn
import torch

CONTENT_LAYER = 'relu_16'

def vgg_19(device):

    vgg_19 = models.vgg19(pretrained=True).features
    model = nn.Sequential()

    i = 0
    for layer in vgg_19.children():
        if isinstance(layer, nn.Conv2d):
            i += 1
            name = 'conv_{}'.format(i)
        elif isinstance(layer, nn.ReLU):
            name = 'relu_{}'.format(i)
            layer = nn.ReLU(inplace=False)
        elif isinstance(layer, nn.MaxPool2d):
            name = 'pool_{}'.format(i)
        elif isinstance(layer, nn.BatchNorm2d):
            name = 'bn_{}'.format(i)
        else:
            raise RuntimeError('Unrecognized layer: {}'.format(layer.__class__.__name__))

        model.add_module(name, layer)
        if name == CONTENT_LAYER:
            break

    model = model.to(device)
    model = torch.nn.DataParallel(model)

    for param in model.parameters():
        param.requires_grad = False

    for param in vgg_19.parameters():
        param.requires_grad = False

    return model


# Standard libs
import os, argparse
from pathlib import Path

# Other necessary libs
import openpyxl, yaml               # pip install openpyxl
from operator import itemgetter

def process_args():
    parser = argparse.ArgumentParser(description='Command Line Arguments')
    parser.add_argument('--rootdir', default=Path.cwd(),
                        help='The Working Directory for maps and Models (defaults to Current Directory)')
    parser.add_argument('--schema', default='CHANGEME',
                        help='if not default, becomes DEV_EDW_xxxxx with xxxxx passed as the schema name')
    # boolean
    parser.add_argument( '--yml', action='store_true', help='If flagged as tests, will only generate yml files')
    parser.add_argument( '--silent', action='store_true', help='If flagged will only print file names')
    parser.add_argument( '--debug', action='store_true', help='If flagged will add verbosity to output')

    args = parser.parse_args()

    if not args.schema == 'CHANGEME':
        devedw = f'DEV_EDW_{args.schema}'
        args.schema = devedw

    return args

def setup(args):
    args.mapdir = args.rootdir / 'mappings'
    args.modeldir = args.rootdir / 'models'
    args.snapdir= args.rootdir / 'snapshots'
    args.archivedir = args.rootdir / 'models_archive'

    print( 'Mapping docs (XLS) in:', args.mapdir )
    print( 'Output (SQL, YML) going to:', args.modeldir )

    # Create the Directories if they do not exist yet
    if not args.mapdir.exists(): args.mapdir.mkdir()
    if not args.modeldir.exists(): args.modeldir.mkdir()
    if not args.snapdir.exists(): args.snapdir.mkdir()
    if not args.archivedir.exists(): args.archivedir.mkdir()

    # Show existing XLS files in Mapdir
    for afile in args.mapdir.glob('*.xlsx'):
        print(f'\tMapping found: {afile}')

def sheet2dict(sheet):
    print(f'\t<<<< Processing {sheet}')

    fields = None
    for rownum, row in enumerate( sheet.iter_rows( values_only = True) ):
        if rownum == 0:
            fields = row
            continue
        row = dict(zip( fields, row ))

        yield row

def sheet2list(sheet):
    for row in sheet.iter_rows( values_only = True ):
        if not row:
            continue
        yield row

###### START OF SCD SPECIFIC CODE
def build_scd2( schema, table, cols, keycol ):

    #keycol = 'UNIQUE_DIM_KEY'
    schema = 'EDW_STAGING_SNAPSHOT'
    all_cols = cols + [ keycol ]

    cols_str = ', '.join( all_cols )
    src_table = table.replace( 'DIM_' , 'DSV_' )
    snap=f'''

    {{%% snapshot %s_SNAPSHOT_STEP1 %%}}

    {{{{
        config(
        target_schema='%s',
        unique_key='%s',
        strategy='check',
        check_cols={cols},
        )
    }}}}

    select {cols_str}
        from  {{{{ref('%s') }}}}

    {{%% endsnapshot %%}}
    '''%(table,schema,keycol,src_table)

    return snap

def build_scd6 ( schema, table, keycol, scd1_cols = [], scd2_cols = [], scd3_cols = [], scd0_cols = [] ):
    #print(scd0_cols)
    keycol = keycol.upper()
    #keycol='UNIQUE_DIM_KEY'
    if not keycol: keycol = 'UNIQUE_ID_KEY'

    exclude_col = ['UNIQUE_ID_KEY', 'CURRENT_INDICATOR', 'EFFECTIVE_DATE', 'END_DATE', 'LOAD_DATETIME',
    'UPDATE_DATETIME', 'EFFECTIVE_TIMESTAMP', 'END_TIMESTAMP',
    'LOAD_BATCH_ID', 'UPDATE_BATCH_ID', 'PRIMARY_SOURCE_SYSTEM']

    start_sql = f'''\n\n WITH  SCD AS ( \n\tSELECT  {keycol},'''

    sql2 = []
    for col in scd2_cols:
        if col in exclude_col: continue
        #sql=f'''\n     lag({col}) over
        #          (partition by {keycol}
        #                order by dbt_updated_at
        #           ) as PREV_{col}'''
        sql = f'''\n {col}'''
        sql2.append( sql )

    sql3 = []
    for col in scd3_cols:
        if col in exclude_col: continue
        sql = f'''\n     lag({col}) over
                  (partition by {keycol}
                        order by dbt_updated_at
                   ) as {col}'''
                   #) as PREV_{col}'''
        sql3.append(sql)

    sql1 = []
    for col in scd1_cols:
        if col in exclude_col: continue
        sql=f'''\n     last_value({col}) over
        (partition by {keycol}
        order by dbt_updated_at
            ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
            ) as {col}'''
            #) as CUR_{col}'''
        sql1.append(sql)

    sql0 = []
    for col in scd0_cols:
        if col in exclude_col: continue
        sql=f'''\n     first_value({col}) over
        (partition by {keycol}
        order by dbt_updated_at
            ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
            ) as {col}'''
            #) as CUR_{col}'''
        sql0.append( sql )

    end_sql = f'''\n,DBT_VALID_FROM AS EFFECTIVE_TIMESTAMP \n,DBT_VALID_TO   AS END_TIMESTAMP \n,
    CASE WHEN CAST(DBT_VALID_FROM AS DATE) = '1901-01-01' then CAST(DBT_VALID_FROM AS DATE)
      WHEN CAST(DBT_VALID_FROM AS DATE) <> '1901-01-01' THEN dateadd(day,1,CAST(DBT_VALID_FROM AS DATE))
    else CAST(DBT_VALID_FROM AS DATE) end as EFFECTIVE_DATE\n,
    CAST(DBT_VALID_TO AS DATE) as END_DATE \n,CASE WHEN DBT_VALID_TO IS NULL THEN 'Y' ELSE 'N' END AS CURRENT_INDICATOR'''
    #CAST(DBT_VALID_FROM AS DATE) as EFFECTIVE_DATE\n,
    sql1[ 0 ] = start_sql + sql1[ 0 ]
    scd_all = sql1 + sql2 + sql3 + sql0
    #+ end_sql

    scd_str = ', '.join( scd_all )
    scd_str = scd_str.strip(',')

    scd_str = scd_str + end_sql + '\n'+"     \n\tFROM {{ ref('" + table + "_SCDALL_STEP2') }})\n\n"
    scd_str += 'select * from SCD\n'

    return scd_str

def make_scd( scd_dict ):
    '''
    scd={
        'table':'STG_POLICYTEST',
        'schema':'X10057301',
        'scd1':['POLICY_ID','CLAIM_ID'],
        'scd2':['PTYPE'],
        'scd3':['OWNER_NAME','ADDR'],
        'keycol':'POLICY_ID'
    }
    '''

    keycol = 'KEY'

    #print(scd_dict)
    #cols=scd_dict['scd2']+scd_dict['scd3']+scd_dict['scd0']
    cols = scd_dict[ 'scd2' ] + scd_dict[ 'scd3' ] + scd_dict[ 'scd0' ]

    scd2_sql = build_scd2( scd_dict[ 'schema' ], scd_dict[ 'table' ], cols, scd_dict[ keycol ])

    scd6_sql = build_scd6( scd_dict[ 'schema' ], scd_dict[ 'table' ], scd_dict[ keycol ],
        scd1_cols = scd_dict[ 'scd1' ], scd2_cols = scd_dict[ 'scd2' ], scd3_cols = scd_dict[ 'scd3' ], scd0_cols = scd_dict[ 'scd0' ])

    return scd2_sql, scd6_sql

def write_dbt( table, snapdir, scd2_sql, modeldir, scd6_sql):
    snapfile = snapdir/(table+'_SNAPSHOT_STEP1.sql')
    print('\twriting to', snapfile)
    with open( snapfile, 'w') as fw:
        fw.write( scd2_sql )
    #table_split,split=table.split('_STEP1')
    if table.startswith('DIM'):
        model_file = modeldir/(table+'.sql')
    else:
        model_file = modeldir/(table+'_alt.sql')
    print('\twriting to', model_file)
    with open( model_file, 'w' ) as fw:
        fw.write( scd6_sql )

def get_scdinfo( schema, table, col_rows):
    '''
    {'Order#': None, 'SCD TYPE': None, 'Staging Layer Column Name': None, ' 'CPI': None,
    'Unique': None, 'Not Null': None, 'Test Expression': None, 'Accepted Values': None}
    '''

    scd = {
        'table': table,
        'schema': schema,
        'KEY':'',
        'scd1':[], 'scd2':[], 'scd3':[], 'scd0':[],
    }

    for row in col_rows:
        col = row[ 'Staging Layer Column Name' ]
        if not col: continue
        if str(row[ 'SCD' ]).upper() == 'KEY':  scd[ 'KEY' ] = col.upper()

    for row in col_rows:
        col = row[ 'Staging Layer Column Name' ]
        #print('2',row)
        if not col: continue
        if scd[ 'KEY' ] == col.upper(): continue

        if not row[ 'SCD' ]:
            if row[ 'SCD' ] != 0:
                row[ 'SCD' ] = '1'
        if col not in scd[ 'scd'+ str(row[ 'SCD' ])]:
            scd['scd'+ str(row[ 'SCD' ])].append(col)

        # print('~~SCD:' , row['SCD'] )                                 # DEBUG
        # print('SCD APPEND >>> ',scd['scd' + str(row[ 'SCD' ])])       # DEBUG
        # print('2', scd['scd'+str(row[ 'SCD' ])])                      # DEBUG

    return scd

def write_models( modeldir, tgt_table, scd1_sql, scd2_sql):
    scd1_file = modeldir/(tgt_table+'_SCD1.sql')
    print('\twriting to', scd1_file)
    with open( scd1_file, 'w' ) as fw:
        fw.write( scd1_sql )

    scd2_file = modeldir/(tgt_table+'_SCD2.sql')
    print('\twriting to', scd2_file)
    with open( scd2_file, 'w' ) as fw:
        fw.write( scd2_sql )

def write_final_sql( args, tgt_table, scd_dict ):
    modeldir = args.modeldir

    scd1_r_cols = []
    scd1_cols = scd_dict[ 'scd1' ]
    #+ scd_dict['scd0']

    exclude_col=[ 'UNIQUE_ID_KEY', 'CURRENT_INDICATOR', 'EFFECTIVE_DATE', 'END_DATE', 'LOAD_DATETIME', 'UPDATE_DATETIME', 'EFFECTIVE_TIMESTAMP', 'END_TIMESTAMP',
    'LOAD_BATCH_ID', 'UPDATE_BATCH_ID', 'PRIMARY_SOURCE_SYSTEM' ]
    scd1_cols = [ col.upper() for col in scd1_cols ]
    exclude_col = [ col.upper() for col in exclude_col ]

    scd1_remaining_cols = set( scd1_cols ) - set( exclude_col )

    scd1_str = ', '.join( scd1_remaining_cols )
    keycol = scd_dict[ 'KEY' ]

    src_table = tgt_table.replace( 'DIM_', 'DSV_' )

    # prefix = ''' {{ config(schema = 'EDW_STAGING', tags=["dim"] ) }}\n ----SRC LAYER----\nWITH\n'''
    prefix = '''\n ----SRC LAYER----\nWITH\n'''

    sql1 = prefix+f'''SCD1 as ( SELECT {scd1_str} , {keycol}    \n\t--, '1901-01-01' as DBT_VALID_FROM, '2099-12-31' as DBT_VALID_TO\n\tfrom      {{{{ ref( '{src_table}') }}}} )'''
    sql2 = f'''SCD2 as ( SELECT *    \n\tFROM      {{{{ ref('{tgt_table}_SNAPSHOT_STEP1') }}}} )'''
    sql3 = f'''FINAL as ( SELECT *
            FROM  SCD2
                INNER JOIN SCD1 USING( {keycol} )  )\nselect * from FINAL'''

    all_sql_str = ',\n'.join([ sql1, sql2, sql3 ])

    final_file = modeldir/(tgt_table+'_SCDALL_STEP2.sql')
    print( '\twriting to', final_file )
    with open( final_file, 'w' ) as fw:
        if not args.yml: fw.write( all_sql_str )
    print('========== done')

###### END OF SCD SPECIFIC ADDED CODE

def get_source_tables( col_rows, alias):
    # This is visited for each table in the Tables tab.
    tables_raw = {}
    tables = {}
    schemas = {}
    cpi = {}
    select_cols = []
    missing_keys = []
    expected_keys = ['Source Schema', 'Source Table', 'Source Column', 'Datatype', 'Automated Logic', 'Manual Logic', 'Order#', 'Staging Layer Column Name',
    'Staging Layer Datatype', 'CPI', 'Unique', 'Not Null', 'Test Expression', 'Accepted Values', 'SCD', 'Remove Column',
    'Equal Rowcount', 'Mutually Exclusive Ranges', 'Unique Combination of Columns', 'Accepted Range']
    RC = 'Remove Column'

    for idx, row in enumerate(col_rows):
        is_derived = False
        # print( f'>> Looping through rows: {row}')
        # print( f'>> Rows have the following keys: {row.keys()}')
        # print( f'<< Compared to:                  {expected_keys}' )
        for each_value in expected_keys:
            if each_value in row.keys(): continue
                # print( f'The key {each_value} is present...')
            else:
                missing_keys.append( each_value )

        if not row.get( 'Source Table' ):
            continue
        indent = nl = ''
        #if idx % 5 == 0:
        #    nl = '\n\t\t\t\t'
        src_table = row[ 'Source Table' ].strip()

        if '(' in src_table:
            src_table = src_table[ :src_table.find('(') ].strip()
        if src_table not in tables:
            tables[ src_table ] = []
            tables_raw[ src_table ] = []
            cpi[ src_table ] = []

        tgt_col = row[ 'Staging Layer Column Name' ].strip()

        # if there is a value in the REMOVE COLUMN in XLS, do not include it in the list of source columns
        if RC in row.keys():
            if not row[ 'Remove Column' ]:
                # print( f'< ADDING {tgt_col.upper().strip()} to select_cols: {select_cols}' )            # DEBUG
                defaultval = row.get('Set Default')
                if not defaultval == None:
                    modtgt_col = f"coalesce( {tgt_col}, MD5( '-1111' )) as {tgt_col}"
                    select_cols.append( modtgt_col.strip() )
                else:
                    select_cols.append( tgt_col.upper().strip() )

        src_test = row[ 'Source Column' ]
        # if there is no Source Column, assume (DERIVED)
        if not src_test:
            src_col = '(DERIVED)'
            is_derived = True
        else:
            src_col = row[ 'Source Column' ].strip()
            if '(DERIVED)' in src_col:
                is_derived = True
        # print( f'+++ New src_col: {src_col}')                         # DEBUG

        # if a Source Column is listed as:  (DERIVED) columnname
        if is_derived:
            derivedname = src_col.upper().replace('(DERIVED)','').strip()
            if derivedname == '':
                derivedname = tgt_col
        else:
            derivedname=''

        #If DERIVED, the source and target are set to the same value
        #unless there is Manual Logic driving the value
        if is_derived and row[ 'Manual Logic' ]:
            src_col = tgt_col
        elif is_derived and not row[ 'Manual Logic' ]:
            if not derivedname == '':
                src_col = derivedname
            else:
                src_col = tgt_col

        #if derived and has manual logic, use that, unless it's a HASH
        elif is_derived and 'HASH' not in row[ 'Manual Logic' ]:
            src_col = row[ 'Manual Logic' ].strip()
        elif src_col != tgt_col:
            src_col = src_col
        else:
            src_col = tgt_col

        final_col = f'{src_col:<50} as {tgt_col:>50}'

        tables[ src_table ].append( nl + final_col )
        tables_raw[ src_table ].append( final_col )
        for tablekey, tableval in alias.items():
            if tablekey == src_table:
                act_table = tableval
                schemas[act_table] = row[ 'Source Schema' ]
                continue

        if row[ 'CPI' ] == 'CPI':
            cpi[ src_table ].append( f'SHA256({src_col}) as {src_col}' )
        else:
            cpi[ src_table ].append( src_col )

    missing_keys = list(set( missing_keys ))
    print( f'~~~~ XLS file is Missing Key Columns: {missing_keys}\n')
    # print( f'\n\n<<<<< DEBUGGING get_source_tables:\nSCH: {schemas.items()}\n TR: {tables_raw.items()}\n  T: {tables.items()}\nCPI: {cpi.items()}\nCOL: {select_cols}' )      # DEBUG

    return schemas, tables_raw, tables, cpi, select_cols

##################################################
# Building out the Extract SQL statement
##################################################
def build_extract_sql( schemas, col_rows, alias ):
    '''
    COLUMN_ROWS: [{
    'Source Schema': 'STAGING',
    'Source Table': 'INV',
    'Source Column': '[DERIVED]',
    'Datatype': None,
    'Automated Logic': None,
    'Manual Logic': 'Hash Key: INVOICE_NUMBER',
    'Order#': 1,
    'Staging Layer Column Name': 'UNIQUE_ID_KEY',
    'Staging Layer Datatype': 'CHAR(32)',
    'CPI': None,
    'Unique': None,
    'Not Null': 'not_null',
    'Test Expression': None,
    'Accepted Values': None,
    'SCD': KEY, 1 or 2
    None: None}]
    '''

    all_sql = []
    col_rows_final = {}

    for table, cols in col_rows.items():
        if table not in col_rows_final:
            col_rows_final[ table ] = []
        for col in cols:
            if col not in col_rows_final[ table ]:
                col_rows_final[ table ].append( col )

    comb_table = {}
    for table, cols in col_rows_final.items():
        for k, v in alias.items():
            if k == table:
                if v not in comb_table:
                    comb_table[ v ] = cols
                else:
                    for each in cols:
                        if each in comb_table[ v ]: continue
                        comb_table[ v ].append( each )
            elif v == table:
                if v not in comb_table:
                    comb_table[ v ] = cols
                else:
                    for each in cols:
                        if each in comb_table[ v ]: continue
                        comb_table[ v ].append( each )

    # print( f'=======\nCOMB_TABLE: {comb_table}')          # DEBUG
    # print( f'COL_ROWS_FINAL: {col_rows_final}')           # DEBUG

    for table, cols in comb_table.items():
        cols_str =  ', '.join( cols )

        sql = f'''SELECT {cols_str} \n\t\t\t\tFROM {schemas[ table ]}.{table};
            python /usr/local/etl/run/run_sql2csv.py  --fname /cifs/mswg9/groups/IT/ETL/snowflake/extracts/{table}.csv --operation vsql --t_schema BASE  --t_environment uat2 --tgtdb vertica --sql "SELECT {cols_str} from  {schemas[ table ]}.{table}"

        '''
        all_sql.append(sql)

    # print( f'\n\n<<<< BUILD EXTRACT\n{cols_str}' )                # DEBUG
    return all_sql

def build_ddl_sql( rows, alias ):
    '''
    COLUMN_ROWS: [{
    'Source Schema': 'STAGING',
    'Source Table': 'INV',
    'Source Column': '[DERIVED]',
    'Datatype': None,
    'Automated Logic': None,
    'Manual Logic': 'Hash Key: INVOICE_NUMBER',
    'Order#': 1,
    'Staging Layer Column Name': 'UNIQUE_ID_KEY',
    'Staging Layer Datatype': 'CHAR(32)',
    'CPI': None,
    'Unique': None,
    'Not Null': 'not_null',
    'Test Expression': None,
    'Accepted Values': None,
    'SCD TYPE': 1,
    None: None}
    ]

    create or replace table POLICY_OWNERS_ADDR (
        POLICY_ID   number(38,0),
        OWNER_ADDR  text,
        ZIP         number(38,0),
        VOID_IND    text
    );
    '''

    tables = {}
    insert_cols = {}
    insert_vals = {}
    unique_cols = set()
    tbl = ''

    for idx, row in enumerate(rows):
        if not row.get('Source Table'):
            continue
        try:
            col = row.get('Source Column',' ')
            col = col.replace('(DERIVED)','').strip()
            stg_tgt_dtype = row['Staging Layer Datatype']
        except:
            print(f'Error proessing row {idx+1}: {row}')

        # print( f'\n====== ALIAS: {alias.items()}')              # DEBUG
        for tbl_alias, tbl_name in alias.items():
            tbl_alias = tbl_alias.strip()
            if tbl_alias == row.get('Source Table',' ').strip():
                act_table = tbl_name
                tbl = act_table
                continue

        tbl_col = tbl, col

        if tbl_col in unique_cols:
            continue
        unique_cols.add(tbl_col)

        if tbl not in tables:
            tables[tbl] = []
        if tbl not in insert_cols:
            insert_cols[tbl] = []
        cleandt = str(stg_tgt_dtype).replace('\xa0', '').strip()

        cleanstr = f'{str(col):<50} {str(cleandt):>50}'
        tables[tbl].append(f'{cleanstr}')

        # print( f'>>>> {tables[ tbl ]}')                    # DEBUG

        insert_cols[tbl].append(col)
        val = 'x'
        if stg_tgt_dtype == None:
            val = "'textval'"
        elif 'INT' in stg_tgt_dtype.upper() or 'NUM' in stg_tgt_dtype.upper():
            val = str(idx)
        elif 'DATE' in stg_tgt_dtype.upper():
            val = "'2222-02-22'"
        else:
            val = "'textval'"

        if tbl not in insert_vals:
            insert_vals[tbl] = []
        insert_vals[tbl].append(val)

    # print( f'\n\n >>>> TABLE DEFINITION\n{tables[ tbl ]}')                    # DEBUG

    ddl_sql = []
    for tbl, cols in tables.items():
        col_str = ',\n\t'.join(cols)

        sql = f"'create or replace TABLE {tbl} (\n\t{col_str}\n\t);'"
        ddl_sql.append(sql)

    # insert into mytable (col1, col3) select to_date('2222-02-22'), to_timestamp('2222-02-22T23:59:59');

    inserts_sql = ['\n']
    for tbl, cols in insert_cols.items():
        val_str = ', '.join(insert_vals[tbl])
        col_str = ', '.join(cols)                 # // debug

        sql = f"'-- insert into {tbl} ( {col_str} )  values ( {val_str} ); '"
        inserts_sql.append(sql)

    all_sql = ddl_sql + inserts_sql

    return all_sql

def get_config( alter_sql, tgt_table, dml_block ):
    '''
    Builds the CONFIG dynamically based on the file name
        # {{ config(   only if...:
        # ADD: materialized = 'view' if the tgt_table starts with DSV
        # ADD: tags = [ "fact" ]  if the tgt_table starts with FACT%
        # Only add post_hook = (" if there are alter statements.
        # ") <- close post_hook
        # }} <- close config
    '''
    concat_config = ''
    config = {}
    compileif = ['view','tags','alter']

    if tgt_table.startswith('DSV'):
        config['view'] = f"materialized = 'view'"
    elif tgt_table.startswith( 'FACT' ):
        config['tags'] = f'tags = [ "fact" ]'

    # Dimension, Fact and Factless Fact tables will get post hook (Alter statements for Referential Integrity)
    if tgt_table.startswith('DIM') or tgt_table.startswith('FACT') or tgt_table.startswith('FLF') :
        if len(alter_sql) > 1: config['alter'] = f'post_hook = ("\n{alter_sql.strip()} {dml_block}\n") '

    compile_cnt = 0
    for checkval in compileif:
        if checkval in config and compile_cnt == 0:
            concat_config = f'{{{{ config( \n{config.get(checkval)} '
            compile_cnt += 1
        elif checkval in config:
            concat_config += f'\n,\t {config.get(checkval)}'
            compile_cnt += 1
    if compile_cnt > 0:
        concat_config  += '\n) }}'

    return concat_config

def build_source_sql(args, tables, alias):
    all_sql = []
    act_table = ''
    # print( '--- BUILDING SOURCE SQL' )
    for table, cols in tables.items():
        cols_str = ', '.join(cols)
        for k, v in alias.items():
            if k == table:
                act_table = v
                continue
        # TODO:  remove entirely if all tables will be staged via DBT.  They will then all have model references.
        # if not alias['Schema'].upper() == 'STAGING' :
            # sql = f"SRC_{table} as ( SELECT *     from     {{{{ source('CORESUITE', {act_table!r}) }}}} ),"
            # sql = f"SRC_{table} as ( SELECT *     from     {{{{ ref( {act_table!r} ) }}}} ),"
        # else:
        sql = f"SRC_{table:<14} as ( SELECT *     FROM     {{{{ ref( {act_table!r} ) }}}} ),"
        all_sql.append(sql)

    for table, cols in tables.items():
        cols_str = ', '.join(cols)
        for k, v in alias.items():
            if k == table:
                act_table = v
                continue

        sql = f"//SRC_{table:<14} as ( SELECT *     FROM     {act_table}) ,"
        all_sql.append(sql)

    if not args.silent:
        print(f'\n=== Build SOURCE SQL\n{all_sql}')
    return all_sql

def build_logic_sql( args, col_rows, incl_trim):
    tables = {}

    for idx, row in enumerate(col_rows):
        if idx == 0: firstcol = True
        if not row.get( 'Source Table' ):
            continue
        tbl = row[ 'Source Table' ]
        src_col = row[ 'Source Column' ]
        col = src_col
        tgt_col = row[ 'Staging Layer Column Name']
        mlogic = row[ 'Manual Logic' ]
        set_default = row.get('Default')

        hasname = ''

        # If the column is derived, check for a column name and assign to HASNAME variable.
        is_derived = False
        if '(DERIVED)' in col.upper():
            is_derived = True
            hasname = col.upper().replace('(DERIVED)','').strip()

        # If there is not Manual Logic, checkf or Automated logic and use that in its place.  Manual Trumps automatic.
        if not row[ 'Manual Logic' ]:
            # Get Automated logic as a substitute
            if row[ 'Automated Logic']:
                mlogic = row[ 'Automated Logic' ]
                mlogic = mlogic.upper()
            else:
                mlogic = ''
        else:
            mlogic = row[ 'Manual Logic']

        # If the Logic is a HASH/COMPOSITE for derived columns, create the field as SURROGATE_KEY
        dtype = row[ 'Datatype' ]
        if is_derived:
            # col = '--- NEW ' + row[ 'Staging Layer Column Name' ]
            if dtype == None:
                mlogic = mlogic.upper()
                if 'HASH' in mlogic or 'COMPOSITE' in mlogic:
                    hashcol = ''
                    hashcol = mlogic.replace('HASH:', '').replace('COMPOSITE:', '').replace('\n','').replace(' ','').strip()
                    temp1 = hashcol.split(',')
                    joincol = (', ').join(temp1)
                    hashcol = ("','").join(temp1)
                    col1 = f"{{ dbt_utils.generate_surrogate_key ( [ '{hashcol}' ] ) }}"
                    surrogate = f"{{{col1}}}"
                elif mlogic:
                    # If not HASH or COMPOSITE, but has logic, just use Manual Logic as is
                    col = mlogic.strip()
                else:
                    col = src_col
                    # col = 'NULL'
            # If the field is DERIVED and a DATE field, then it will get a special case statement built for the logic.
            # This requires a column-name after (DERIVED)
            elif dtype == 'DATE':
                col = f"CASE WHEN {hasname} is null then '-1' \n\t\t\tWHEN {hasname} < '1901-01-01' then '-2' \n\t\t\tWHEN {hasname} > '2099-12-31' then '-3' \n\t\t\tELSE regexp_replace( {hasname}, '[^0-9]+', '') \n\t\t\t\tEND :: INTEGER {' '*40}"

        # Check to see if there is a 'Set Default' for NULL values and wrap col in it.  If it's HASH or COMPOSITE, test for any null and use the set default MD5.
        # If there is no column name, default to just MD5 default.
        # If there is a default/use replace nulls with the MD5 using coalesce
        if set_default and not ( col == 'NULL' or col == ''):
            if is_derived:
                if 'HASH' in mlogic or 'COMPOSITE' in mlogic:
                    if ',' in joincol:
                        col = f"CASE WHEN  nullif(array_to_string(array_construct_compact( {joincol} ),''),'') is NULL \n\t\t\tthen MD5( '{set_default}' ) ELSE {surrogate} \n\t\t\t\tEND {' '*40}"
                    else:
                        col = f"CASE WHEN {joincol} is NULL \n\t\t\tthen MD5( '{set_default}' ) ELSE {surrogate} \n\t\t\t\tEND {' '*40}"
                elif hasname  == '':
                    col = f"MD5( '{set_default}' )"
                else:
                    col = hasname
                    col = f"COALESCE( {hasname},  MD5( '{set_default}' ) ) "
            else:
                col = f"COALESCE( {col}, MD5( '{set_default}' ) ) "
        else:
            if is_derived:
                if 'HASH' in mlogic or 'COMPOSITE' in mlogic:
                    col = f" {surrogate} "

        # tgt_col = str(row['Staging Layer Column Name']).strip()

        #Only do these for Automated Logic
        logic = row[ 'Automated Logic' ]
        if logic:
            logic = logic.strip()

        # INVOICE_HEADER > HDR from alias lookup
        # REF (R1) -> R1
        if '(' in tbl:
            tbl = tbl[ :tbl.find('(') ]

        src_tbl = tbl
        tgt_tbl = f'LOGIC_{tbl}'

        if src_tbl not in tables:
            firstcol = True
            tables[ src_tbl ] = []

        indent = False
        if dtype and ( 'VARCHAR' in dtype.upper() or 'TEXT' in dtype.upper() ):
            indent = True
            if incl_trim: col = f"NULLIF( TRIM( {col} ),'')"
            else: f'{col}'

        if logic:
            indent = True
            if logic.lower() == 'cast date':
                col = f'cast( {col} as DATE )'
            elif logic.lower() == 'cast text':
                col = f'cast( {col} as TEXT )'
            elif logic.lower() == 'upper_trim' or logic.lower() == 'upper,trim' or logic.lower() == 'trim':         # Newly added
                col = f'upper( trim( {col} ))'
            elif logic.lower() == 'upper':
                col = f'upper( {col} )'
            else:
                print(f'!! Unsupported Operation !! <{logic}>')

        # Add the AS ... layer here to alias back to the column
        src_col = src_col.upper().replace('(DERIVED)','')
        if firstcol == True or col.startswith('--- NEW'):
            if len(col)>50:
                col = f"\n\t\t  {col}\n{' ':<60} as {src_col:>50}"
            else:
                col = f"\n\t\t  {col:<50} as {src_col:>50}"
                firstcol = False
        else:
            if len(col)>50:
                col = f"\n\t\t, {col:<50}\n{' ':<60} as {src_col:>50}"
            else:
                col = f"\n\t\t, {col:<50} as {src_col:>50}"

        # Only 1st time col is mentioned, it gets appended
        if col not in tables[ src_tbl ]:
            tables[ src_tbl ].append(col)

    final_sql = []

    for tbl, cols in tables.items():
        # if not defined, select everything via asterisk (*)
        if not cols:
            cols_str = ' * '
        else:
            cols_str = ' '.join(cols)
        tgt_tbl = f'LOGIC_{tbl}'            # Removing 'SRC_'

        sql = f'''\n{tgt_tbl} as ( SELECT {cols_str} \n\t\tFROM SRC_{tbl}
            )'''
        final_sql.append( sql )

        cleanprint = ' '.join( final_sql )                           # DEBUG
        if not args.silent:
            print(f'\n==== Build LOGIC SQL\n {cleanprint}')          # DEBUG

    return ',\n'.join( final_sql )

def build_rename_sql( args, tables):
    rename_sql = []

    # print(f'\n\n<<<<< DEBUGGING:  BUILD RENAME\n{tables}')        # DEBUG
    for table, cols in tables.items():
        cols_str = '\n\t\t, '.join(cols)
        sql = f'\nRENAME_{table:<10} as ( SELECT \n\t\t  {cols_str.strip() } \n\t\t\t\tFROM     LOGIC_{table}   )'
        rename_sql.append(sql)

    all_sql = ', '.join(rename_sql)                                # DEBUG
    if not args.silent:
        print(f'\n==== Build RENAME SQL\n{all_sql}')                   # DEBUG
    return all_sql

def build_filter_sql(table_rows):
    filter_sql = []
    for row in table_rows:
        if not row['Filter Conditions']:
            filters = []
        else:
            filters = row['Filter Conditions'].split(';')
        filters_str = ''
        if filters:
            filters_str = f"\n{'WHERE ':>50}"
            filters_str += ' AND '.join(filters)

        alias = row['Alias']
        from_table = alias or row['Source Table']

        if not alias:
            if not row['Source Table']:
                continue
        source_table = f'RENAME_{from_table}'

        sql = f'FILTER_{from_table:<30} as ( SELECT * FROM    {source_table} {filters_str}  ),'
        filter_sql.append(sql)

    # cleanprint = ' '.join( filter_sql )                             # DEBUG
    # print( f'\n==== Build FILTER SQL\n{cleanprint}')                # DEBUG
    return filter_sql

def build_join_sql(tgt, table_rows, select_cols):
    join_sql = []

    #{ 'POLICY_CLAIMS':'P', 'STG_POLICYTEST':'T', 'POLICY_OWNERS':'O', 'POLICY_OWNERS_ADDRESS':'POA' }
    final_alias = ''
    aliases = {}
    for row in table_rows:
        if not row['Source Schema']:
            continue
        alias = row['Alias']
        aliases[row['Source Table']] = alias
        if not row['Join Type']:
            final_alias = alias

    join_lvls = build_join_levels(table_rows)

    all_sql = []

    made_cte = set()
    # print( f'\n<< JOIN LEVELS: {join_lvls}\n')                               # DEBUG
    # All join_lvls should have a value.  If not, uncomment the above and see where the issue is.

    for lvl in sorted(join_lvls):
        # print( f'<< Currently on lvl: {lvl}')                               # DEBUG
        parent_table, join_sql = do_join( made_cte, join_lvls[ lvl ] )
        made_cte.add( parent_table )
        join_sql_str = '\n\t\t'.join(join_sql)

        if 'COALESCE(' in parent_table.upper():
            wascoalesce = parent_table.upper().replace('COALESCE(','').strip()
            sql = f'{wascoalesce} as ( SELECT * \n\t\t\t\tFROM  FILTER_{wascoalesce}\n{join_sql_str} )'
        else:
            sql = f'{parent_table} as ( SELECT * \n\t\t\t\tFROM  FILTER_{parent_table}\n{join_sql_str} )'
        all_sql.append(sql)

    # Used by final select
    if select_cols:
        select = ''
        for col in select_cols:
            if col == select_cols[ 0 ]:
                select += f'\n\t\t  {col}'
            else:
                select += f'\n\t\t, {col}'
        # select = select.strip(',')
    else:
        select = '*'

    final_sql = f'\nSELECT {select} \nFROM {final_alias}'
    if not all_sql:
        sql = f' JOIN_{alias:<10}  as  ( SELECT * \n\t\t\t\tFROM  FILTER_{alias} )'
        all_sql = [sql + '\n' + f' SELECT * FROM  JOIN_{alias}']
    else:
        all_sql[-1] = all_sql[-1] + final_sql

    all_sql_str = ',\n'.join(all_sql)

    # print( f'\n\n=== Build JOIN SQL\n{all_sql}')                   # DEBUG
    return all_sql_str

def do_join(made_cte, table_rows):
    '''
        select *
            from FPO
                LEFT JOIN FPAD USING( POLICY_ID )
                LEFT JOIN FPAD2 on FPO.POLICY_ID = FPAD2.POLICY_IDX;
    '''
    join_sql = []
    parent_join_list = []
    # The sorted_rows group items together by Parent Join Alias with INNER JOINS being outdented
    sorted_rows = []
    sorted_rows = sorted( table_rows, key = itemgetter( 'Parent Join Alias' ))

    # for row in table_rows:
    for row in sorted_rows:
        join_type = row[ 'Join Type' ]
        if not join_type:
            continue
        join_type = join_type.replace('_', ' ').upper()
        source_table = row[ 'Alias' ]
        parent_join = row[ 'Parent Table Join' ]
        child_join = row[ 'Child Table Join' ]
        parent_table = parent_join.split('.')[ 0 ]

        using_join = ''
        parent_join_col = parent_join.split('.')[ -1 ]
        child_join_col = child_join.split('.')[ -1 ]

        if parent_join_col == child_join_col:
            using_join = f' using( {parent_join_col} ) '

        if 'coalesce' in parent_table:
            tempparent = parent_join.upper().replace('COALESCE(','').strip()
            parent_join = f'coalesce( FILTER_{tempparent}'
        else:
            parent_join = f'FILTER_{parent_join}'

        if source_table in made_cte:
            sql = f'\t\t\t\t{join_type} {source_table} ON  {parent_join} = {child_join} '
            if using_join:
                sql = f'\t\t\t\t{join_type} {source_table} {using_join} '
        else:
            if parent_table in parent_join_list and not join_type.lower().startswith('inner'):
                sql = f'\t\t\t\t\t\t{join_type} FILTER_{source_table} ON  {parent_join} =  FILTER_{child_join} '
            else:
                sql = f'\t\t\t\t{join_type} FILTER_{source_table} ON  {parent_join} =  FILTER_{child_join} '
                if using_join:
                    sql = f'\t\t\t\t{join_type} {join_type}  FILTER_{source_table} {using_join} '
                parent_join_list.append( parent_table )

        join_sql.append( sql )

        # print( f'\n\n==== DO JOIN\nPT:{parent_table}\nSQL:{join_sql}')
    return parent_table, join_sql

def build_join_levels(table_rows):
    lvls = {}
    for row in table_rows:
        if not row[ 'Parent Table Join' ]:
            continue
        lvl = row[ 'Parent Join Number' ]
        if not lvl: lvl = 0
        if lvl not in lvls:
            lvls[ lvl ] = []
        # Add Parent Join Alias to determine if it's a sub join
        row[ 'Parent Join Alias' ] = row[ 'Parent Table Join'].split('.')[0]
        lvls[ lvl ].append(row)

        # print( f'\n\n---- Build JOIN LEVELS\n{lvls[ lvl ]} ')                   # DEBUG
    return lvls

def make_yml(tgt, col_rows, table_rows):
    '''
    version: 2

models:
    - name: orders
    tests:
    - dbt_utils.expression_is_true:
            expression: "colA + colB = totalAB"
    - dbt_utils.equality:
            compare_model: ref( 'other_table_name' )
    columns:
    - name: order_id
        tests:
            - unique
            - not_null
            - dbt_utils.expression_is_true:
                expression: "colA + colB = totalAB"
    - name: status
        tests:
            - accepted_values:
                values: [ 'placed', 'shipped', 'completed', 'returned' ]
    - name: REVENUE_CENTER_HKEY
        tests:
            - dbt_utils.relationships_where:
                to: ref('DIM_REVENUE_CENTER')
                field: REVENUE_CENTER_HKEY
                from_condition: REVENUE_CENTER_HKEY <> '40c5dea533476acdd01f7ef0e84de22f'
    '''

    modeltests = []
    for row in table_rows:
        modeltests += get_model_tests(row)

    print( f'<<< Going in with modeltests: {modeltests}' )        # DEBUG

    schema_dict = {
        'version': 2,
        'models': [
            {'name': tgt,
             'tests': modeltests,
             'columns': [],
             }
        ]
    }

    # print( f'@@@ ADDED MODELTEST: {schema_dict}' )                  # DEBUG

    ymlcols = []
    cols_processed = set()
    for row in table_rows:
        col = row['Staging Layer Column Name']

        if not col:
            continue
        if col in cols_processed:
            continue
        cols_processed.add(col)

        tests = get_column_tests(row, tgt)

        if tests:
            ymlrow = {
                'name': col,
                'tests': tests,
            }
            ymlcols.append(ymlrow)

    y = {}
    y = schema_dict['models'][0]
    y['columns'] = ymlcols

    return schema_dict

def get_model_tests(row):
    '''
    https://github.com/fishtown-analytics/dbt-utils
    '''
    test_dict = {}
    test_cols = ['Unique', 'Model Equality', 'Model Equal Rowcount', 'Test Expression',  ]
    # Globally replacing the relationship with relationship_where

    tests = []
    add = {'severity': 'warn'}

    for atest in test_cols:
        if atest not in row:
            continue
        if not row[atest]:
            continue

        if atest == 'Unique':
            if not row.get(atest).lower() == 'unique' :
                test =   str(row[ atest ]).replace('COMPOSITE:','').strip()
                row[atest] = {
                    'dbt_utils.unique_combination_of_columns':
                    {
                        'combination_of_columns': [each.strip() for each in test.split(',')],
                        'severity': 'warn'
                    }
                }
            else: continue

        if atest == 'Model Equality':
            # print(f' $$$$$$$$ FOUND Model Equality ->  {row[ atest ]}')               # DEBUG
            test =   str(row[ atest ]).strip()
            row[atest] = {
                'dbt_utils.equality':
                {
                    'compare_model': f"ref( {row[ 'Staging Layer Column Name' ]} )",
                    'compare_columns': [each.strip() for each in test.split(',')]
                    # 'compare_columns': [ f'ref({row[ atest ]!r})' ]
                }
            }

        if atest == 'Model Equal Rowcount':
            # print(f' $$$$$$$$ FOUND Model Equal Rowcount -=>  {row[ atest ]}')        # DEBUG
            row[atest] = {
                'dbt_utils.equal_rowcount':
                {
                    'compare_model': f'ref({row[ atest ]!r})'
                }
            }

        # Relationship moved to column test...

        if atest == 'Test Expression':
            if row.get('Staging Layer Column Name','x') in row.get('Test Expression','y'):
                # If true, then should be a model test, else column test
                test =   str(row[ atest ]).strip()
                row[atest] = {
                    'dbt_utils.expression_is_true':
                    {
                        'severity': 'warn',
                        'expression': test
                    }
                }
            else: continue
        tests.append(row[atest])

    return tests

def get_column_tests(row, tgt):
    '''
    https://github.com/fishtown-analytics/dbt-utils
    '''
    test_dict = {}
    test_cols = [ 'Unique', 'Not Null', 'Test Expression', 'Accepted Values', 'Relationship', 'Set Default']
    # 'Equal Rowcount',  'Mutually Exclusive Ranges', 'Unique Combination of Columns', 'Accepted Range']

    tests = []
    add = {'severity': 'warn'}

    for atest in test_cols:
        if atest not in row:
            continue
        if not row[atest]:
            continue

        # If a FLF or FACT build, do not include the has_default test.  Will require knowing the layer during build.
        if atest == 'Set Default':
            test = str( row[ atest ] ).strip()
            row['Default']= test
            if tgt.startswith('F'):
                row[atest]=''
            else:
                row[atest] = {
                    'has_default':
                    {
                        'severity': 'warn',
                        'hashval': [each.strip() for each in test.split(',')]
                    }
                }

        if atest == 'Unique':
            row[atest] = {
                'unique':
                {
                    'severity': 'warn'
                }
            }

        if atest == 'Not Null':
            if 'where' in row[atest]:
                # TODO: Fix quoting around where
                test = str( row[ atest ] ).upper().replace( 'WHERE','').strip()
                row[atest] = {
                    'dbt_utils.not_null_where':
                    {
                        'where': test
                    }
                }
            else:
                row[atest] = {
                    'not_null':
                    {
                        'severity': 'warn'
                    }
                }

        if atest == 'Accepted Values':
            if 'not:' in row[atest]:
            # Split to break them into separate rows...
                test = str( row[ atest ] ).upper().replace( 'NOT:','').strip()
                row[atest] = {
                    'not_accepted_values':
                    {
                        'severity': 'warn',
                        'values': [each.strip() for each in test.split(',')]
                    }
                }
            else:
                test = str( row[ atest ] )
                row[atest] = {
                    'accepted_valids':
                    {
                        'ignore': '',
                        'severity': 'warn',
                        'values': [each.strip() for each in test.split(',')]
                    }
                }

        if atest == 'Test Expression':
            # If the column name is not listed, then it can be a column test, else model test
            if not row.get('Staging Layer Column Name','x') in str(row[ atest ]).strip():
                test =   str(row[ atest ]).strip()
                row[atest] = {
                    'expression_is_true_ignore':
                    {
                        'ignore': '',
                        'severity': 'warn',
                        'expression': test
                    }
                }
            else: continue

        if atest == 'Relationship':
            ignore_in = ''
            # Changed to a relationships_where instead to handle ignoring dummy value:  md5('-1111')
            # Determine if the field is different (Based on TableName.ColumnName )
            if '.' in row[atest]:
                ref_table = row[ atest ].split('.')[0]
                ref_field = row[ atest ].split('.')[1]
                ref_ignore = row[ 'Staging Layer Column Name' ]
            else:
                ref_table = row[ atest ]
                ref_field = row[ 'Staging Layer Column Name' ]
                ref_ignore = row[ 'Staging Layer Column Name' ]

            if ref_field == 'DATE_KEY':
                ignore_in = " -1, -2, -3 "
            else:
                ignore_in = " '40c5dea533476acdd01f7ef0e84de22f', 'fcbcdcb8f6b1c597c5fdc7a54cd321ae' "

            row[atest] = {
                'dbt_utils.relationships_where':
                {
                    'to': f'ref( {ref_table!r} )',
                    'field': f"{ref_field}",
                    'from_condition': f"{ref_ignore} not in ( {ignore_in} ) "
                }
            }

        if not row[atest] == '':
            tests.append(row[atest])

    return tests

def build( args, modeldir, tgt_table, table_rows, col_rows, alter_sql, dml_block, incl_trim ):
    yml = make_yml( tgt_table, table_rows, col_rows )
    alias = {}
    newvalue = ''

    for thisrow in table_rows:
        thisrow_dict = {}
        for key, value in thisrow.items():
            if not value: value = ''
            if not key: key = ''
            thisrow_dict[key] = value
        thisrow = thisrow_dict

        for key, value in sorted( thisrow.items() ):
            if not key: continue
            if key not in ['Source Table', 'Alias', 'Source Schema']: continue
            if value not in alias:
                if 'Alias' in key:
                    alias[ value ] = []
                    newvalue = value
            if 'Source Table' in key:
                alias[ newvalue ] = value
            if 'Source Schema' in key:
                alias[ 'Schema' ] = value
            if 'Set Default' in key:
                alias[ 'Set Default' ] = value

    schemas, table_cols, table_cols_formatted, cpi, select_cols = get_source_tables( col_rows, alias)

    # SQL to pull from oracle/db2/vertica
    extract_sql = build_extract_sql( schemas, cpi, alias )
    extract_sql_str = '\n'.join( extract_sql )

    ##################################################
    # IF NOT a STG_ file do not generate the EXTRACT_FROM_SRC file
    ##################################################
    if tgt_table.startswith('STG') and not args.yml:
        with open(modeldir/(tgt_table + '_extract_from_src.sql'), 'w') as fw:
            fw.write( extract_sql_str )

    # SQL to pull from oracle/db2/vertica
    ddl_sql = build_ddl_sql( col_rows, alias )
    ddl_sql_str = '\n'.join( ddl_sql )
    if not args.silent:
        print(f'\n~~~~ DDL :\n{str( ddl_sql_str )}')  # DEBUG

    ##################################################
    # Potentially not used and can be removed.
    ##################################################
    # with open(modeldir/(tgt_table + '_create_in_snowflake_ddl.sql'), 'w') as fw:
    #     fw.write(
    #         f'USE WAREHOUSE WH_ETL;\nUSE ROLE ETL_ADMIN;\nUSE DATABASE DBTEST;\nUSE SCHEMA {args.schema} !!!;\n\n')
    #     fw.write(extract_sql_str)

    ################################
    # SRC LAYER
    config = get_config( alter_sql, tgt_table, dml_block )
    all_sql = build_source_sql( args, table_cols_formatted, alias )
    all_sql[ 0 ] = config + '\n\n---- SRC LAYER ----\nWITH\n' + all_sql[ 0 ]

    # LOGIC LAYER                   $$$ FIX FORMATTING
    if tgt_table.startswith( 'DST' ) or tgt_table.startswith( 'STG' ): incl_trim = True
    all_sql[ -1 ] += '\n\n---- LOGIC LAYER ----\n'
    logic_sql = build_logic_sql( args, col_rows, incl_trim )
    all_sql.append( logic_sql )

    # RENAME LAYER
    all_sql[-1] += '\n\n---- RENAME LAYER ----\n,'
    # print( f'\n\n<<<<< DEBUGGING table_cols_formatted  going into RENAME LAYER\n{ table_cols_formatted }' )
    # {'INVOICE': ['\n\t\t\t\n\t\t\tINVOICE_NUMBER as MEDICAL_INVOICE_NUMBER']}    <- Table_cols_formatted.  Only one column
    # For every wb('Tables')['Alias'], walk through and find matching wb('Columns')['Source Table'].
    # for each match, bring ['Source Column'] as ['Staging Layer Column Name']
    rename_sql = build_rename_sql( args, table_cols_formatted )
    # all_sql += rename_sql
    all_sql.append( rename_sql )

    # FILTER LAYER
    all_sql[-1] += '\n\n---- FILTER LAYER (uses aliases) ----\n,'
    filter_sql = build_filter_sql( table_rows )
    all_sql += filter_sql

    # JOIN LAYER
    all_sql[-1] += '\n\n---- JOIN LAYER ----\n'
    join_sql = build_join_sql( tgt_table, table_rows, select_cols )
    all_sql.append( join_sql )
    # print( f'\n\n<<<<< JOINs LAYER \n{all_sql}')                     # DEBUG

    sql_str = '\n'.join( all_sql )
    ################################
    # Write out the final finals...

    with open(modeldir/(tgt_table + '.yml'), 'w') as fw:
        # print( f'\n\n~~~~> YML:\n{yml}' )                          # DEBUG
        fw.write(yaml.dump(yml))

    ##### Output SQL for all other non-DIM sql (STG, DST, DSV)
    if not tgt_table.startswith( 'DIM' ) and not args.yml:
        with open(modeldir/(tgt_table + '.sql'), 'w') as fw:
            fw.write(sql_str)

    # print(f'\n\n===== CURRENT SQL =\n{sql_str}')                   # DEBUG

def main():
    args = process_args()
    setup(args)
    count = 0
    plural = ''
    scd6_sql = ''

    print('\n-----')
    for xls in args.mapdir.iterdir():
        print(f'\t<< Processing: {xls}')
        if xls.name.startswith('XDIM_'):
            print(f'\t\tNOT STAGE MAPPING -- skipping: {xls.name}')
            continue
        elif 'xls' not in xls.suffix.lower():
            print(f'\t-- Skipping  : {xls}')
            continue

        print(f"in {os.path.basename(xls)}")
        wb = openpyxl.load_workbook(xls)

        print(f'\n\n==== FOUND WORKBOOKS: {wb.sheetnames}')         # DEBUG

        #  CHECK for new Layout:  DSV_TABLES, DST_TABLES, DIM_TABLES,
        #    DST_COLUMNS, DSV_COLUMS, DIM_COLUMNS
        new_sheets = [ 'STG Tables', 'STG Tables', 'DST Tables', 'DSV Tables', 'DIM Tables', 'DST Columns', 'DSV Columns', 'DIM Columns', 'DML']
        missing_sheets =[]
        expected_cols = [ 'Source Schema', 'Source Table', 'Source Column', 'Datatype', 'Automated Logic', 'Manual Logic', 'Order#' ,'Staging Layer Column Name' ,'Staging Layer Datatype', 'CPI' ,'Unique' ,'Not Null' ,'Test Expression' ,'Accepted Values', 'SCD' ]

        # If the workbook contains a DML tab, capture it to append to the CONFIG block

        dml_block, dml_sql = '',''
        if 'DML' in wb.sheetnames:
            dml_tab = wb['DML']
            #replace all newlines in the cell value with a space
            # save the contents of the first column of the dml tab to a string
            try:
                for row in dml_tab.rows:
                    if row[0].value is not None:
                        dml_sql += row[0].value.replace('\n', ' ')
                # dml_sql = '\n'.join([' '.join( row[0].value.replace('\n',' ') ) for row in dml_tab.rows])
                dml_block = f'\n{dml_sql}'
            except Exception as e:
                print(f'\t\tERROR: {e}')
                continue

        # Try to BULK handle the UNIFIED Columns/tables
        if xls.name.startswith('UNIFIED'):

            for each_sheet in new_sheets:
                if each_sheet in wb.sheetnames:
                    continue
                else:
                    missing_sheets.append( each_sheet )

            if len( missing_sheets ) > 0:
                missing_sheets =  list(set( missing_sheets ))
                print( f'\n\n~~~ There are missing sheets for the new format: {missing_sheets}')

            buildorder = ['STG','DST','DSV','DIM']

            for currentorder in buildorder:
                tgt_table = xls.stem.replace('UNIFIED', currentorder )
                curtable = f"{currentorder} Tables"
                curcol =   f"{currentorder} Columns"
                maxrow = wb[curtable].max_row
                print(f'{curtable} contains {maxrow} rows')

                if maxrow <= 2:
                    # Check to see if the First Cell of the table has a value, if not, append to missing_sheets
                    testfield = wb[curtable].cell(2,1).value
                    if testfield == None:
                        missing_sheets.append( curtable )
                        missing_sheets.append( curcol )
                        continue

                # Moved to be outside of DIM table creation script
                alter_sql = ''
                args.currentorder = currentorder

                # print (f"<< Testing for {eachval} Table for >> {tgt_table}'")
                if not curtable in missing_sheets:
                    table_rows = [row for row in sheet2dict(wb[ curtable ])]
                    col_rows = [row for row in sheet2dict(wb[ curcol ])]

                    # Only check on SCD for the DIM tables/columns
                    is_scd=False
                    if currentorder == 'DIM':
                        if not args.silent:
                            print( f'\n\n~~Current tab/cols are: {curtable}; {curcol}')         # DEBUG

                            missing_cols =[]
                            for each_col in expected_cols:
                                if each_col in col_rows[0].keys():
                                    continue
                                else:
                                    missing_cols.append( each_col )
                            if len(missing_cols)>0:
                                missing_cols =  list(set( missing_cols ))
                                print( f'\n\n~~~ There are missing colums for the current table: {missing_cols}')

                        for row in col_rows:
                        #print(row)
                            if row.get( 'SCD' ): is_scd = True; break
                            # print('NOT DIM MAPPING, skipping',xls)
                            break
                        if not is_scd: continue

                    # replaced tgt_table with curtable < If this fails
                        scd_dict = get_scdinfo( args.schema, tgt_table, col_rows)
                        scd2_sql, scd6_sql = make_scd( scd_dict )

                        # Workbook name is Case Sensitive.  Need to correct
                        if 'Alter' in wb:
                            alter_sql = '\n'
                            for sql in sheet2list(wb['Alter']):
                                if not sql[0]:
                                    continue
                                alter_sql += sql[0] + '\n'
                            alter_sql = alter_sql.replace('\\xa0', ' ')

                        # Put config in front of scd6_sql
                        config = get_config( alter_sql, tgt_table, dml_block )
                        scd6_sql = config + scd6_sql

                        # write_dbt: replaced scd_dict['table'] w/ curtable
                        if not args.yml:
                            write_dbt( scd_dict[ 'table' ], args.modeldir, scd2_sql, args.modeldir, scd6_sql )
                            write_final_sql( args, tgt_table, scd_dict )

                # tgt_table = xls.stem
                print(f'Tgt table: {tgt_table} ')  # Should be the XLS filename

                # Start to process the build of each Layer
                if currentorder == 'STG':
                    incl_trim = True
                else:
                    incl_trim = False
                build(args, args.modeldir, tgt_table, table_rows, col_rows, alter_sql, dml_block, incl_trim )

        ##############
        # Standard file processing...
        if not xls.name.startswith('UNIFIED'):
            tgt_table = xls.stem
            print(f'Tgt table: {tgt_table} ')  # Should be the XLS filename
            alter_sql = ''
            config = ''
            # args.currentorder = {tgt_table}.split()

            table_rows = [row for row in sheet2dict(wb['Tables'])]
            col_rows = [row for row in sheet2dict(wb['Columns'])]
            missing_cols =[]

            for each_col in expected_cols:
                if each_col in col_rows[0].keys():
                    continue
                else:
                    missing_cols.append( each_col )
            if len(missing_cols)>0:
                missing_cols =  list(set( missing_cols ))
                print( f'\n\n~~~ There are missing colums for the current table: {missing_cols}')

        # Workbook name is Case Sensitive.  Need to correct
            if 'Alter' in wb:
                alter_sql = '\n'
                for sql in sheet2list(wb['Alter']):
                    if not sql[0]:
                        continue
                    alter_sql += sql[0] + '\n'
                alter_sql = alter_sql.replace('\\xa0', ' ')
                config = get_config( alter_sql, tgt_table, dml_block )

            # Only check on SCD for the DIM tables/columns
            if xls.name.startswith('DIM'):
                is_scd = False
                for row in col_rows:
                    #print(row)
                    if row.get( 'SCD' ): is_scd = True; break
                    # print('NOT DIM MAPPING, skipping',xls)
                    break
                if not is_scd: continue

                # replaced tgt_table with curtable < If this fails
                # Put config in front of scd6_sql
                scd_dict = get_scdinfo( args.schema, tgt_table, col_rows)

                if is_scd:
                    scd2_sql, scd6_sql = make_scd( scd_dict )
                    scd6_sql = config + scd6_sql
                else:
                    scd6_sql = config

                # write_dbt: replaced scd_dict['table'] w/ curtable
                if not args.yml:
                    write_dbt( scd_dict[ 'table' ], args.modeldir, scd2_sql, args.modeldir, scd6_sql )
                    write_final_sql( args, tgt_table, scd_dict)

            # Start to process the build of each Layer
            if  xls.name.startswith('STG'):
                incl_trim = True
            else:
                incl_trim = False
            build(args, args.modeldir, tgt_table, table_rows, col_rows, alter_sql, dml_block, incl_trim)

        count += 1
    if count > 1: plural = 's.'
    else: plural = '.'

    print(f'Processed: {count} file{plural}')
    if count > 0 : print(f'Models written to: {args.modeldir}')

    input('[ Press Enter to exit ] ')

if __name__ == '__main__':
    main()


import pandas as pd
def a_df(a_dict):
    result =  pd.DataFrame(a_dict)
    return result

def assert_df(df1, df2):
    df1_dicts = df1.to_dict("records")
    df2_dicts = df2.to_dict("records")
    for item in df1_dicts:
        if item not in df2_dicts:
            assert False, f"{item} not in df2"
    for item in df2_dicts:
        if item not in df1_dicts:
            assert False, f"{item} not in df1"


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import nltk
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
import pickle
import numpy as np

from keras.models import load_model
model = load_model('chatbot_model.h5')
import json
import random
from googleplaces import GooglePlaces, types, lang
import requests

intents = json.loads(open('intents.json', encoding='utf-8').read())
words = pickle.load(open('words.pkl','rb'))
classes = pickle.load(open('classes.pkl','rb'))

def clean_up_sentence(sentence):
    # tokenize the pattern - split words into array
    sentence_words = nltk.word_tokenize(sentence)
    sentence_words = [lemmatizer.lemmatize(word.lower()) for word in sentence_words]
    return sentence_words

# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence

def bow(sentence, words, show_details=True):
    # tokenize the pattern
    sentence_words = clean_up_sentence(sentence)
    # bag of words - matrix of N words, vocabulary matrix
    bag = [0]*len(words)
    for s in sentence_words:
        for i,w in enumerate(words):
            if w == s:
                # assign 1 if current word is in the vocabulary position
                bag[i] = 1
                if show_details:
                    print ("found in bag: %s" % w)
    return(np.array(bag))

def predict_class(sentence, model):
    # filter out predictions below a threshold
    p = bow(sentence, words,show_details=False)
    res = model.predict(np.array([p]))[0]
    ERROR_THRESHOLD = 0.25
    results = [[i,r] for i,r in enumerate(res) if r>ERROR_THRESHOLD]
    # sort by strength of probability
    results.sort(key=lambda x: x[1], reverse=True)
    return_list = []
    for r in results:
        return_list.append({"intent": classes[r[0]], "probability": str(r[1])})
    return return_list

def getResponse(ints, intents_json):
    tag = ints[0]['intent']
    list_of_intents = intents_json['intents']
    for i in list_of_intents:
        if(i['tag']== tag):
            result = random.choice(i['responses'])
            break
    return result

def chatbot_response(msg):
    ints = predict_class(msg, model)
    res = getResponse(ints, intents)
    return res
#Creating GUI with tkinter
import tkinter
from tkinter import *
from tkinter import ttk

def Check_Symptom():
    def send():
        msg = EntryBox.get("1.0",'end-1c').strip()
        EntryBox.delete("0.0",END)

        if msg != '':
            ChatLog.config(state=NORMAL)
            ChatLog.insert(END, "You: " + msg + '\n\n')
            ChatLog.config(foreground="#442265", font=("Verdana", 12 ))

            res = chatbot_response(msg)
            ChatLog.insert(END, "DoctorBot: " + res + '\n\n')

            ChatLog.config(state=DISABLED)
            ChatLog.yview(END)
    windows = Tk()
    windows.title("Symptom Checker")
    windows.geometry("400x500")
    windows.resizable(width=FALSE, height=FALSE)

#Create Chat window
    ChatLog = Text(windows, bd=0, bg="white", height="8", width="50", font="Arial",)

    ChatLog.config(state=DISABLED)

#Bind scrollbar to Chat window
    scrollbar = Scrollbar(windows, command=ChatLog.yview, cursor="heart")
    ChatLog['yscrollcommand'] = scrollbar.set

#Create Button to send message
    SendButton = Button(windows, font=("Verdana",12,'bold'), text="Send", width="12", height=5,bd=0, bg="#32de97", activebackground="#3c9d9b", fg='#ffffff', command= send )

#Create the box to enter message
    EntryBox = Text(windows, bd=0, bg="white",width="29", height="5", font="Arial")
#EntryBox.bind("<Return>", send)

#Place all components on the screen
    scrollbar.place(x=376,y=6, height=386)
    ChatLog.place(x=6,y=6, height=386, width=370)
    EntryBox.place(x=128, y=401, height=90, width=265)
    SendButton.place(x=6, y=401, height=90)
    windows.mainloop()

def display_hospitals():
    root = Tk()
    root.geometry("400x500")
    root.resizable(width=FALSE, height=FALSE)
    root.title("Near-by Hospitals")

    API_KEY = "API_KEY"
    google_places = GooglePlaces(API_KEY)

    query_result = google_places.nearby_search(lat_lng ={'lat': 51.8787, 'lng': -0.4200},radius = 5000,types =[types.TYPE_HOSPITAL])
#for place in query_result.places:
    #place.get_details()
    #show_hospital = (place.name + '\n', place.international_phone_number, '\n', place.website, '\n', place.url )
# Insert elements into the listbox
# Creating a Listbox and
# attaching it to root window
#listbox = show_hospital
    listbox = Listbox(root)

# Adding Listbox to the left
# side of root window
#listbox.pack(side = LEFT, fill = BOTH)
    listbox.place(x=6,y=6, height=466, width=370)

# Creating a Scrollbar and
# attaching it to root window

#scrollbar = Scrollbar(root)

# Adding Scrollbar to the right
# side of root window

#scrollbar.pack(side = RIGHT, fill = BOTH)

    scrollbar =ttk.Scrollbar(root, orient='vertical', command=listbox.yview)
    listbox['yscrollcommand'] = scrollbar.set
    scrollbar.place(x=376,y=6, height=466)

    for place in query_result.places:
        place.get_details()
        listbox.insert(END, '\n', place.name, '\n', place.international_phone_number, '\n', place.website, '\n', place.url )

# Attaching Listbox to Scrollbar
# Since we need to have a vertical
# scroll we use yscrollcommand
    listbox.config(yscrollcommand = scrollbar.set)

# setting scrollbar command parameter
# to listbox.yview method its yview because
# we need to have a vertical view
    scrollbar.config(command = listbox.yview)

    root.mainloop()

base = Tk()
base.title("DoctorBot")
Welcome_user=Label(base, font=("Verdana",10), text="Welcome user, how may I assist you today?", bg="#000000", activebackground="#3c9d9b", fg='#ffffff')
Welcome_user.grid()
Symptom_Checker= Button(base, font=("Verdana",12,'bold'), text="Check Symptoms", bg="#32de97", activebackground="#3c9d9b", fg='#ffffff', command= Check_Symptom)
Symptom_Checker.grid()
nearest_hospital= Button(base, font=("Verdana",12,'bold'), text="Find nearest hospitals", bg="#4242ff", activebackground="#3c9d9b", fg='#ffffff', command=display_hospitals)
nearest_hospital.grid()
base.mainloop()


# 貼り付け先　テスト
#
harituke_saki = 'C:/ACES WEB/SIRIUS/サイト生成先/肉のサイト/beef_ranking/index.php'
f = open(harituke_saki, 'r', encoding="utf-8-sig")
hs_data = f.read()  # ファイルごと読み込む
f.close()

harituke_f = 'C:/xampp/htdocs/niku/貼り付け用.php'
f = open(harituke_f, 'r', encoding="utf-8-sig")
h_data = f.read()  # ファイルごと読み込む
f.close()

rank_n = 'C:/xampp/htdocs/niku/元キーワード他肉ランキング.txt'
f = open(rank_n, 'r', encoding="utf-8-sig")
n_data = f.readlines()  # 一行づつ読み込む
f.close()

rank_b = 'C:/xampp/htdocs/niku/元キーワード部位ランキング.txt'
f = open(rank_b, 'r', encoding="utf-8-sig")
b_data = f.readlines()  # 一行づつ読み込む
f.close()

rank_b = 'C:/xampp/htdocs/niku/ブランド牛のランキング.csv'
f = open(rank_b, 'r', encoding="utf-8-sig")
r_data = f.readlines()  # 一行づつ読み込む
f.close()

out_file = 'C:/xampp/htdocs/niku/niku-site/beef_ranking/index.php'

# ランキング順に並べなおして配列にいれる
nagasa = len(n_data)
rank_n_data = []
for i in range(nagasa):
    kazu = i + 1

    # print(kazu)
    for l_n_data in n_data:
        bun = l_n_data.split(',')
        if kazu == int(bun[3]):
            # print (l_n_data)
            rank_n_data.append(l_n_data)

# rank_n_data にランキング順にデータを移動
# print (rank_n_data)

# 最終的に張り付ける変数
honbun = ''
a = 0
for l_rank_n_data in rank_n_data:
    bunkatu = l_rank_n_data.split(',')

    if '終わり' in l_rank_n_data:
        break
    # 見出し
    a = a + 1
    zyuni = str(a) + '位'

    honbun = honbun + '
' + zyuni + ' ' + bunkatu[0] + '
'
    harituke_f_cp = harituke_f
    # 20210722_ranking_豚肉.txt

    okikaefike = '../inf/20*' + bunkatu[0] + '.txt'
    h_data_cp = h_data
    h_data_cp = h_data_cp.replace('〇ファイル', okikaefike)
    #print((okikaefike))
    honbun = honbun + h_data_cp

cpmoto_data2 = hs_data.replace('〇その他の肉ランキング', honbun)

f = open(out_file, 'w', encoding="utf-8")
f.writelines(cpmoto_data2)
f.close()

################################################################
#
# ブランド牛ランキング
honbun = ''
a = 0
for l_r_data in r_data:  # ブランド牛のランキング順ファイル
    # 三重,松阪牛,mie,matusakagyu,6834
    bunkatu = l_r_data.split(',')

    # 見出し
    a = a + 1
    zyuni = str(a) + '位'

    honbun = honbun + '
' + zyuni + ' ' + bunkatu[1] + '
'
    harituke_f_cp = harituke_f
    #

    okikaefike = '../inf/20*' + bunkatu[1] + '.txt'
    h_data_cp = h_data
    h_data_cp = h_data_cp.replace('〇ファイル', okikaefike)
    # print((okikaefike))
    honbun = honbun + h_data_cp

    if a == 50:
        break
### hs_data を使うと、新データに新データを上書きしているだけ。カスケード処理のように、次々にファイルを上書きする必要がある。
cpmoto_data2 = cpmoto_data2.replace('〇ブランド牛別ランキング', honbun)

#f = open(out_file, 'w', encoding="utf-8")
#f.writelines(cpmoto_data2)
#f.close()

######################################################
#部位別
#
#
# ランキング順に並べなおして配列にいれる
nagasa = len(b_data)
rank_b_data = []
for i in range(nagasa):
    kazu = i + 1

    # print(kazu)
    for l_b_data in b_data:
        bun = l_b_data.split(',')
        if kazu == int(bun[3]):
            # print (l_n_data)
            rank_b_data.append(l_b_data)

# 最終的に張り付ける変数
honbun = ''
a = 0
for l_rank_b_data in rank_b_data:
    bunkatu = l_rank_b_data.split(',')

    if '終わり' in l_rank_b_data:
        break
    # 見出し
    a = a + 1
    zyuni = str(a) + '位'

    honbun = honbun + '
' + zyuni + ' ' + bunkatu[0] + '
'
    harituke_f_cp = harituke_f
    # 20210722_ranking_豚肉.txt

    okikaefike = '../inf/20*' + bunkatu[0] + '.txt'
    h_data_cp = h_data
    h_data_cp = h_data_cp.replace('〇ファイル', okikaefike)
    #print((okikaefike))
    honbun = honbun + h_data_cp

cpmoto_data2 = cpmoto_data2.replace('〇牛肉部位別ランキング', honbun)

f = open(out_file, 'w', encoding="utf-8")
f.writelines(cpmoto_data2)
f.close()

html作成_ランキングページ.py

html作成_ランキングページ.py