diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..23e927f618477bf5819e47366f40bc4ad6a47a59 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,26 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +docs/example_crop.gif filter=lfs diff=lfs merge=lfs -text +docs/example_crop_still.gif filter=lfs diff=lfs merge=lfs -text +docs/example_full.gif filter=lfs diff=lfs merge=lfs -text +docs/example_full_enhanced.gif filter=lfs diff=lfs merge=lfs -text +docs/free_view_result.gif filter=lfs diff=lfs merge=lfs -text +docs/resize_good.gif filter=lfs diff=lfs merge=lfs -text +docs/resize_no.gif filter=lfs diff=lfs merge=lfs -text +docs/using_ref_video.gif filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/chinese_news.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/deyu.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/eluosi.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/fayu.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/imagine.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/japanese.wav filter=lfs diff=lfs merge=lfs -text +examples/ref_video/WDA_AlexandriaOcasioCortez_000.mp4 filter=lfs diff=lfs merge=lfs -text +examples/ref_video/WDA_KatieHill_000.mp4 filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_16.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_17.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_3.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_4.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_5.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_8.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_9.png filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..73c66b605152ba0e143900f8140e24655358d940 --- /dev/null +++ b/.gitignore @@ -0,0 +1,174 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +examples/results/* +gfpgan/* +checkpoints/* +assets/* +results/* +Dockerfile +start_docker.sh +start.sh + +checkpoints + +# Mac +.DS_Store diff --git a/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/.ipynb_checkpoints/Untitled-checkpoint.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..363fcab7ed6e9634e198cf5555ceb88932c9a245 --- /dev/null +++ b/.ipynb_checkpoints/Untitled-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/.ipynb_checkpoints/app-checkpoint.py b/.ipynb_checkpoints/app-checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..926031c37dc21f6d341da9f050e35f39c87eddf0 --- /dev/null +++ b/.ipynb_checkpoints/app-checkpoint.py @@ -0,0 +1,608 @@ +import os, sys +import tempfile +import gradio as gr +from src.gradio_demo import SadTalker +# from src.utils.text2speech import TTSTalker +from huggingface_hub import snapshot_download + +import torch +import librosa +from scipy.io.wavfile import write +from transformers import WavLMModel + +import utils +from models import SynthesizerTrn +from mel_processing import mel_spectrogram_torch +from speaker_encoder.voice_encoder import SpeakerEncoder + +import time +from textwrap import dedent + +import mdtex2html +from loguru import logger +from transformers import AutoModel, AutoTokenizer + +from tts_voice import tts_order_voice +import edge_tts +import tempfile +import anyio + + +def get_source_image(image): + return image + +try: + import webui # in webui + in_webui = True +except: + in_webui = False + + +def toggle_audio_file(choice): + if choice == False: + return gr.update(visible=True), gr.update(visible=False) + else: + return gr.update(visible=False), gr.update(visible=True) + +def ref_video_fn(path_of_ref_video): + if path_of_ref_video is not None: + return gr.update(value=True) + else: + return gr.update(value=False) + +def download_model(): + REPO_ID = 'vinthony/SadTalker-V002rc' + snapshot_download(repo_id=REPO_ID, local_dir='./checkpoints', local_dir_use_symlinks=True) + +def sadtalker_demo(): + + download_model() + + sad_talker = SadTalker(lazy_load=True) + # tts_talker = TTSTalker() + +download_model() +sad_talker = SadTalker(lazy_load=True) + + +# ChatGLM2 & FreeVC + +''' +def get_wavlm(): + os.system('gdown https://drive.google.com/uc?id=12-cB34qCTvByWT-QtOcZaqwwO21FLSqU') + shutil.move('WavLM-Large.pt', 'wavlm') +''' + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +smodel = SpeakerEncoder('speaker_encoder/ckpt/pretrained_bak_5805000.pt') + +print("Loading FreeVC(24k)...") +hps = utils.get_hparams_from_file("configs/freevc-24.json") +freevc_24 = SynthesizerTrn( + hps.data.filter_length // 2 + 1, + hps.train.segment_size // hps.data.hop_length, + **hps.model).to(device) +_ = freevc_24.eval() +_ = utils.load_checkpoint("checkpoint/freevc-24.pth", freevc_24, None) + +print("Loading WavLM for content...") +cmodel = WavLMModel.from_pretrained("microsoft/wavlm-large").to(device) + +def convert(model, src, tgt): + with torch.no_grad(): + # tgt + wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate) + wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20) + if model == "FreeVC" or model == "FreeVC (24kHz)": + g_tgt = smodel.embed_utterance(wav_tgt) + g_tgt = torch.from_numpy(g_tgt).unsqueeze(0).to(device) + else: + wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).to(device) + mel_tgt = mel_spectrogram_torch( + wav_tgt, + hps.data.filter_length, + hps.data.n_mel_channels, + hps.data.sampling_rate, + hps.data.hop_length, + hps.data.win_length, + hps.data.mel_fmin, + hps.data.mel_fmax + ) + # src + wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate) + wav_src = torch.from_numpy(wav_src).unsqueeze(0).to(device) + c = cmodel(wav_src).last_hidden_state.transpose(1, 2).to(device) + # infer + if model == "FreeVC": + audio = freevc.infer(c, g=g_tgt) + elif model == "FreeVC-s": + audio = freevc_s.infer(c, mel=mel_tgt) + else: + audio = freevc_24.infer(c, g=g_tgt) + audio = audio[0][0].data.cpu().float().numpy() + if model == "FreeVC" or model == "FreeVC-s": + write("out.wav", hps.data.sampling_rate, audio) + else: + write("out.wav", 24000, audio) + out = "out.wav" + return out + +# GLM2 + +language_dict = tts_order_voice + +# fix timezone in Linux +os.environ["TZ"] = "Asia/Shanghai" +try: + time.tzset() # type: ignore # pylint: disable=no-member +except Exception: + # Windows + logger.warning("Windows, cant run time.tzset()") + +# model_name = "THUDM/chatglm2-6b" +model_name = "THUDM/chatglm2-6b-int4" + +RETRY_FLAG = False + +tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + +# model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda() + +# 4/8 bit +# model = AutoModel.from_pretrained("THUDM/chatglm2-6b", trust_remote_code=True).quantize(4).cuda() + +has_cuda = torch.cuda.is_available() + +# has_cuda = False # force cpu + +if has_cuda: + model_glm = ( + AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half() + ) # 3.92G +else: + model_glm = AutoModel.from_pretrained( + model_name, trust_remote_code=True + ).float() # .float() .half().float() + +model_glm = model_glm.eval() + +_ = """Override Chatbot.postprocess""" + + +def postprocess(self, y): + if y is None: + return [] + for i, (message, response) in enumerate(y): + y[i] = ( + None if message is None else mdtex2html.convert((message)), + None if response is None else mdtex2html.convert(response), + ) + return y + + +gr.Chatbot.postprocess = postprocess + + +def parse_text(text): + """copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/""" + lines = text.split("\n") + lines = [line for line in lines if line != ""] + count = 0 + for i, line in enumerate(lines): + if "```" in line: + count += 1 + items = line.split("`") + if count % 2 == 1: + lines[i] = f'
'
+            else:
+                lines[i] = "
" + else: + if i > 0: + if count % 2 == 1: + line = line.replace("`", r"\`") + line = line.replace("<", "<") + line = line.replace(">", ">") + line = line.replace(" ", " ") + line = line.replace("*", "*") + line = line.replace("_", "_") + line = line.replace("-", "-") + line = line.replace(".", ".") + line = line.replace("!", "!") + line = line.replace("(", "(") + line = line.replace(")", ")") + line = line.replace("$", "$") + lines[i] = "
" + line + text = "".join(lines) + return text + + +def predict( + RETRY_FLAG, input, chatbot, max_length, top_p, temperature, history, past_key_values +): + try: + chatbot.append((parse_text(input), "")) + except Exception as exc: + logger.error(exc) + logger.debug(f"{chatbot=}") + _ = """ + if chatbot: + chatbot[-1] = (parse_text(input), str(exc)) + yield chatbot, history, past_key_values + # """ + yield chatbot, history, past_key_values + + for response, history, past_key_values in model_glm.stream_chat( + tokenizer, + input, + history, + past_key_values=past_key_values, + return_past_key_values=True, + max_length=max_length, + top_p=top_p, + temperature=temperature, + ): + chatbot[-1] = (parse_text(input), parse_text(response)) + # chatbot[-1][-1] = parse_text(response) + + yield chatbot, history, past_key_values, parse_text(response) + + +def trans_api(input, max_length=4096, top_p=0.8, temperature=0.2): + if max_length < 10: + max_length = 4096 + if top_p < 0.1 or top_p > 1: + top_p = 0.85 + if temperature <= 0 or temperature > 1: + temperature = 0.01 + try: + res, _ = model_glm.chat( + tokenizer, + input, + history=[], + past_key_values=None, + max_length=max_length, + top_p=top_p, + temperature=temperature, + ) + # logger.debug(f"{res=} \n{_=}") + except Exception as exc: + logger.error(f"{exc=}") + res = str(exc) + + return res + + +def reset_user_input(): + return gr.update(value="") + + +def reset_state(): + return [], [], None, "" + + +# Delete last turn +def delete_last_turn(chat, history): + if chat and history: + chat.pop(-1) + history.pop(-1) + return chat, history + + +# Regenerate response +def retry_last_answer( + user_input, chatbot, max_length, top_p, temperature, history, past_key_values +): + if chatbot and history: + # Removing the previous conversation from chat + chatbot.pop(-1) + # Setting up a flag to capture a retry + RETRY_FLAG = True + # Getting last message from user + user_input = history[-1][0] + # Removing bot response from the history + history.pop(-1) + + yield from predict( + RETRY_FLAG, # type: ignore + user_input, + chatbot, + max_length, + top_p, + temperature, + history, + past_key_values, + ) + +# print + +def print(text): + return text + +# TTS + +async def text_to_speech_edge(text, language_code): + voice = language_dict[language_code] + communicate = edge_tts.Communicate(text, voice) + with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp_file: + tmp_path = tmp_file.name + + await communicate.save(tmp_path) + + return tmp_path + + +with gr.Blocks(title="ChatGLM2-6B-int4", theme=gr.themes.Soft(text_size="sm"), analytics_enabled=False) as demo: + gr.HTML("
" + "

📺💕🎶 - ChatGLM2+声音克隆+视频对话：和喜欢的角色畅所欲言吧！

" + "
") + gr.Markdown("##
🥳 - ChatGLM2+FreeVC+SadTalker，为您打造沉浸式的视频对话体验，支持中英双语
") + gr.Markdown("##
🌊 - 更多精彩应用，尽在[滔滔AI](http://www.talktalkai.com)；滔滔AI，为爱滔滔！💕
") + gr.Markdown("###
⭐ - 如果您喜欢这个程序，欢迎给我的[GitHub项目](https://github.com/KevinWang676/ChatGLM2-Voice-Cloning)点赞支持！
") + + with gr.Tab("🍻 - ChatGLM2聊天区"): + with gr.Accordion("📒 相关信息", open=False): + _ = f""" ChatGLM2的可选参数信息： + * Low temperature: responses will be more deterministic and focused; High temperature: responses more creative. + * Suggested temperatures -- translation: up to 0.3; chatting: > 0.4 + * Top P controls dynamic vocabulary selection based on context.\n + 如果您想让ChatGLM2进行角色扮演并与之对话，请先输入恰当的提示词，如“请你扮演成动漫角色蜡笔小新并和我进行对话”；您也可以为ChatGLM2提供自定义的角色设定\n + 当您使用声音克隆功能时，请先在此程序的对应位置上传一段您喜欢的音频 + """ + gr.Markdown(dedent(_)) + chatbot = gr.Chatbot(height=300) + with gr.Row(): + with gr.Column(scale=4): + with gr.Column(scale=12): + user_input = gr.Textbox( + label="请在此处和GLM2聊天 (按回车键即可发送)", + placeholder="聊点什么吧", + ) + RETRY_FLAG = gr.Checkbox(value=False, visible=False) + with gr.Column(min_width=32, scale=1): + with gr.Row(): + submitBtn = gr.Button("开始和GLM2交流吧", variant="primary") + deleteBtn = gr.Button("删除最新一轮对话", variant="secondary") + retryBtn = gr.Button("重新生成最新一轮对话", variant="secondary") + + with gr.Accordion("🔧 更多设置", open=False): + with gr.Row(): + emptyBtn = gr.Button("清空所有聊天记录") + max_length = gr.Slider( + 0, + 32768, + value=8192, + step=1.0, + label="Maximum length", + interactive=True, + ) + top_p = gr.Slider( + 0, 1, value=0.85, step=0.01, label="Top P", interactive=True + ) + temperature = gr.Slider( + 0.01, 1, value=0.95, step=0.01, label="Temperature", interactive=True + ) + + + with gr.Row(): + test1 = gr.Textbox(label="GLM2的最新回答 (可编辑)", lines = 3) + with gr.Column(): + language = gr.Dropdown(choices=list(language_dict.keys()), value="普通话 (中国大陆)-Xiaoxiao-女", label="请选择文本对应的语言及您喜欢的说话人") + tts_btn = gr.Button("生成对应的音频吧", variant="primary") + output_audio = gr.Audio(type="filepath", label="为您生成的音频", interactive=False) + + tts_btn.click(text_to_speech_edge, inputs=[test1, language], outputs=[output_audio]) + + with gr.Row(): + model_choice = gr.Dropdown(choices=["FreeVC", "FreeVC-s", "FreeVC (24kHz)"], value="FreeVC (24kHz)", label="Model", visible=False) + audio1 = output_audio + audio2 = gr.Audio(label="请上传您喜欢的声音进行声音克隆", type='filepath') + clone_btn = gr.Button("开始AI声音克隆吧", variant="primary") + audio_cloned = gr.Audio(label="为您生成的专属声音克隆音频", type='filepath') + + clone_btn.click(convert, inputs=[model_choice, audio1, audio2], outputs=[audio_cloned]) + + history = gr.State([]) + past_key_values = gr.State(None) + + user_input.submit( + predict, + [ + RETRY_FLAG, + user_input, + chatbot, + max_length, + top_p, + temperature, + history, + past_key_values, + ], + [chatbot, history, past_key_values, test1], + show_progress="full", + ) + submitBtn.click( + predict, + [ + RETRY_FLAG, + user_input, + chatbot, + max_length, + top_p, + temperature, + history, + past_key_values, + ], + [chatbot, history, past_key_values, test1], + show_progress="full", + api_name="predict", + ) + submitBtn.click(reset_user_input, [], [user_input]) + + emptyBtn.click( + reset_state, outputs=[chatbot, history, past_key_values, test1], show_progress="full" + ) + + retryBtn.click( + retry_last_answer, + inputs=[ + user_input, + chatbot, + max_length, + top_p, + temperature, + history, + past_key_values, + ], + # outputs = [chatbot, history, last_user_message, user_message] + outputs=[chatbot, history, past_key_values, test1], + ) + deleteBtn.click(delete_last_turn, [chatbot, history], [chatbot, history]) + + with gr.Accordion("📔 提示词示例", open=False): + etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """ + examples = gr.Examples( + examples=[ + ["Explain the plot of Cinderella in a sentence."], + [ + "How long does it take to become proficient in French, and what are the best methods for retaining information?" + ], + ["What are some common mistakes to avoid when writing code?"], + ["Build a prompt to generate a beautiful portrait of a horse"], + ["Suggest four metaphors to describe the benefits of AI"], + ["Write a pop song about leaving home for the sandy beaches."], + ["Write a summary demonstrating my ability to tame lions"], + ["鲁迅和周树人什么关系"], + ["从前有一头牛，这头牛后面有什么？"], + ["正无穷大加一大于正无穷大吗？"], + ["正无穷大加正无穷大大于正无穷大吗？"], + ["-2的平方根等于什么"], + ["树上有5只鸟，猎人开枪打死了一只。树上还有几只鸟？"], + ["树上有11只鸟，猎人开枪打死了一只。树上还有几只鸟？提示：需考虑鸟可能受惊吓飞走。"], + ["鲁迅和周树人什么关系 用英文回答"], + ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"], + [f"{etext} 翻成中文，列出3个版本"], + [f"{etext} \n 翻成中文，保留原意，但使用文学性的语言。不要写解释。列出3个版本"], + ["js 判断一个数是不是质数"], + ["js 实现python 的 range(10)"], + ["js 实现python 的 [*(range(10)]"], + ["假定 1 + 2 = 4, 试求 7 + 8"], + ["Erkläre die Handlung von Cinderella in einem Satz."], + ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch"], + ], + inputs=[user_input], + examples_per_page=30, + ) + + with gr.Accordion("For Chat/Translation API", open=False, visible=False): + input_text = gr.Text() + tr_btn = gr.Button("Go", variant="primary") + out_text = gr.Text() + tr_btn.click( + trans_api, + [input_text, max_length, top_p, temperature], + out_text, + # show_progress="full", + api_name="tr", + ) + _ = """ + input_text.submit( + trans_api, + [input_text, max_length, top_p, temperature], + out_text, + show_progress="full", + api_name="tr1", + ) + # """ + with gr.Tab("📺 - 视频聊天区"): + with gr.Row().style(equal_height=False): + with gr.Column(variant='panel'): + with gr.Tabs(elem_id="sadtalker_source_image"): + with gr.TabItem('图片上传'): + with gr.Row(): + source_image = gr.Image(label="请上传一张您喜欢角色的图片", source="upload", type="filepath", elem_id="img2img_image").style(width=512) + + + with gr.Tabs(elem_id="sadtalker_driven_audio"): + with gr.TabItem('💡您还可以将视频下载到本地'): + + with gr.Row(): + driven_audio = audio_cloned + driven_audio_no = gr.Audio(label="Use IDLE mode, no audio is required", source="upload", type="filepath", visible=False) + + with gr.Column(): + use_idle_mode = gr.Checkbox(label="Use Idle Animation", visible=False) + length_of_audio = gr.Number(value=5, label="The length(seconds) of the generated video.", visible=False) + use_idle_mode.change(toggle_audio_file, inputs=use_idle_mode, outputs=[driven_audio, driven_audio_no]) # todo + + with gr.Row(): + ref_video = gr.Video(label="Reference Video", source="upload", type="filepath", elem_id="vidref", visible=False).style(width=512) + + with gr.Column(): + use_ref_video = gr.Checkbox(label="Use Reference Video", visible=False) + ref_info = gr.Radio(['pose', 'blink','pose+blink', 'all'], value='pose', label='Reference Video',info="How to borrow from reference Video?((fully transfer, aka, video driving mode))", visible=False) + + ref_video.change(ref_video_fn, inputs=ref_video, outputs=[use_ref_video]) # todo + + + with gr.Column(variant='panel'): + with gr.Tabs(elem_id="sadtalker_checkbox"): + with gr.TabItem('视频设置'): + with gr.Column(variant='panel'): + # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width + # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width + with gr.Row(): + pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0, visible=False) # + exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1, visible=False) # + blink_every = gr.Checkbox(label="use eye blink", value=True, visible=False) + + with gr.Row(): + size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model?", visible=False) # + preprocess_type = gr.Radio(['crop', 'full'], value='crop', label='是否聚焦角色面部', info="crop：视频会聚焦角色面部；full：视频会显示图片全貌") + + with gr.Row(): + is_still_mode = gr.Checkbox(label="静态模式 (开启静态模式，角色的面部动作会减少；默认开启)", value=True) + facerender = gr.Radio(['facevid2vid','pirender'], value='facevid2vid', label='facerender', info="which face render?", visible=False) + + with gr.Row(): + batch_size = gr.Slider(label="Batch size (数值越大，生成速度越快；若显卡性能好，可增大数值)", step=1, maximum=32, value=2) + enhancer = gr.Checkbox(label="GFPGAN as Face enhancer", value=True, visible=False) + + submit = gr.Button('开始视频聊天吧', elem_id="sadtalker_generate", variant='primary') + + with gr.Tabs(elem_id="sadtalker_genearted"): + gen_video = gr.Video(label="为您生成的专属视频", format="mp4").style(width=256) + + + + submit.click( + fn=sad_talker.test, + inputs=[source_image, + driven_audio, + preprocess_type, + is_still_mode, + enhancer, + batch_size, + size_of_image, + pose_style, + facerender, + exp_weight, + use_ref_video, + ref_video, + ref_info, + use_idle_mode, + length_of_audio, + blink_every + ], + outputs=[gen_video] + ) + gr.Markdown("###
注意❗：请不要生成会对个人以及组织造成侵害的内容，此程序仅供科研、学习及个人娱乐使用。
") + gr.Markdown("
💡- 如何使用此程序：输入您对ChatGLM的提问后，依次点击“开始和GLM2交流吧”、“生成对应的音频吧”、“开始AI声音克隆吧”、“开始视频聊天吧”三个按键即可；使用声音克隆功能时，请先上传一段您喜欢的音频
") + gr.HTML(''' +
+

🌊🏞️🎶 - 江水东流急，滔滔无尽声。 明·顾璘 +

+
+ ''') + + +demo.queue().launch(show_error=True, debug=True) diff --git a/.ipynb_checkpoints/app_sadtalker-checkpoint.py b/.ipynb_checkpoints/app_sadtalker-checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..900ad8f68d937e5b9de42cdb69d7218d924f70a1 --- /dev/null +++ b/.ipynb_checkpoints/app_sadtalker-checkpoint.py @@ -0,0 +1,111 @@ +import os, sys +import gradio as gr +from src.gradio_demo import SadTalker + + +try: + import webui # in webui + in_webui = True +except: + in_webui = False + + +def toggle_audio_file(choice): + if choice == False: + return gr.update(visible=True), gr.update(visible=False) + else: + return gr.update(visible=False), gr.update(visible=True) + +def ref_video_fn(path_of_ref_video): + if path_of_ref_video is not None: + return gr.update(value=True) + else: + return gr.update(value=False) + +def sadtalker_demo(checkpoint_path='checkpoints', config_path='src/config', warpfn=None): + + sad_talker = SadTalker(checkpoint_path, config_path, lazy_load=True) + + with gr.Blocks(analytics_enabled=False) as sadtalker_interface: + gr.Markdown("

😭 SadTalker: Learning Realistic 3D Motion Coefficients for Stylized Audio-Driven Single Image Talking Face Animation (CVPR 2023)

\ + Arxiv       \ + Homepage       \ + Github
") + + with gr.Row().style(equal_height=False): + with gr.Column(variant='panel'): + with gr.Tabs(elem_id="sadtalker_source_image"): + with gr.TabItem('Upload image'): + with gr.Row(): + source_image = gr.Image(label="Source image", source="upload", type="filepath", elem_id="img2img_image").style(width=512) + + with gr.Tabs(elem_id="sadtalker_driven_audio"): + with gr.TabItem('Upload OR TTS'): + with gr.Column(variant='panel'): + driven_audio = gr.Audio(label="Input audio", source="upload", type="filepath") + + if sys.platform != 'win32' and not in_webui: + from src.utils.text2speech import TTSTalker + tts_talker = TTSTalker() + with gr.Column(variant='panel'): + input_text = gr.Textbox(label="Generating audio from text", lines=5, placeholder="please enter some text here, we genreate the audio from text using @Coqui.ai TTS.") + tts = gr.Button('Generate audio',elem_id="sadtalker_audio_generate", variant='primary') + tts.click(fn=tts_talker.test, inputs=[input_text], outputs=[driven_audio]) + + with gr.Column(variant='panel'): + with gr.Tabs(elem_id="sadtalker_checkbox"): + with gr.TabItem('Settings'): + gr.Markdown("need help? please visit our [best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md) for more detials") + with gr.Column(variant='panel'): + # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width + # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width + pose_style = gr.Slider(minimum=0, maximum=46, step=1, label="Pose style", value=0) # + size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model?") # + preprocess_type = gr.Radio(['crop', 'resize','full', 'extcrop', 'extfull'], value='crop', label='preprocess', info="How to handle input image?") + is_still_mode = gr.Checkbox(label="Still Mode (fewer hand motion, works with preprocess `full`)") + batch_size = gr.Slider(label="batch size in generation", step=1, maximum=10, value=2) + enhancer = gr.Checkbox(label="GFPGAN as Face enhancer") + submit = gr.Button('Generate', elem_id="sadtalker_generate", variant='primary') + + with gr.Tabs(elem_id="sadtalker_genearted"): + gen_video = gr.Video(label="Generated video", format="mp4").style(width=256) + + if warpfn: + submit.click( + fn=warpfn(sad_talker.test), + inputs=[source_image, + driven_audio, + preprocess_type, + is_still_mode, + enhancer, + batch_size, + size_of_image, + pose_style + ], + outputs=[gen_video] + ) + else: + submit.click( + fn=sad_talker.test, + inputs=[source_image, + driven_audio, + preprocess_type, + is_still_mode, + enhancer, + batch_size, + size_of_image, + pose_style + ], + outputs=[gen_video] + ) + + return sadtalker_interface + + +if __name__ == "__main__": + + demo = sadtalker_demo() + demo.queue() + demo.launch(share=True) + + diff --git a/ChatGLM2-SadTalker-VC-1/.flake8 b/ChatGLM2-SadTalker-VC-1/.flake8 new file mode 100644 index 0000000000000000000000000000000000000000..4ad227243f71bcc84b6253f65f4cc06d98bc9f42 --- /dev/null +++ b/ChatGLM2-SadTalker-VC-1/.flake8 @@ -0,0 +1,21 @@ +[flake8] +ignore = + # E203 whitespace before ':' + E203 + D203, + # line too long + E501 +per-file-ignores = + # imported but unused + # __init__.py: F401 + test_*.py: F401 +exclude = + .git, + __pycache__, + docs/source/conf.py, + old, + build, + dist, + .venv + pad*.py +max-complexity = 25 \ No newline at end of file diff --git a/ChatGLM2-SadTalker-VC-1/.gitattributes b/ChatGLM2-SadTalker-VC-1/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..1e8a20b6bd8a30656a0d54968fa8b6ee5461b5bf --- /dev/null +++ b/ChatGLM2-SadTalker-VC-1/.gitattributes @@ -0,0 +1,52 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoints/BFM_Fitting/01_MorphableModel.mat filter=lfs diff=lfs merge=lfs -text +checkpoints/BFM_Fitting/BFM09_model_info.mat filter=lfs diff=lfs merge=lfs -text +checkpoints/facevid2vid_00189-model.pth.tar filter=lfs diff=lfs merge=lfs -text +checkpoints/mapping_00229-model.pth.tar filter=lfs diff=lfs merge=lfs -text +checkpoints/shape_predictor_68_face_landmarks.dat filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/chinese_news.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/deyu.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/eluosi.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/fayu.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/imagine.wav filter=lfs diff=lfs merge=lfs -text +examples/driven_audio/japanese.wav filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_16.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_17.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_3.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_4.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_5.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_8.png filter=lfs diff=lfs merge=lfs -text +examples/source_image/art_9.png filter=lfs diff=lfs merge=lfs -text diff --git a/ChatGLM2-SadTalker-VC-1/.gitignore b/ChatGLM2-SadTalker-VC-1/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..400ce70510811cd910fdd17d2f2ce1fb97123562 --- /dev/null +++ b/ChatGLM2-SadTalker-VC-1/.gitignore @@ -0,0 +1,159 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +results/ +checkpoints/ +gradio_cached_examples/ +gfpgan/ +start.sh \ No newline at end of file diff --git a/ChatGLM2-SadTalker-VC-1/.ipynb_checkpoints/requirements-checkpoint.txt b/ChatGLM2-SadTalker-VC-1/.ipynb_checkpoints/requirements-checkpoint.txt new file mode 100644 index 0000000000000000000000000000000000000000..eb3726a42398926c205728f9f1c8fde3db185338 --- /dev/null +++ b/ChatGLM2-SadTalker-VC-1/.ipynb_checkpoints/requirements-checkpoint.txt @@ -0,0 +1,12 @@ +scipy +transformers +librosa==0.8.1 +webrtcvad==2.0.10 +protobuf +cpm_kernels +mdtex2html +sentencepiece +accelerate +loguru +edge_tts +altair \ No newline at end of file diff --git a/ChatGLM2-SadTalker-VC-1/Dockerfile b/ChatGLM2-SadTalker-VC-1/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..5ddc6e3d8b246534a58f9612a88b309fa7e10795 --- /dev/null +++ b/ChatGLM2-SadTalker-VC-1/Dockerfile @@ -0,0 +1,59 @@ +FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + git \ + zip \ + unzip \ + git-lfs \ + wget \ + curl \ + # ffmpeg \ + ffmpeg \ + x264 \ + # python build dependencies \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libffi-dev \ + liblzma-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN useradd -m -u 1000 user +USER user +ENV HOME=/home/user \ + PATH=/home/user/.local/bin:${PATH} +WORKDIR ${HOME}/app + +RUN curl https://pyenv.run | bash +ENV PATH=${HOME}/.pyenv/shims:${HOME}/.pyenv/bin:${PATH} +ENV PYTHON_VERSION=3.10.9 +RUN pyenv install ${PYTHON_VERSION} && \ + pyenv global ${PYTHON_VERSION} && \ + pyenv rehash && \ + pip install --no-cache-dir -U pip setuptools wheel + +RUN pip install --no-cache-dir -U torch==1.12.1 torchvision==0.13.1 +COPY --chown=1000 requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -U -r /tmp/requirements.txt + +COPY --chown=1000 . ${HOME}/app +RUN ls -a +ENV PYTHONPATH=${HOME}/app \ + PYTHONUNBUFFERED=1 \ + GRADIO_ALLOW_FLAGGING=never \ + GRADIO_NUM_PORTS=1 \ + GRADIO_SERVER_NAME=0.0.0.0 \ + GRADIO_THEME=huggingface \ + SYSTEM=spaces +CMD ["python", "app.py"] \ No newline at end of file diff --git a/ChatGLM2-SadTalker-VC-1/LICENSE b/ChatGLM2-SadTalker-VC-1/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..b2a615ac931ce1e81df51deb56c3df2414b59e63 --- /dev/null +++ b/ChatGLM2-SadTalker-VC-1/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Tencent AI Lab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ChatGLM2-SadTalker-VC-1/README.md b/ChatGLM2-SadTalker-VC-1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..64db4f5eb15e38729a1919941603d723ff804629 --- /dev/null +++ b/ChatGLM2-SadTalker-VC-1/README.md @@ -0,0 +1,15 @@ +--- +title: ChatGLM2-SadTalker +emoji: 📺 +colorFrom: purple +colorTo: green +sdk: gradio +sdk_version: 3.23.0 +app_file: app.py +pinned: false +license: mit +duplicated_from: kevinwang676/ChatGLM2-SadTalker-VC +--- + + +Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference \ No newline at end of file diff --git a/ChatGLM2-SadTalker-VC-1/packages.txt b/ChatGLM2-SadTalker-VC-1/packages.txt new file mode 100644 index 0000000000000000000000000000000000000000..3101f5ec0d503c773ae2fcc863e1594cb689fc69 --- /dev/null +++ b/ChatGLM2-SadTalker-VC-1/packages.txt @@ -0,0 +1,2 @@ +ffmpeg +libsndfile1 \ No newline at end of file diff --git a/ChatGLM2-SadTalker-VC-1/requirements.txt b/ChatGLM2-SadTalker-VC-1/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..eb3726a42398926c205728f9f1c8fde3db185338 --- /dev/null +++ b/ChatGLM2-SadTalker-VC-1/requirements.txt @@ -0,0 +1,12 @@ +scipy +transformers +librosa==0.8.1 +webrtcvad==2.0.10 +protobuf +cpm_kernels +mdtex2html +sentencepiece +accelerate +loguru +edge_tts +altair \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..b2a615ac931ce1e81df51deb56c3df2414b59e63 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Tencent AI Lab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 0c8ae58f16fd04ab262695aa6fffa314002d690e..c330bdce5abbcb70982bc54b99389ab9919fbb3f 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,268 @@ ---- -title: ChatGLM2 SadTalker -emoji: 📚 -colorFrom: blue -colorTo: blue -sdk: gradio -sdk_version: 3.37.0 -app_file: app.py -pinned: false -license: mit ---- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +
+ + + + + + +     [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Winfredy/SadTalker/blob/main/quick_demo.ipynb)   [![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/vinthony/SadTalker)   [![sd webui-colab](https://img.shields.io/badge/Automatic1111-Colab-green)](https://colab.research.google.com/github/camenduru/stable-diffusion-webui-colab/blob/main/video/stable/stable_diffusion_1_5_video_webui_colab.ipynb)   [![Replicate](https://replicate.com/cjwbw/sadtalker/badge)](https://replicate.com/cjwbw/sadtalker) + +
+ Wenxuan Zhang *,1,2   + Xiaodong Cun *,2  + Xuan Wang 3  + Yong Zhang 2  + Xi Shen 2 
+ Yu Guo1   + Ying Shan 2   + Fei Wang 1   +
+
+
+ 1 Xi'an Jiaotong University   2 Tencent AI Lab   3 Ant Group   +
+
+CVPR 2023 +
+
+ + +![sadtalker](https://user-images.githubusercontent.com/4397546/222490039-b1f6156b-bf00-405b-9fda-0c9a9156f991.gif) + +TL;DR:       single portrait image 🙎‍♂️      +       audio 🎤       =       talking head video 🎞. + +
+ +
+ + + +## 🔥 Highlight + +- 🔥 The extension of the [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) is online. Checkout more details [here](docs/webui_extension.md). + +https://user-images.githubusercontent.com/4397546/231495639-5d4bb925-ea64-4a36-a519-6389917dac29.mp4 + +- 🔥 `full image mode` is online! checkout [here](https://github.com/Winfredy/SadTalker#full-bodyimage-generation) for more details. + +| still+enhancer in v0.0.1 | still + enhancer in v0.0.2 | [input image @bagbag1815](https://twitter.com/bagbag1815/status/1642754319094108161) | +|:--------------------: |:--------------------: | :----: | +| | | + +- 🔥 Several new mode, eg, `still mode`, `reference mode`, `resize mode` are online for better and custom applications. + +- 🔥 Happy to see more community demos at [bilibili](https://search.bilibili.com/all?keyword=sadtalker&from_source=webtop_search&spm_id_from=333.1007&search_source=3 +), [Youtube](https://www.youtube.com/results?search_query=sadtalker&sp=CAM%253D) and [twitter #sadtalker](https://twitter.com/search?q=%23sadtalker&src=typed_query). + +## 📋 Changelog (Previous changelog can be founded [here](docs/changlelog.md)) + +- __[2023.06.12]__: add more new features in WEBUI extension, see the discussion [here](https://github.com/OpenTalker/SadTalker/discussions/386). + +- __[2023.06.05]__: release a new 512 beta face model. Fixed some bugs and improve the performance. + +- __[2023.04.15]__: Adding automatic1111 colab by @camenduru, thanks for this awesome colab: [![sd webui-colab](https://img.shields.io/badge/Automatic1111-Colab-green)](https://colab.research.google.com/github/camenduru/stable-diffusion-webui-colab/blob/main/video/stable/stable_diffusion_1_5_video_webui_colab.ipynb). + +- __[2023.04.12]__: adding a more detailed sd-webui installation document, fixed reinstallation problem. + +- __[2023.04.12]__: Fixed the sd-webui safe issues becasue of the 3rd packages, optimize the output path in `sd-webui-extension`. + +- __[2023.04.08]__: ❗️❗️❗️ In v0.0.2, we add a logo watermark to the generated video to prevent abusing since it is very realistic. + +- __[2023.04.08]__: v0.0.2, full image animation, adding baidu driver for download checkpoints. Optimizing the logic about enhancer. + + +## 🚧 TODO: See the Discussion https://github.com/OpenTalker/SadTalker/issues/280 + +## If you have any problem, please view our [FAQ](docs/FAQ.md) before opening an issue. + + + +## ⚙️ 1. Installation. + +Tutorials from communities: [中文windows教程](https://www.bilibili.com/video/BV1Dc411W7V6/) | [日本語コース](https://br-d.fanbox.cc/posts/5685086?utm_campaign=manage_post_page&utm_medium=share&utm_source=twitter) + +### Linux: + +1. Installing [anaconda](https://www.anaconda.com/), python and git. + +2. Creating the env and install the requirements. + ```bash + git clone https://github.com/Winfredy/SadTalker.git + + cd SadTalker + + conda create -n sadtalker python=3.8 + + conda activate sadtalker + + pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113 + + conda install ffmpeg + + pip install -r requirements.txt + + ### tts is optional for gradio demo. + ### pip install TTS + + ``` +### Windows ([中文windows教程](https://www.bilibili.com/video/BV1Dc411W7V6/)): + +1. Install [Python 3.10.6](https://www.python.org/downloads/windows/), checking "Add Python to PATH". +2. Install [git](https://git-scm.com/download/win) manually (OR `scoop install git` via [scoop](https://scoop.sh/)). +3. Install `ffmpeg`, following [this instruction](https://www.wikihow.com/Install-FFmpeg-on-Windows) (OR using `scoop install ffmpeg` via [scoop](https://scoop.sh/)). +4. Download our SadTalker repository, for example by running `git clone https://github.com/Winfredy/SadTalker.git`. +5. Download the `checkpoint` and `gfpgan` [below↓](https://github.com/Winfredy/SadTalker#-2-download-trained-models). +5. Run `start.bat` from Windows Explorer as normal, non-administrator, user, a gradio WebUI demo will be started. + +### Macbook: + +More tips about installnation on Macbook and the Docker file can be founded [here](docs/install.md) + +## 📥 2. Download Trained Models. + +You can run the following script to put all the models in the right place. + +```bash +bash scripts/download_models.sh +``` + +Other alternatives: +> we also provide an offline patch (`gfpgan/`), thus, no model will be downloaded when generating. + +**Google Driver**: download our pre-trained model from [ this link (main checkpoints)](https://drive.google.com/file/d/1gwWh45pF7aelNP_P78uDJL8Sycep-K7j/view?usp=sharing) and [ gfpgan (offline patch)](https://drive.google.com/file/d/19AIBsmfcHW6BRJmeqSFlG5fL445Xmsyi?usp=sharing) + +**Github Release Page**: download all the files from the [lastest github release page](https://github.com/Winfredy/SadTalker/releases), and then, put it in ./checkpoints. + +**百度云盘**: we provided the downloaded model in [checkpoints, 提取码: sadt.](https://pan.baidu.com/s/1P4fRgk9gaSutZnn8YW034Q?pwd=sadt) And [gfpgan, 提取码: sadt.](https://pan.baidu.com/s/1kb1BCPaLOWX1JJb9Czbn6w?pwd=sadt) + + + +
Model Details + + +Model explains: + +##### New version +| Model | Description +| :--- | :---------- +|checkpoints/mapping_00229-model.pth.tar | Pre-trained MappingNet in Sadtalker. +|checkpoints/mapping_00109-model.pth.tar | Pre-trained MappingNet in Sadtalker. +|checkpoints/SadTalker_V0.0.2_256.safetensors | packaged sadtalker checkpoints of old version, 256 face render). +|checkpoints/SadTalker_V0.0.2_512.safetensors | packaged sadtalker checkpoints of old version, 512 face render). +|gfpgan/weights | Face detection and enhanced models used in `facexlib` and `gfpgan`. + + +##### Old version +| Model | Description +| :--- | :---------- +|checkpoints/auido2exp_00300-model.pth | Pre-trained ExpNet in Sadtalker. +|checkpoints/auido2pose_00140-model.pth | Pre-trained PoseVAE in Sadtalker. +|checkpoints/mapping_00229-model.pth.tar | Pre-trained MappingNet in Sadtalker. +|checkpoints/mapping_00109-model.pth.tar | Pre-trained MappingNet in Sadtalker. +|checkpoints/facevid2vid_00189-model.pth.tar | Pre-trained face-vid2vid model from [the reappearance of face-vid2vid](https://github.com/zhanglonghao1992/One-Shot_Free-View_Neural_Talking_Head_Synthesis). +|checkpoints/epoch_20.pth | Pre-trained 3DMM extractor in [Deep3DFaceReconstruction](https://github.com/microsoft/Deep3DFaceReconstruction). +|checkpoints/wav2lip.pth | Highly accurate lip-sync model in [Wav2lip](https://github.com/Rudrabha/Wav2Lip). +|checkpoints/shape_predictor_68_face_landmarks.dat | Face landmark model used in [dilb](http://dlib.net/). +|checkpoints/BFM | 3DMM library file. +|checkpoints/hub | Face detection models used in [face alignment](https://github.com/1adrianb/face-alignment). +|gfpgan/weights | Face detection and enhanced models used in `facexlib` and `gfpgan`. + +The final folder will be shown as: + +image + + +
+ +## 🔮 3. Quick Start ([Best Practice](docs/best_practice.md)). + +### WebUI Demos: + +**Online**: [Huggingface](https://huggingface.co/spaces/vinthony/SadTalker) | [SDWebUI-Colab](https://colab.research.google.com/github/camenduru/stable-diffusion-webui-colab/blob/main/video/stable/stable_diffusion_1_5_video_webui_colab.ipynb) | [Colab](https://colab.research.google.com/github/Winfredy/SadTalker/blob/main/quick_demo.ipynb) + +**Local Autiomatic1111 stable-diffusion webui extension**: please refer to [Autiomatic1111 stable-diffusion webui docs](docs/webui_extension.md). + +**Local gradio demo(highly recommanded!)**: Similar to our [hugging-face demo](https://huggingface.co/spaces/vinthony/SadTalker) can be run by: + +```bash +## you need manually install TTS(https://github.com/coqui-ai/TTS) via `pip install tts` in advanced. +python app.py +``` + +**Local gradio demo(highly recommanded!)**: + +- windows: just double click `webui.bat`, the requirements will be installed automatically. +- Linux/Mac OS: run `bash webui.sh` to start the webui. + + +### Manually usages: + +##### Animating a portrait image from default config: +```bash +python inference.py --driven_audio \ + --source_image \ + --enhancer gfpgan +``` +The results will be saved in `results/$SOME_TIMESTAMP/*.mp4`. + +##### Full body/image Generation: + +Using `--still` to generate a natural full body video. You can add `enhancer` to improve the quality of the generated video. + +```bash +python inference.py --driven_audio \ + --source_image \ + --result_dir \ + --still \ + --preprocess full \ + --enhancer gfpgan +``` + +More examples and configuration and tips can be founded in the [ >>> best practice documents <<<](docs/best_practice.md). + +## 🛎 Citation + +If you find our work useful in your research, please consider citing: + +```bibtex +@article{zhang2022sadtalker, + title={SadTalker: Learning Realistic 3D Motion Coefficients for Stylized Audio-Driven Single Image Talking Face Animation}, + author={Zhang, Wenxuan and Cun, Xiaodong and Wang, Xuan and Zhang, Yong and Shen, Xi and Guo, Yu and Shan, Ying and Wang, Fei}, + journal={arXiv preprint arXiv:2211.12194}, + year={2022} +} +``` + + + +## 💗 Acknowledgements + +Facerender code borrows heavily from [zhanglonghao's reproduction of face-vid2vid](https://github.com/zhanglonghao1992/One-Shot_Free-View_Neural_Talking_Head_Synthesis) and [PIRender](https://github.com/RenYurui/PIRender). We thank the authors for sharing their wonderful code. In training process, We also use the model from [Deep3DFaceReconstruction](https://github.com/microsoft/Deep3DFaceReconstruction) and [Wav2lip](https://github.com/Rudrabha/Wav2Lip). We thank for their wonderful work. + +See also these wonderful 3rd libraries we use: + +- **Face Utils**: https://github.com/xinntao/facexlib +- **Face Enhancement**: https://github.com/TencentARC/GFPGAN +- **Image/Video Enhancement**:https://github.com/xinntao/Real-ESRGAN + +## 🥂 Extensions: + +- [SadTalker-Video-Lip-Sync](https://github.com/Zz-ww/SadTalker-Video-Lip-Sync) from [@Zz-ww](https://github.com/Zz-ww): SadTalker for Video Lip Editing + +## 🥂 Related Works +- [StyleHEAT: One-Shot High-Resolution Editable Talking Face Generation via Pre-trained StyleGAN (ECCV 2022)](https://github.com/FeiiYin/StyleHEAT) +- [CodeTalker: Speech-Driven 3D Facial Animation with Discrete Motion Prior (CVPR 2023)](https://github.com/Doubiiu/CodeTalker) +- [VideoReTalking: Audio-based Lip Synchronization for Talking Head Video Editing In the Wild (SIGGRAPH Asia 2022)](https://github.com/vinthony/video-retalking) +- [DPE: Disentanglement of Pose and Expression for General Video Portrait Editing (CVPR 2023)](https://github.com/Carlyx/DPE) +- [3D GAN Inversion with Facial Symmetry Prior (CVPR 2023)](https://github.com/FeiiYin/SPI/) +- [T2M-GPT: Generating Human Motion from Textual Descriptions with Discrete Representations (CVPR 2023)](https://github.com/Mael-zys/T2M-GPT) + +## 📢 Disclaimer + +This is not an official product of Tencent. This repository can only be used for personal/research/non-commercial purposes. + +LOGO: color and font suggestion: [ChatGPT](ai.com), logo font：[Montserrat Alternates +](https://fonts.google.com/specimen/Montserrat+Alternates?preview.text=SadTalker&preview.text_type=custom&query=mont). + +All the copyright of the demo images and audio are from communities users or the geneartion from stable diffusion. Free free to contact us if you feel uncomfortable. + diff --git a/Untitled.ipynb b/Untitled.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..15bcb473ec31ec0d3edb41780d371992b34fc0c6 --- /dev/null +++ b/Untitled.ipynb @@ -0,0 +1,1886 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "c69a901b-8c3f-4188-97bd-5594c4496ec5", + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import HfApi\n", + "api = HfApi()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "8b249adc-ccd0-4145-86ce-64509ad276cf", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "faf7b41b81e54705bae8921f2a86e9fd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(HTML(value='
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "using safetensor as default\n", + "{'checkpoint': 'checkpoints/SadTalker_V0.0.2_256.safetensors', 'dir_of_BFM_fitting': 'src/config', 'audio2pose_yaml_path': 'src/config/auido2pose.yaml', 'audio2exp_yaml_path': 'src/config/auido2exp.yaml', 'use_safetensor': True, 'mappingnet_checkpoint': 'checkpoints/mapping_00229-model.pth.tar', 'facerender_yaml': 'src/config/facerender.yaml'}\n", + "/tmp/gradio/ee3af0f3e1518c57a03610bc7c5b9f774dd29f5c/image.png\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "landmark Det:: 100%|██████████| 1/1 [00:00<00:00, 26.62it/s]\n", + "3DMM Extraction In Video:: 100%|██████████| 1/1 [00:00<00:00, 119.22it/s]\n", + "mel:: 100%|██████████| 92/92 [00:00<00:00, 45445.29it/s]\n", + "audio2exp:: 100%|██████████| 10/10 [00:00<00:00, 102.60it/s]\n", + "Face Renderer:: 100%|██████████| 46/46 [00:05<00:00, 8.18it/s]\n", + "Numba: Attempted to fork from a non-main thread, the TBB library may be in an invalid state in the child process.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The generated video is named ./results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.mp4\n", + "face enhancer....\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Face Enhancer:: 0%| | 0/92 [00:00 142\u001b[0m \u001b[43mAutoModel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrust_remote_code\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mcuda()\u001b[38;5;241m.\u001b[39mhalf()\n\u001b[1;32m 143\u001b[0m ) \u001b[38;5;66;03m# 3.92G\u001b[39;00m\n\u001b[1;32m 145\u001b[0m model_glm \u001b[38;5;241m=\u001b[39m model_glm\u001b[38;5;241m.\u001b[39meval()\n\u001b[1;32m 147\u001b[0m _ \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\"\"\u001b[39m\u001b[38;5;124mOverride Chatbot.postprocess\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:488\u001b[0m, in \u001b[0;36m_BaseAutoModelClass.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m 486\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 487\u001b[0m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mregister(config\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m, model_class, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 488\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 489\u001b[0m \u001b[43m \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mhub_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 490\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 491\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(config) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m 492\u001b[0m model_class \u001b[38;5;241m=\u001b[39m _get_model_class(config, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping)\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/modeling_utils.py:2610\u001b[0m, in \u001b[0;36mPreTrainedModel.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m 2607\u001b[0m \u001b[38;5;66;03m# We'll need to download and cache each checkpoint shard if the checkpoint is sharded.\u001b[39;00m\n\u001b[1;32m 2608\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_sharded:\n\u001b[1;32m 2609\u001b[0m \u001b[38;5;66;03m# rsolved_archive_file becomes a list of files that point to the different checkpoint shards in this case.\u001b[39;00m\n\u001b[0;32m-> 2610\u001b[0m resolved_archive_file, sharded_metadata \u001b[38;5;241m=\u001b[39m \u001b[43mget_checkpoint_shard_files\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2611\u001b[0m \u001b[43m \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2612\u001b[0m \u001b[43m \u001b[49m\u001b[43mresolved_archive_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2613\u001b[0m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2614\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2615\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2616\u001b[0m \u001b[43m \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2617\u001b[0m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2618\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_auth_token\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2619\u001b[0m \u001b[43m \u001b[49m\u001b[43muser_agent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muser_agent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2620\u001b[0m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2621\u001b[0m \u001b[43m \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msubfolder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2622\u001b[0m \u001b[43m \u001b[49m\u001b[43m_commit_hash\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcommit_hash\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 2623\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2625\u001b[0m \u001b[38;5;66;03m# load pt weights early so that we know which dtype to init the model under\u001b[39;00m\n\u001b[1;32m 2626\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m from_pt:\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/utils/hub.py:958\u001b[0m, in \u001b[0;36mget_checkpoint_shard_files\u001b[0;34m(pretrained_model_name_or_path, index_filename, cache_dir, force_download, proxies, resume_download, local_files_only, use_auth_token, user_agent, revision, subfolder, _commit_hash)\u001b[0m\n\u001b[1;32m 955\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m shard_filename \u001b[38;5;129;01min\u001b[39;00m tqdm(shard_filenames, desc\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDownloading shards\u001b[39m\u001b[38;5;124m\"\u001b[39m, disable\u001b[38;5;241m=\u001b[39m\u001b[38;5;129;01mnot\u001b[39;00m show_progress_bar):\n\u001b[1;32m 956\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 957\u001b[0m \u001b[38;5;66;03m# Load from URL\u001b[39;00m\n\u001b[0;32m--> 958\u001b[0m cached_filename \u001b[38;5;241m=\u001b[39m \u001b[43mcached_file\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 959\u001b[0m \u001b[43m \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 960\u001b[0m \u001b[43m \u001b[49m\u001b[43mshard_filename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 961\u001b[0m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 962\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 963\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 964\u001b[0m \u001b[43m \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 965\u001b[0m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 966\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_auth_token\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_auth_token\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 967\u001b[0m \u001b[43m \u001b[49m\u001b[43muser_agent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muser_agent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 968\u001b[0m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 969\u001b[0m \u001b[43m \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msubfolder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 970\u001b[0m \u001b[43m \u001b[49m\u001b[43m_commit_hash\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_commit_hash\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 971\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 972\u001b[0m \u001b[38;5;66;03m# We have already dealt with RepositoryNotFoundError and RevisionNotFoundError when getting the index, so\u001b[39;00m\n\u001b[1;32m 973\u001b[0m \u001b[38;5;66;03m# we don't have to catch them here.\u001b[39;00m\n\u001b[1;32m 974\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m EntryNotFoundError:\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/transformers/utils/hub.py:417\u001b[0m, in \u001b[0;36mcached_file\u001b[0;34m(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, use_auth_token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash)\u001b[0m\n\u001b[1;32m 414\u001b[0m user_agent \u001b[38;5;241m=\u001b[39m http_user_agent(user_agent)\n\u001b[1;32m 415\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 416\u001b[0m \u001b[38;5;66;03m# Load from URL or cache if already cached\u001b[39;00m\n\u001b[0;32m--> 417\u001b[0m resolved_file \u001b[38;5;241m=\u001b[39m \u001b[43mhf_hub_download\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 418\u001b[0m \u001b[43m \u001b[49m\u001b[43mpath_or_repo_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 419\u001b[0m \u001b[43m \u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 420\u001b[0m \u001b[43m \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m==\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43msubfolder\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 421\u001b[0m \u001b[43m \u001b[49m\u001b[43mrepo_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrepo_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 422\u001b[0m \u001b[43m \u001b[49m\u001b[43mrevision\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrevision\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 423\u001b[0m \u001b[43m \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 424\u001b[0m \u001b[43m \u001b[49m\u001b[43muser_agent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muser_agent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 425\u001b[0m \u001b[43m \u001b[49m\u001b[43mforce_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mforce_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 426\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 427\u001b[0m \u001b[43m \u001b[49m\u001b[43mresume_download\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 428\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_auth_token\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_auth_token\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 429\u001b[0m \u001b[43m \u001b[49m\u001b[43mlocal_files_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlocal_files_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 430\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 432\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m RepositoryNotFoundError:\n\u001b[1;32m 433\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mEnvironmentError\u001b[39;00m(\n\u001b[1;32m 434\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath_or_repo_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m is not a local folder and is not a valid model identifier \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 435\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlisted on \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://huggingface.co/models\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mIf this is a private repository, make sure to \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 436\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpass a token having permission to this repo with `use_auth_token` or log in with \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 437\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`huggingface-cli login` and pass `use_auth_token=True`.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 438\u001b[0m )\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:118\u001b[0m, in \u001b[0;36mvalidate_hf_hub_args.._inner_fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m check_use_auth_token:\n\u001b[1;32m 116\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m smoothly_deprecate_use_auth_token(fn_name\u001b[38;5;241m=\u001b[39mfn\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, has_token\u001b[38;5;241m=\u001b[39mhas_token, kwargs\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[0;32m--> 118\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1364\u001b[0m, in \u001b[0;36mhf_hub_download\u001b[0;34m(repo_id, filename, subfolder, repo_type, revision, library_name, library_version, cache_dir, local_dir, local_dir_use_symlinks, user_agent, force_download, force_filename, proxies, etag_timeout, resume_download, token, local_files_only, legacy_cache_layout)\u001b[0m\n\u001b[1;32m 1361\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m temp_file_manager() \u001b[38;5;28;01mas\u001b[39;00m temp_file:\n\u001b[1;32m 1362\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdownloading \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m to \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, url, temp_file\u001b[38;5;241m.\u001b[39mname)\n\u001b[0;32m-> 1364\u001b[0m \u001b[43mhttp_get\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1365\u001b[0m \u001b[43m \u001b[49m\u001b[43murl_to_download\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1366\u001b[0m \u001b[43m \u001b[49m\u001b[43mtemp_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1367\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproxies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1368\u001b[0m \u001b[43m \u001b[49m\u001b[43mresume_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1369\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1370\u001b[0m \u001b[43m \u001b[49m\u001b[43mexpected_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexpected_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1371\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1373\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m local_dir \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 1374\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mStoring \u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m in cache at \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mblob_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:544\u001b[0m, in \u001b[0;36mhttp_get\u001b[0;34m(url, temp_file, proxies, resume_size, headers, timeout, max_retries, expected_size)\u001b[0m\n\u001b[1;32m 542\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunk: \u001b[38;5;66;03m# filter out keep-alive new chunks\u001b[39;00m\n\u001b[1;32m 543\u001b[0m progress\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;28mlen\u001b[39m(chunk))\n\u001b[0;32m--> 544\u001b[0m \u001b[43mtemp_file\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43mchunk\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 546\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m expected_size \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m expected_size \u001b[38;5;241m!=\u001b[39m temp_file\u001b[38;5;241m.\u001b[39mtell():\n\u001b[1;32m 547\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mEnvironmentError\u001b[39;00m(\n\u001b[1;32m 548\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConsistency check failed: file should be of size \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexpected_size\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m but has size\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 549\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtemp_file\u001b[38;5;241m.\u001b[39mtell()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdisplayed_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m).\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mWe are sorry for the inconvenience. Please retry download and\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 550\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m pass `force_download=True, resume_download=False` as argument.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mIf the issue persists, please let us\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 551\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m know by opening an issue on https://github.com/huggingface/huggingface_hub.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 552\u001b[0m )\n", + "File \u001b[0;32m/opt/conda/lib/python3.10/tempfile.py:483\u001b[0m, in \u001b[0;36m_TemporaryFileWrapper.__getattr__..func_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 481\u001b[0m \u001b[38;5;129m@_functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 482\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfunc_wrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 483\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mOSError\u001b[0m: [Errno 28] No space left on device" + ] + } + ], + "source": [ + "import os\n", + "import torch\n", + "import librosa\n", + "import gradio as gr\n", + "from scipy.io.wavfile import write\n", + "from transformers import WavLMModel\n", + "\n", + "import utils\n", + "from models import SynthesizerTrn\n", + "from mel_processing import mel_spectrogram_torch\n", + "from speaker_encoder.voice_encoder import SpeakerEncoder\n", + "\n", + "import time\n", + "from textwrap import dedent\n", + "\n", + "import mdtex2html\n", + "from loguru import logger\n", + "from transformers import AutoModel, AutoTokenizer\n", + "\n", + "from tts_voice import tts_order_voice\n", + "import edge_tts\n", + "import tempfile\n", + "import anyio\n", + "\n", + "import os, sys\n", + "import gradio as gr\n", + "from src.gradio_demo import SadTalker \n", + "\n", + "\n", + "try:\n", + " import webui # in webui\n", + " in_webui = True\n", + "except:\n", + " in_webui = False\n", + "\n", + "\n", + "def toggle_audio_file(choice):\n", + " if choice == False:\n", + " return gr.update(visible=True), gr.update(visible=False)\n", + " else:\n", + " return gr.update(visible=False), gr.update(visible=True)\n", + " \n", + "def ref_video_fn(path_of_ref_video):\n", + " if path_of_ref_video is not None:\n", + " return gr.update(value=True)\n", + " else:\n", + " return gr.update(value=False)\n", + "\n", + "sad_talker = SadTalker(\"checkpoints\", \"src/config\", lazy_load=True)\n", + "\n", + "'''\n", + "def get_wavlm():\n", + " os.system('gdown https://drive.google.com/uc?id=12-cB34qCTvByWT-QtOcZaqwwO21FLSqU')\n", + " shutil.move('WavLM-Large.pt', 'wavlm')\n", + "'''\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "\n", + "smodel = SpeakerEncoder('speaker_encoder/ckpt/pretrained_bak_5805000.pt')\n", + "\n", + "print(\"Loading FreeVC(24k)...\")\n", + "hps = utils.get_hparams_from_file(\"configs/freevc-24.json\")\n", + "freevc_24 = SynthesizerTrn(\n", + " hps.data.filter_length // 2 + 1,\n", + " hps.train.segment_size // hps.data.hop_length,\n", + " **hps.model).to(device)\n", + "_ = freevc_24.eval()\n", + "_ = utils.load_checkpoint(\"checkpoint/freevc-24.pth\", freevc_24, None)\n", + "\n", + "print(\"Loading WavLM for content...\")\n", + "cmodel = WavLMModel.from_pretrained(\"microsoft/wavlm-large\").to(device)\n", + " \n", + "def convert(model, src, tgt):\n", + " with torch.no_grad():\n", + " # tgt\n", + " wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate)\n", + " wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20)\n", + " if model == \"FreeVC\" or model == \"FreeVC (24kHz)\":\n", + " g_tgt = smodel.embed_utterance(wav_tgt)\n", + " g_tgt = torch.from_numpy(g_tgt).unsqueeze(0).to(device)\n", + " else:\n", + " wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).to(device)\n", + " mel_tgt = mel_spectrogram_torch(\n", + " wav_tgt, \n", + " hps.data.filter_length,\n", + " hps.data.n_mel_channels,\n", + " hps.data.sampling_rate,\n", + " hps.data.hop_length,\n", + " hps.data.win_length,\n", + " hps.data.mel_fmin,\n", + " hps.data.mel_fmax\n", + " )\n", + " # src\n", + " wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate)\n", + " wav_src = torch.from_numpy(wav_src).unsqueeze(0).to(device)\n", + " c = cmodel(wav_src).last_hidden_state.transpose(1, 2).to(device)\n", + " # infer\n", + " if model == \"FreeVC\":\n", + " audio = freevc.infer(c, g=g_tgt)\n", + " elif model == \"FreeVC-s\":\n", + " audio = freevc_s.infer(c, mel=mel_tgt)\n", + " else:\n", + " audio = freevc_24.infer(c, g=g_tgt)\n", + " audio = audio[0][0].data.cpu().float().numpy()\n", + " if model == \"FreeVC\" or model == \"FreeVC-s\":\n", + " write(\"out.wav\", hps.data.sampling_rate, audio)\n", + " else:\n", + " write(\"out.wav\", 24000, audio)\n", + " out = \"out.wav\"\n", + " return out\n", + "\n", + "# GLM2\n", + "\n", + "language_dict = tts_order_voice\n", + "\n", + "# fix timezone in Linux\n", + "os.environ[\"TZ\"] = \"Asia/Shanghai\"\n", + "try:\n", + " time.tzset() # type: ignore # pylint: disable=no-member\n", + "except Exception:\n", + " # Windows\n", + " logger.warning(\"Windows, cant run time.tzset()\")\n", + "\n", + "# model_name = \"THUDM/chatglm2-6b\"\n", + "model_name = \"THUDM/chatglm2-6b\"\n", + "\n", + "RETRY_FLAG = False\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n", + "\n", + "# model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda()\n", + "\n", + "# 4/8 bit\n", + "# model = AutoModel.from_pretrained(\"THUDM/chatglm2-6b\", trust_remote_code=True).quantize(4).cuda()\n", + "\n", + "has_cuda = torch.cuda.is_available()\n", + "\n", + "# has_cuda = False # force cpu\n", + "\n", + "\n", + "model_glm = (\n", + " AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half()\n", + ") # 3.92G\n", + "\n", + "model_glm = model_glm.eval()\n", + "\n", + "_ = \"\"\"Override Chatbot.postprocess\"\"\"\n", + "\n", + "\n", + "def postprocess(self, y):\n", + " if y is None:\n", + " return []\n", + " for i, (message, response) in enumerate(y):\n", + " y[i] = (\n", + " None if message is None else mdtex2html.convert((message)),\n", + " None if response is None else mdtex2html.convert(response),\n", + " )\n", + " return y\n", + "\n", + "\n", + "gr.Chatbot.postprocess = postprocess\n", + "\n", + "\n", + "def parse_text(text):\n", + " \"\"\"copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/\"\"\"\n", + " lines = text.split(\"\\n\")\n", + " lines = [line for line in lines if line != \"\"]\n", + " count = 0\n", + " for i, line in enumerate(lines):\n", + " if \"```\" in line:\n", + " count += 1\n", + " items = line.split(\"`\")\n", + " if count % 2 == 1:\n", + " lines[i] = f'
'\n",
+    "            else:\n",
+    "                lines[i] = \"
\"\n", + " else:\n", + " if i > 0:\n", + " if count % 2 == 1:\n", + " line = line.replace(\"`\", r\"\\`\")\n", + " line = line.replace(\"<\", \"<\")\n", + " line = line.replace(\">\", \">\")\n", + " line = line.replace(\" \", \" \")\n", + " line = line.replace(\"*\", \"*\")\n", + " line = line.replace(\"_\", \"_\")\n", + " line = line.replace(\"-\", \"-\")\n", + " line = line.replace(\".\", \".\")\n", + " line = line.replace(\"!\", \"!\")\n", + " line = line.replace(\"(\", \"(\")\n", + " line = line.replace(\")\", \")\")\n", + " line = line.replace(\"$\", \"$\")\n", + " lines[i] = \"
\" + line\n", + " text = \"\".join(lines)\n", + " return text\n", + "\n", + "\n", + "def predict(\n", + " RETRY_FLAG, input, chatbot, max_length, top_p, temperature, history, past_key_values\n", + "):\n", + " try:\n", + " chatbot.append((parse_text(input), \"\"))\n", + " except Exception as exc:\n", + " logger.error(exc)\n", + " logger.debug(f\"{chatbot=}\")\n", + " _ = \"\"\"\n", + " if chatbot:\n", + " chatbot[-1] = (parse_text(input), str(exc))\n", + " yield chatbot, history, past_key_values\n", + " # \"\"\"\n", + " yield chatbot, history, past_key_values\n", + "\n", + " for response, history, past_key_values in model_glm.stream_chat(\n", + " tokenizer,\n", + " input,\n", + " history,\n", + " past_key_values=past_key_values,\n", + " return_past_key_values=True,\n", + " max_length=max_length,\n", + " top_p=top_p,\n", + " temperature=temperature,\n", + " ):\n", + " chatbot[-1] = (parse_text(input), parse_text(response))\n", + " # chatbot[-1][-1] = parse_text(response)\n", + "\n", + " yield chatbot, history, past_key_values, parse_text(response)\n", + "\n", + "\n", + "def trans_api(input, max_length=4096, top_p=0.8, temperature=0.2):\n", + " if max_length < 10:\n", + " max_length = 4096\n", + " if top_p < 0.1 or top_p > 1:\n", + " top_p = 0.85\n", + " if temperature <= 0 or temperature > 1:\n", + " temperature = 0.01\n", + " try:\n", + " res, _ = model_glm.chat(\n", + " tokenizer,\n", + " input,\n", + " history=[],\n", + " past_key_values=None,\n", + " max_length=max_length,\n", + " top_p=top_p,\n", + " temperature=temperature,\n", + " )\n", + " # logger.debug(f\"{res=} \\n{_=}\")\n", + " except Exception as exc:\n", + " logger.error(f\"{exc=}\")\n", + " res = str(exc)\n", + "\n", + " return res\n", + "\n", + "\n", + "def reset_user_input():\n", + " return gr.update(value=\"\")\n", + "\n", + "\n", + "def reset_state():\n", + " return [], [], None, \"\"\n", + "\n", + "\n", + "# Delete last turn\n", + "def delete_last_turn(chat, history):\n", + " if chat and history:\n", + " chat.pop(-1)\n", + " history.pop(-1)\n", + " return chat, history\n", + "\n", + "\n", + "# Regenerate response\n", + "def retry_last_answer(\n", + " user_input, chatbot, max_length, top_p, temperature, history, past_key_values\n", + "):\n", + " if chatbot and history:\n", + " # Removing the previous conversation from chat\n", + " chatbot.pop(-1)\n", + " # Setting up a flag to capture a retry\n", + " RETRY_FLAG = True\n", + " # Getting last message from user\n", + " user_input = history[-1][0]\n", + " # Removing bot response from the history\n", + " history.pop(-1)\n", + "\n", + " yield from predict(\n", + " RETRY_FLAG, # type: ignore\n", + " user_input,\n", + " chatbot,\n", + " max_length,\n", + " top_p,\n", + " temperature,\n", + " history,\n", + " past_key_values,\n", + " )\n", + "\n", + "# print\n", + "\n", + "def print(text):\n", + " return text\n", + "\n", + "# TTS\n", + "\n", + "async def text_to_speech_edge(text, language_code):\n", + " voice = language_dict[language_code]\n", + " communicate = edge_tts.Communicate(text, voice)\n", + " with tempfile.NamedTemporaryFile(delete=False, suffix=\".mp3\") as tmp_file:\n", + " tmp_path = tmp_file.name\n", + "\n", + " await communicate.save(tmp_path)\n", + "\n", + " return tmp_path\n", + "\n", + "\n", + "with gr.Blocks(title=\"ChatGLM2-6B-int4\", theme=gr.themes.Soft(text_size=\"sm\")) as demo:\n", + " gr.HTML(\"
\"\n", + " \"

🥳💕🎶 - ChatGLM2 + 声音克隆：和你喜欢的角色畅所欲言吧！

\"\n", + " \"
\")\n", + " gr.Markdown(\"##
💡 - 第二代ChatGLM大语言模型 + FreeVC变声，为您打造独一无二的沉浸式对话体验，支持中英双语
\")\n", + " gr.Markdown(\"##
🌊 - 更多精彩应用，尽在[滔滔AI](http://www.talktalkai.com)；滔滔AI，为爱滔滔！💕
\")\n", + " gr.Markdown(\"###
⭐ - 如果您喜欢这个程序，欢迎给我的[Github项目](https://github.com/KevinWang676/ChatGLM2-Voice-Cloning)点赞支持！
\")\n", + " with gr.Tab(\"Chat\"):\n", + " with gr.Accordion(\"📒 相关信息\", open=False):\n", + " _ = f\"\"\" ChatGLM2的可选参数信息：\n", + " * Low temperature: responses will be more deterministic and focused; High temperature: responses more creative.\n", + " * Suggested temperatures -- translation: up to 0.3; chatting: > 0.4\n", + " * Top P controls dynamic vocabulary selection based on context.\\n\n", + " 如果您想让ChatGLM2进行角色扮演并与之对话，请先输入恰当的提示词，如“请你扮演成动漫角色蜡笔小新并和我进行对话”；您也可以为ChatGLM2提供自定义的角色设定\\n\n", + " 当您使用声音克隆功能时，请先在此程序的对应位置上传一段您喜欢的音频\n", + " \"\"\"\n", + " gr.Markdown(dedent(_))\n", + " chatbot = gr.Chatbot(height=300)\n", + " with gr.Row():\n", + " with gr.Column(scale=4):\n", + " with gr.Column(scale=12):\n", + " user_input = gr.Textbox(\n", + " label=\"请在此处和GLM2聊天 (按回车键即可发送)\",\n", + " placeholder=\"聊点什么吧\",\n", + " )\n", + " RETRY_FLAG = gr.Checkbox(value=False, visible=False)\n", + " with gr.Column(min_width=32, scale=1):\n", + " with gr.Row():\n", + " submitBtn = gr.Button(\"开始和GLM2交流吧\", variant=\"primary\")\n", + " deleteBtn = gr.Button(\"删除最新一轮对话\", variant=\"secondary\")\n", + " retryBtn = gr.Button(\"重新生成最新一轮对话\", variant=\"secondary\")\n", + " \n", + " with gr.Accordion(\"🔧 更多设置\", open=False):\n", + " with gr.Row():\n", + " emptyBtn = gr.Button(\"清空所有聊天记录\")\n", + " max_length = gr.Slider(\n", + " 0,\n", + " 32768,\n", + " value=8192,\n", + " step=1.0,\n", + " label=\"Maximum length\",\n", + " interactive=True,\n", + " )\n", + " top_p = gr.Slider(\n", + " 0, 1, value=0.85, step=0.01, label=\"Top P\", interactive=True\n", + " )\n", + " temperature = gr.Slider(\n", + " 0.01, 1, value=0.95, step=0.01, label=\"Temperature\", interactive=True\n", + " )\n", + "\n", + "\n", + " with gr.Row():\n", + " test1 = gr.Textbox(label=\"GLM2的最新回答 (可编辑)\", lines = 3)\n", + " with gr.Column():\n", + " language = gr.Dropdown(choices=list(language_dict.keys()), value=\"普通话 (中国大陆)-Xiaoxiao-女\", label=\"请选择文本对应的语言及您喜欢的说话人\")\n", + " tts_btn = gr.Button(\"生成对应的音频吧\", variant=\"primary\")\n", + " output_audio = gr.Audio(type=\"filepath\", label=\"为您生成的音频\", interactive=False)\n", + " \n", + " tts_btn.click(text_to_speech_edge, inputs=[test1, language], outputs=[output_audio])\n", + " \n", + " with gr.Row():\n", + " model_choice = gr.Dropdown(choices=[\"FreeVC\", \"FreeVC-s\", \"FreeVC (24kHz)\"], value=\"FreeVC (24kHz)\", label=\"Model\", visible=False) \n", + " audio1 = output_audio\n", + " audio2 = gr.Audio(label=\"请上传您喜欢的声音进行声音克隆\", type='filepath')\n", + " clone_btn = gr.Button(\"开始AI声音克隆吧\", variant=\"primary\")\n", + " audio_cloned = gr.Audio(label=\"为您生成的专属声音克隆音频\", type='filepath')\n", + " \n", + " clone_btn.click(convert, inputs=[model_choice, audio1, audio2], outputs=[audio_cloned])\n", + " \n", + " history = gr.State([])\n", + " past_key_values = gr.State(None)\n", + " \n", + " user_input.submit(\n", + " predict,\n", + " [\n", + " RETRY_FLAG,\n", + " user_input,\n", + " chatbot,\n", + " max_length,\n", + " top_p,\n", + " temperature,\n", + " history,\n", + " past_key_values,\n", + " ],\n", + " [chatbot, history, past_key_values, test1],\n", + " show_progress=\"full\",\n", + " )\n", + " submitBtn.click(\n", + " predict,\n", + " [\n", + " RETRY_FLAG,\n", + " user_input,\n", + " chatbot,\n", + " max_length,\n", + " top_p,\n", + " temperature,\n", + " history,\n", + " past_key_values,\n", + " ],\n", + " [chatbot, history, past_key_values, test1],\n", + " show_progress=\"full\",\n", + " api_name=\"predict\",\n", + " )\n", + " submitBtn.click(reset_user_input, [], [user_input])\n", + " \n", + " emptyBtn.click(\n", + " reset_state, outputs=[chatbot, history, past_key_values, test1], show_progress=\"full\"\n", + " )\n", + "\n", + " retryBtn.click(\n", + " retry_last_answer,\n", + " inputs=[\n", + " user_input,\n", + " chatbot,\n", + " max_length,\n", + " top_p,\n", + " temperature,\n", + " history,\n", + " past_key_values,\n", + " ],\n", + " # outputs = [chatbot, history, last_user_message, user_message]\n", + " outputs=[chatbot, history, past_key_values, test1],\n", + " )\n", + " deleteBtn.click(delete_last_turn, [chatbot, history], [chatbot, history])\n", + "\n", + " with gr.Accordion(\"📔 提示词示例\", open=False):\n", + " etext = \"\"\"In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. \"\"\"\n", + " examples = gr.Examples(\n", + " examples=[\n", + " [\"Explain the plot of Cinderella in a sentence.\"],\n", + " [\n", + " \"How long does it take to become proficient in French, and what are the best methods for retaining information?\"\n", + " ],\n", + " [\"What are some common mistakes to avoid when writing code?\"],\n", + " [\"Build a prompt to generate a beautiful portrait of a horse\"],\n", + " [\"Suggest four metaphors to describe the benefits of AI\"],\n", + " [\"Write a pop song about leaving home for the sandy beaches.\"],\n", + " [\"Write a summary demonstrating my ability to tame lions\"],\n", + " [\"鲁迅和周树人什么关系\"],\n", + " [\"从前有一头牛，这头牛后面有什么？\"],\n", + " [\"正无穷大加一大于正无穷大吗？\"],\n", + " [\"正无穷大加正无穷大大于正无穷大吗？\"],\n", + " [\"-2的平方根等于什么\"],\n", + " [\"树上有5只鸟，猎人开枪打死了一只。树上还有几只鸟？\"],\n", + " [\"树上有11只鸟，猎人开枪打死了一只。树上还有几只鸟？提示：需考虑鸟可能受惊吓飞走。\"],\n", + " [\"鲁迅和周树人什么关系 用英文回答\"],\n", + " [\"以红楼梦的行文风格写一张委婉的请假条。不少于320字。\"],\n", + " [f\"{etext} 翻成中文，列出3个版本\"],\n", + " [f\"{etext} \\n 翻成中文，保留原意，但使用文学性的语言。不要写解释。列出3个版本\"],\n", + " [\"js 判断一个数是不是质数\"],\n", + " [\"js 实现python 的 range(10)\"],\n", + " [\"js 实现python 的 [*(range(10)]\"],\n", + " [\"假定 1 + 2 = 4, 试求 7 + 8\"],\n", + " [\"Erkläre die Handlung von Cinderella in einem Satz.\"],\n", + " [\"Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch\"],\n", + " ],\n", + " inputs=[user_input],\n", + " examples_per_page=30,\n", + " )\n", + " \n", + " with gr.Accordion(\"For Chat/Translation API\", open=False, visible=False):\n", + " input_text = gr.Text()\n", + " tr_btn = gr.Button(\"Go\", variant=\"primary\")\n", + " out_text = gr.Text()\n", + " tr_btn.click(\n", + " trans_api,\n", + " [input_text, max_length, top_p, temperature],\n", + " out_text,\n", + " # show_progress=\"full\",\n", + " api_name=\"tr\",\n", + " )\n", + " _ = \"\"\"\n", + " input_text.submit(\n", + " trans_api,\n", + " [input_text, max_length, top_p, temperature],\n", + " out_text,\n", + " show_progress=\"full\",\n", + " api_name=\"tr1\",\n", + " )\n", + " # \"\"\"\n", + "\n", + " with gr.Tab(\"Video\"):\n", + " with gr.Row().style(equal_height=False):\n", + " with gr.Column(variant='panel'):\n", + " with gr.Tabs(elem_id=\"sadtalker_source_image\"):\n", + " with gr.TabItem('Upload image'):\n", + " with gr.Row():\n", + " source_image = gr.Image(label=\"Source image\", source=\"upload\", type=\"filepath\", elem_id=\"img2img_image\").style(width=512)\n", + " \n", + " with gr.Tabs(elem_id=\"sadtalker_driven_audio\"):\n", + " with gr.TabItem('Upload OR TTS'):\n", + " with gr.Column(variant='panel'):\n", + " driven_audio = gr.Audio(label=\"Input audio\", source=\"upload\", type=\"filepath\")\n", + " \n", + " with gr.Column(variant='panel'): \n", + " with gr.Tabs(elem_id=\"sadtalker_checkbox\"):\n", + " with gr.TabItem('Settings'):\n", + " gr.Markdown(\"need help? please visit our [best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md) for more detials\")\n", + " with gr.Column(variant='panel'):\n", + " # width = gr.Slider(minimum=64, elem_id=\"img2img_width\", maximum=2048, step=8, label=\"Manually Crop Width\", value=512) # img2img_width\n", + " # height = gr.Slider(minimum=64, elem_id=\"img2img_height\", maximum=2048, step=8, label=\"Manually Crop Height\", value=512) # img2img_width\n", + " pose_style = gr.Slider(minimum=0, maximum=46, step=1, label=\"Pose style\", value=0) # \n", + " size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info=\"use 256/512 model?\") # \n", + " preprocess_type = gr.Radio(['crop', 'resize','full', 'extcrop', 'extfull'], value='crop', label='preprocess', info=\"How to handle input image?\")\n", + " is_still_mode = gr.Checkbox(label=\"Still Mode (fewer hand motion, works with preprocess `full`)\")\n", + " batch_size = gr.Slider(label=\"batch size in generation\", step=1, maximum=10, value=2)\n", + " enhancer = gr.Checkbox(label=\"GFPGAN as Face enhancer\")\n", + " submit = gr.Button('Generate', elem_id=\"sadtalker_generate\", variant='primary')\n", + " \n", + " with gr.Tabs(elem_id=\"sadtalker_genearted\"):\n", + " gen_video = gr.Video(label=\"Generated video\", format=\"mp4\").style(width=256)\n", + " \n", + " submit.click(\n", + " fn=sad_talker.test, \n", + " inputs=[source_image,\n", + " driven_audio,\n", + " preprocess_type,\n", + " is_still_mode,\n", + " enhancer,\n", + " batch_size, \n", + " size_of_image,\n", + " pose_style\n", + " ], \n", + " outputs=[gen_video]\n", + " )\n", + "\n", + "\n", + " gr.Markdown(\"###
注意❗：请不要生成会对个人以及组织造成侵害的内容，此程序仅供科研、学习及个人娱乐使用。
\")\n", + " gr.Markdown(\"
💡 - 如何使用此程序：输入您对ChatGLM的提问后，依次点击“开始和GLM2交流吧”、“生成对应的音频吧”、“开始AI声音克隆吧”三个按键即可；使用声音克隆功能时，请先上传一段您喜欢的音频
\")\n", + " gr.HTML('''\n", + "
\n", + "

🌊🏞️🎶 - 江水东流急，滔滔无尽声。 明·顾璘\n", + "

\n", + "
\n", + " ''')\n", + "\n", + "\n", + "demo.queue().launch(show_error=True, debug=True, share=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cba1dd99-c797-4459-9a06-c294f30275de", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/__pycache__/commons.cpython-310.pyc b/__pycache__/commons.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88674edcc64cf71b49cd28129dbc4a475174d0b0 Binary files /dev/null and b/__pycache__/commons.cpython-310.pyc differ diff --git a/__pycache__/mel_processing.cpython-310.pyc b/__pycache__/mel_processing.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a5deab4fa5812bfebf834caf7f9822dfd0fca6e Binary files /dev/null and b/__pycache__/mel_processing.cpython-310.pyc differ diff --git a/__pycache__/models.cpython-310.pyc b/__pycache__/models.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f843f02f3427631f908743196e3623ddad148361 Binary files /dev/null and b/__pycache__/models.cpython-310.pyc differ diff --git a/__pycache__/modules.cpython-310.pyc b/__pycache__/modules.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d550a79a7f381077f72e5bd1e9135ad5224f7a2e Binary files /dev/null and b/__pycache__/modules.cpython-310.pyc differ diff --git a/__pycache__/tts_voice.cpython-310.pyc b/__pycache__/tts_voice.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..853a35398412db6e2691fad2c0f9c9f104d17d92 Binary files /dev/null and b/__pycache__/tts_voice.cpython-310.pyc differ diff --git a/__pycache__/utils.cpython-310.pyc b/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..72b1a7a95cdd17ba7f787608507c0cd4b12b2263 Binary files /dev/null and b/__pycache__/utils.cpython-310.pyc differ diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..926031c37dc21f6d341da9f050e35f39c87eddf0 --- /dev/null +++ b/app.py @@ -0,0 +1,608 @@ +import os, sys +import tempfile +import gradio as gr +from src.gradio_demo import SadTalker +# from src.utils.text2speech import TTSTalker +from huggingface_hub import snapshot_download + +import torch +import librosa +from scipy.io.wavfile import write +from transformers import WavLMModel + +import utils +from models import SynthesizerTrn +from mel_processing import mel_spectrogram_torch +from speaker_encoder.voice_encoder import SpeakerEncoder + +import time +from textwrap import dedent + +import mdtex2html +from loguru import logger +from transformers import AutoModel, AutoTokenizer + +from tts_voice import tts_order_voice +import edge_tts +import tempfile +import anyio + + +def get_source_image(image): + return image + +try: + import webui # in webui + in_webui = True +except: + in_webui = False + + +def toggle_audio_file(choice): + if choice == False: + return gr.update(visible=True), gr.update(visible=False) + else: + return gr.update(visible=False), gr.update(visible=True) + +def ref_video_fn(path_of_ref_video): + if path_of_ref_video is not None: + return gr.update(value=True) + else: + return gr.update(value=False) + +def download_model(): + REPO_ID = 'vinthony/SadTalker-V002rc' + snapshot_download(repo_id=REPO_ID, local_dir='./checkpoints', local_dir_use_symlinks=True) + +def sadtalker_demo(): + + download_model() + + sad_talker = SadTalker(lazy_load=True) + # tts_talker = TTSTalker() + +download_model() +sad_talker = SadTalker(lazy_load=True) + + +# ChatGLM2 & FreeVC + +''' +def get_wavlm(): + os.system('gdown https://drive.google.com/uc?id=12-cB34qCTvByWT-QtOcZaqwwO21FLSqU') + shutil.move('WavLM-Large.pt', 'wavlm') +''' + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +smodel = SpeakerEncoder('speaker_encoder/ckpt/pretrained_bak_5805000.pt') + +print("Loading FreeVC(24k)...") +hps = utils.get_hparams_from_file("configs/freevc-24.json") +freevc_24 = SynthesizerTrn( + hps.data.filter_length // 2 + 1, + hps.train.segment_size // hps.data.hop_length, + **hps.model).to(device) +_ = freevc_24.eval() +_ = utils.load_checkpoint("checkpoint/freevc-24.pth", freevc_24, None) + +print("Loading WavLM for content...") +cmodel = WavLMModel.from_pretrained("microsoft/wavlm-large").to(device) + +def convert(model, src, tgt): + with torch.no_grad(): + # tgt + wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate) + wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20) + if model == "FreeVC" or model == "FreeVC (24kHz)": + g_tgt = smodel.embed_utterance(wav_tgt) + g_tgt = torch.from_numpy(g_tgt).unsqueeze(0).to(device) + else: + wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).to(device) + mel_tgt = mel_spectrogram_torch( + wav_tgt, + hps.data.filter_length, + hps.data.n_mel_channels, + hps.data.sampling_rate, + hps.data.hop_length, + hps.data.win_length, + hps.data.mel_fmin, + hps.data.mel_fmax + ) + # src + wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate) + wav_src = torch.from_numpy(wav_src).unsqueeze(0).to(device) + c = cmodel(wav_src).last_hidden_state.transpose(1, 2).to(device) + # infer + if model == "FreeVC": + audio = freevc.infer(c, g=g_tgt) + elif model == "FreeVC-s": + audio = freevc_s.infer(c, mel=mel_tgt) + else: + audio = freevc_24.infer(c, g=g_tgt) + audio = audio[0][0].data.cpu().float().numpy() + if model == "FreeVC" or model == "FreeVC-s": + write("out.wav", hps.data.sampling_rate, audio) + else: + write("out.wav", 24000, audio) + out = "out.wav" + return out + +# GLM2 + +language_dict = tts_order_voice + +# fix timezone in Linux +os.environ["TZ"] = "Asia/Shanghai" +try: + time.tzset() # type: ignore # pylint: disable=no-member +except Exception: + # Windows + logger.warning("Windows, cant run time.tzset()") + +# model_name = "THUDM/chatglm2-6b" +model_name = "THUDM/chatglm2-6b-int4" + +RETRY_FLAG = False + +tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + +# model = AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda() + +# 4/8 bit +# model = AutoModel.from_pretrained("THUDM/chatglm2-6b", trust_remote_code=True).quantize(4).cuda() + +has_cuda = torch.cuda.is_available() + +# has_cuda = False # force cpu + +if has_cuda: + model_glm = ( + AutoModel.from_pretrained(model_name, trust_remote_code=True).cuda().half() + ) # 3.92G +else: + model_glm = AutoModel.from_pretrained( + model_name, trust_remote_code=True + ).float() # .float() .half().float() + +model_glm = model_glm.eval() + +_ = """Override Chatbot.postprocess""" + + +def postprocess(self, y): + if y is None: + return [] + for i, (message, response) in enumerate(y): + y[i] = ( + None if message is None else mdtex2html.convert((message)), + None if response is None else mdtex2html.convert(response), + ) + return y + + +gr.Chatbot.postprocess = postprocess + + +def parse_text(text): + """copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/""" + lines = text.split("\n") + lines = [line for line in lines if line != ""] + count = 0 + for i, line in enumerate(lines): + if "```" in line: + count += 1 + items = line.split("`") + if count % 2 == 1: + lines[i] = f'
'
+            else:
+                lines[i] = "
" + else: + if i > 0: + if count % 2 == 1: + line = line.replace("`", r"\`") + line = line.replace("<", "<") + line = line.replace(">", ">") + line = line.replace(" ", " ") + line = line.replace("*", "*") + line = line.replace("_", "_") + line = line.replace("-", "-") + line = line.replace(".", ".") + line = line.replace("!", "!") + line = line.replace("(", "(") + line = line.replace(")", ")") + line = line.replace("$", "$") + lines[i] = "
" + line + text = "".join(lines) + return text + + +def predict( + RETRY_FLAG, input, chatbot, max_length, top_p, temperature, history, past_key_values +): + try: + chatbot.append((parse_text(input), "")) + except Exception as exc: + logger.error(exc) + logger.debug(f"{chatbot=}") + _ = """ + if chatbot: + chatbot[-1] = (parse_text(input), str(exc)) + yield chatbot, history, past_key_values + # """ + yield chatbot, history, past_key_values + + for response, history, past_key_values in model_glm.stream_chat( + tokenizer, + input, + history, + past_key_values=past_key_values, + return_past_key_values=True, + max_length=max_length, + top_p=top_p, + temperature=temperature, + ): + chatbot[-1] = (parse_text(input), parse_text(response)) + # chatbot[-1][-1] = parse_text(response) + + yield chatbot, history, past_key_values, parse_text(response) + + +def trans_api(input, max_length=4096, top_p=0.8, temperature=0.2): + if max_length < 10: + max_length = 4096 + if top_p < 0.1 or top_p > 1: + top_p = 0.85 + if temperature <= 0 or temperature > 1: + temperature = 0.01 + try: + res, _ = model_glm.chat( + tokenizer, + input, + history=[], + past_key_values=None, + max_length=max_length, + top_p=top_p, + temperature=temperature, + ) + # logger.debug(f"{res=} \n{_=}") + except Exception as exc: + logger.error(f"{exc=}") + res = str(exc) + + return res + + +def reset_user_input(): + return gr.update(value="") + + +def reset_state(): + return [], [], None, "" + + +# Delete last turn +def delete_last_turn(chat, history): + if chat and history: + chat.pop(-1) + history.pop(-1) + return chat, history + + +# Regenerate response +def retry_last_answer( + user_input, chatbot, max_length, top_p, temperature, history, past_key_values +): + if chatbot and history: + # Removing the previous conversation from chat + chatbot.pop(-1) + # Setting up a flag to capture a retry + RETRY_FLAG = True + # Getting last message from user + user_input = history[-1][0] + # Removing bot response from the history + history.pop(-1) + + yield from predict( + RETRY_FLAG, # type: ignore + user_input, + chatbot, + max_length, + top_p, + temperature, + history, + past_key_values, + ) + +# print + +def print(text): + return text + +# TTS + +async def text_to_speech_edge(text, language_code): + voice = language_dict[language_code] + communicate = edge_tts.Communicate(text, voice) + with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp_file: + tmp_path = tmp_file.name + + await communicate.save(tmp_path) + + return tmp_path + + +with gr.Blocks(title="ChatGLM2-6B-int4", theme=gr.themes.Soft(text_size="sm"), analytics_enabled=False) as demo: + gr.HTML("
" + "

📺💕🎶 - ChatGLM2+声音克隆+视频对话：和喜欢的角色畅所欲言吧！

" + "
") + gr.Markdown("##
🥳 - ChatGLM2+FreeVC+SadTalker，为您打造沉浸式的视频对话体验，支持中英双语
") + gr.Markdown("##
🌊 - 更多精彩应用，尽在[滔滔AI](http://www.talktalkai.com)；滔滔AI，为爱滔滔！💕
") + gr.Markdown("###
⭐ - 如果您喜欢这个程序，欢迎给我的[GitHub项目](https://github.com/KevinWang676/ChatGLM2-Voice-Cloning)点赞支持！
") + + with gr.Tab("🍻 - ChatGLM2聊天区"): + with gr.Accordion("📒 相关信息", open=False): + _ = f""" ChatGLM2的可选参数信息： + * Low temperature: responses will be more deterministic and focused; High temperature: responses more creative. + * Suggested temperatures -- translation: up to 0.3; chatting: > 0.4 + * Top P controls dynamic vocabulary selection based on context.\n + 如果您想让ChatGLM2进行角色扮演并与之对话，请先输入恰当的提示词，如“请你扮演成动漫角色蜡笔小新并和我进行对话”；您也可以为ChatGLM2提供自定义的角色设定\n + 当您使用声音克隆功能时，请先在此程序的对应位置上传一段您喜欢的音频 + """ + gr.Markdown(dedent(_)) + chatbot = gr.Chatbot(height=300) + with gr.Row(): + with gr.Column(scale=4): + with gr.Column(scale=12): + user_input = gr.Textbox( + label="请在此处和GLM2聊天 (按回车键即可发送)", + placeholder="聊点什么吧", + ) + RETRY_FLAG = gr.Checkbox(value=False, visible=False) + with gr.Column(min_width=32, scale=1): + with gr.Row(): + submitBtn = gr.Button("开始和GLM2交流吧", variant="primary") + deleteBtn = gr.Button("删除最新一轮对话", variant="secondary") + retryBtn = gr.Button("重新生成最新一轮对话", variant="secondary") + + with gr.Accordion("🔧 更多设置", open=False): + with gr.Row(): + emptyBtn = gr.Button("清空所有聊天记录") + max_length = gr.Slider( + 0, + 32768, + value=8192, + step=1.0, + label="Maximum length", + interactive=True, + ) + top_p = gr.Slider( + 0, 1, value=0.85, step=0.01, label="Top P", interactive=True + ) + temperature = gr.Slider( + 0.01, 1, value=0.95, step=0.01, label="Temperature", interactive=True + ) + + + with gr.Row(): + test1 = gr.Textbox(label="GLM2的最新回答 (可编辑)", lines = 3) + with gr.Column(): + language = gr.Dropdown(choices=list(language_dict.keys()), value="普通话 (中国大陆)-Xiaoxiao-女", label="请选择文本对应的语言及您喜欢的说话人") + tts_btn = gr.Button("生成对应的音频吧", variant="primary") + output_audio = gr.Audio(type="filepath", label="为您生成的音频", interactive=False) + + tts_btn.click(text_to_speech_edge, inputs=[test1, language], outputs=[output_audio]) + + with gr.Row(): + model_choice = gr.Dropdown(choices=["FreeVC", "FreeVC-s", "FreeVC (24kHz)"], value="FreeVC (24kHz)", label="Model", visible=False) + audio1 = output_audio + audio2 = gr.Audio(label="请上传您喜欢的声音进行声音克隆", type='filepath') + clone_btn = gr.Button("开始AI声音克隆吧", variant="primary") + audio_cloned = gr.Audio(label="为您生成的专属声音克隆音频", type='filepath') + + clone_btn.click(convert, inputs=[model_choice, audio1, audio2], outputs=[audio_cloned]) + + history = gr.State([]) + past_key_values = gr.State(None) + + user_input.submit( + predict, + [ + RETRY_FLAG, + user_input, + chatbot, + max_length, + top_p, + temperature, + history, + past_key_values, + ], + [chatbot, history, past_key_values, test1], + show_progress="full", + ) + submitBtn.click( + predict, + [ + RETRY_FLAG, + user_input, + chatbot, + max_length, + top_p, + temperature, + history, + past_key_values, + ], + [chatbot, history, past_key_values, test1], + show_progress="full", + api_name="predict", + ) + submitBtn.click(reset_user_input, [], [user_input]) + + emptyBtn.click( + reset_state, outputs=[chatbot, history, past_key_values, test1], show_progress="full" + ) + + retryBtn.click( + retry_last_answer, + inputs=[ + user_input, + chatbot, + max_length, + top_p, + temperature, + history, + past_key_values, + ], + # outputs = [chatbot, history, last_user_message, user_message] + outputs=[chatbot, history, past_key_values, test1], + ) + deleteBtn.click(delete_last_turn, [chatbot, history], [chatbot, history]) + + with gr.Accordion("📔 提示词示例", open=False): + etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """ + examples = gr.Examples( + examples=[ + ["Explain the plot of Cinderella in a sentence."], + [ + "How long does it take to become proficient in French, and what are the best methods for retaining information?" + ], + ["What are some common mistakes to avoid when writing code?"], + ["Build a prompt to generate a beautiful portrait of a horse"], + ["Suggest four metaphors to describe the benefits of AI"], + ["Write a pop song about leaving home for the sandy beaches."], + ["Write a summary demonstrating my ability to tame lions"], + ["鲁迅和周树人什么关系"], + ["从前有一头牛，这头牛后面有什么？"], + ["正无穷大加一大于正无穷大吗？"], + ["正无穷大加正无穷大大于正无穷大吗？"], + ["-2的平方根等于什么"], + ["树上有5只鸟，猎人开枪打死了一只。树上还有几只鸟？"], + ["树上有11只鸟，猎人开枪打死了一只。树上还有几只鸟？提示：需考虑鸟可能受惊吓飞走。"], + ["鲁迅和周树人什么关系 用英文回答"], + ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"], + [f"{etext} 翻成中文，列出3个版本"], + [f"{etext} \n 翻成中文，保留原意，但使用文学性的语言。不要写解释。列出3个版本"], + ["js 判断一个数是不是质数"], + ["js 实现python 的 range(10)"], + ["js 实现python 的 [*(range(10)]"], + ["假定 1 + 2 = 4, 试求 7 + 8"], + ["Erkläre die Handlung von Cinderella in einem Satz."], + ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch"], + ], + inputs=[user_input], + examples_per_page=30, + ) + + with gr.Accordion("For Chat/Translation API", open=False, visible=False): + input_text = gr.Text() + tr_btn = gr.Button("Go", variant="primary") + out_text = gr.Text() + tr_btn.click( + trans_api, + [input_text, max_length, top_p, temperature], + out_text, + # show_progress="full", + api_name="tr", + ) + _ = """ + input_text.submit( + trans_api, + [input_text, max_length, top_p, temperature], + out_text, + show_progress="full", + api_name="tr1", + ) + # """ + with gr.Tab("📺 - 视频聊天区"): + with gr.Row().style(equal_height=False): + with gr.Column(variant='panel'): + with gr.Tabs(elem_id="sadtalker_source_image"): + with gr.TabItem('图片上传'): + with gr.Row(): + source_image = gr.Image(label="请上传一张您喜欢角色的图片", source="upload", type="filepath", elem_id="img2img_image").style(width=512) + + + with gr.Tabs(elem_id="sadtalker_driven_audio"): + with gr.TabItem('💡您还可以将视频下载到本地'): + + with gr.Row(): + driven_audio = audio_cloned + driven_audio_no = gr.Audio(label="Use IDLE mode, no audio is required", source="upload", type="filepath", visible=False) + + with gr.Column(): + use_idle_mode = gr.Checkbox(label="Use Idle Animation", visible=False) + length_of_audio = gr.Number(value=5, label="The length(seconds) of the generated video.", visible=False) + use_idle_mode.change(toggle_audio_file, inputs=use_idle_mode, outputs=[driven_audio, driven_audio_no]) # todo + + with gr.Row(): + ref_video = gr.Video(label="Reference Video", source="upload", type="filepath", elem_id="vidref", visible=False).style(width=512) + + with gr.Column(): + use_ref_video = gr.Checkbox(label="Use Reference Video", visible=False) + ref_info = gr.Radio(['pose', 'blink','pose+blink', 'all'], value='pose', label='Reference Video',info="How to borrow from reference Video?((fully transfer, aka, video driving mode))", visible=False) + + ref_video.change(ref_video_fn, inputs=ref_video, outputs=[use_ref_video]) # todo + + + with gr.Column(variant='panel'): + with gr.Tabs(elem_id="sadtalker_checkbox"): + with gr.TabItem('视频设置'): + with gr.Column(variant='panel'): + # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width + # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width + with gr.Row(): + pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0, visible=False) # + exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1, visible=False) # + blink_every = gr.Checkbox(label="use eye blink", value=True, visible=False) + + with gr.Row(): + size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model?", visible=False) # + preprocess_type = gr.Radio(['crop', 'full'], value='crop', label='是否聚焦角色面部', info="crop：视频会聚焦角色面部；full：视频会显示图片全貌") + + with gr.Row(): + is_still_mode = gr.Checkbox(label="静态模式 (开启静态模式，角色的面部动作会减少；默认开启)", value=True) + facerender = gr.Radio(['facevid2vid','pirender'], value='facevid2vid', label='facerender', info="which face render?", visible=False) + + with gr.Row(): + batch_size = gr.Slider(label="Batch size (数值越大，生成速度越快；若显卡性能好，可增大数值)", step=1, maximum=32, value=2) + enhancer = gr.Checkbox(label="GFPGAN as Face enhancer", value=True, visible=False) + + submit = gr.Button('开始视频聊天吧', elem_id="sadtalker_generate", variant='primary') + + with gr.Tabs(elem_id="sadtalker_genearted"): + gen_video = gr.Video(label="为您生成的专属视频", format="mp4").style(width=256) + + + + submit.click( + fn=sad_talker.test, + inputs=[source_image, + driven_audio, + preprocess_type, + is_still_mode, + enhancer, + batch_size, + size_of_image, + pose_style, + facerender, + exp_weight, + use_ref_video, + ref_video, + ref_info, + use_idle_mode, + length_of_audio, + blink_every + ], + outputs=[gen_video] + ) + gr.Markdown("###
注意❗：请不要生成会对个人以及组织造成侵害的内容，此程序仅供科研、学习及个人娱乐使用。
") + gr.Markdown("
💡- 如何使用此程序：输入您对ChatGLM的提问后，依次点击“开始和GLM2交流吧”、“生成对应的音频吧”、“开始AI声音克隆吧”、“开始视频聊天吧”三个按键即可；使用声音克隆功能时，请先上传一段您喜欢的音频
") + gr.HTML(''' +
+

🌊🏞️🎶 - 江水东流急，滔滔无尽声。 明·顾璘 +

+
+ ''') + + +demo.queue().launch(show_error=True, debug=True) diff --git a/app_sadtalker.py b/app_sadtalker.py new file mode 100644 index 0000000000000000000000000000000000000000..900ad8f68d937e5b9de42cdb69d7218d924f70a1 --- /dev/null +++ b/app_sadtalker.py @@ -0,0 +1,111 @@ +import os, sys +import gradio as gr +from src.gradio_demo import SadTalker + + +try: + import webui # in webui + in_webui = True +except: + in_webui = False + + +def toggle_audio_file(choice): + if choice == False: + return gr.update(visible=True), gr.update(visible=False) + else: + return gr.update(visible=False), gr.update(visible=True) + +def ref_video_fn(path_of_ref_video): + if path_of_ref_video is not None: + return gr.update(value=True) + else: + return gr.update(value=False) + +def sadtalker_demo(checkpoint_path='checkpoints', config_path='src/config', warpfn=None): + + sad_talker = SadTalker(checkpoint_path, config_path, lazy_load=True) + + with gr.Blocks(analytics_enabled=False) as sadtalker_interface: + gr.Markdown("") + + with gr.Row().style(equal_height=False): + with gr.Column(variant='panel'): + with gr.Tabs(elem_id="sadtalker_source_image"): + with gr.TabItem('Upload image'): + with gr.Row(): + source_image = gr.Image(label="Source image", source="upload", type="filepath", elem_id="img2img_image").style(width=512) + + with gr.Tabs(elem_id="sadtalker_driven_audio"): + with gr.TabItem('Upload OR TTS'): + with gr.Column(variant='panel'): + driven_audio = gr.Audio(label="Input audio", source="upload", type="filepath") + + if sys.platform != 'win32' and not in_webui: + from src.utils.text2speech import TTSTalker + tts_talker = TTSTalker() + with gr.Column(variant='panel'): + input_text = gr.Textbox(label="Generating audio from text", lines=5, placeholder="please enter some text here, we genreate the audio from text using @Coqui.ai TTS.") + tts = gr.Button('Generate audio',elem_id="sadtalker_audio_generate", variant='primary') + tts.click(fn=tts_talker.test, inputs=[input_text], outputs=[driven_audio]) + + with gr.Column(variant='panel'): + with gr.Tabs(elem_id="sadtalker_checkbox"): + with gr.TabItem('Settings'): + gr.Markdown("need help? please visit our [best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md) for more detials") + with gr.Column(variant='panel'): + # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width + # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width + pose_style = gr.Slider(minimum=0, maximum=46, step=1, label="Pose style", value=0) # + size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model?") # + preprocess_type = gr.Radio(['crop', 'resize','full', 'extcrop', 'extfull'], value='crop', label='preprocess', info="How to handle input image?") + is_still_mode = gr.Checkbox(label="Still Mode (fewer hand motion, works with preprocess `full`)") + batch_size = gr.Slider(label="batch size in generation", step=1, maximum=10, value=2) + enhancer = gr.Checkbox(label="GFPGAN as Face enhancer") + submit = gr.Button('Generate', elem_id="sadtalker_generate", variant='primary') + + with gr.Tabs(elem_id="sadtalker_genearted"): + gen_video = gr.Video(label="Generated video", format="mp4").style(width=256) + + if warpfn: + submit.click( + fn=warpfn(sad_talker.test), + inputs=[source_image, + driven_audio, + preprocess_type, + is_still_mode, + enhancer, + batch_size, + size_of_image, + pose_style + ], + outputs=[gen_video] + ) + else: + submit.click( + fn=sad_talker.test, + inputs=[source_image, + driven_audio, + preprocess_type, + is_still_mode, + enhancer, + batch_size, + size_of_image, + pose_style + ], + outputs=[gen_video] + ) + + return sadtalker_interface + + +if __name__ == "__main__": + + demo = sadtalker_demo() + demo.queue() + demo.launch(share=True) + + diff --git a/checkpoint/__init__.py b/checkpoint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/checkpoint/freevc-24.pth b/checkpoint/freevc-24.pth new file mode 100644 index 0000000000000000000000000000000000000000..d256c31ffd327d6002980112b891f0db5f7ae849 --- /dev/null +++ b/checkpoint/freevc-24.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b39a86fefbc9ec6e30be8d26ee2a6aa5ffe6d235f6ab15773d01cdf348e5b20 +size 472644351 diff --git a/checkpoints/SadTalker_V0.0.2_256.safetensors b/checkpoints/SadTalker_V0.0.2_256.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1d0eb9787332ff6c6a603c0e79ebada49010270a --- /dev/null +++ b/checkpoints/SadTalker_V0.0.2_256.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c211f5d6de003516bf1bbda9f47049a4c9c99133b1ab565c6961e5af16477bff +size 725066984 diff --git a/checkpoints/SadTalker_V0.0.2_512.safetensors b/checkpoints/SadTalker_V0.0.2_512.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..356c229c4a1424e157e4b22686ba72fd3daa8e81 --- /dev/null +++ b/checkpoints/SadTalker_V0.0.2_512.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e063f7ff5258240bdb0f7690783a7b1374e6a4a81ce8fa33456f4cd49694340 +size 725066984 diff --git a/checkpoints/mapping_00109-model.pth.tar b/checkpoints/mapping_00109-model.pth.tar new file mode 100644 index 0000000000000000000000000000000000000000..009c3190f5d903c56a2fb0a085d605dc782a83c9 --- /dev/null +++ b/checkpoints/mapping_00109-model.pth.tar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84a8642468a3fcfdd9ab6be955267043116c2bec2284686a5262f1eaf017f64c +size 155779231 diff --git a/checkpoints/mapping_00229-model.pth.tar b/checkpoints/mapping_00229-model.pth.tar new file mode 100644 index 0000000000000000000000000000000000000000..6400233ae3fa5ff9426800ef761fd6c830bc0cd7 --- /dev/null +++ b/checkpoints/mapping_00229-model.pth.tar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62a1e06006cc963220f6477438518ed86e9788226c62ae382ddc42fbcefb83f1 +size 155521183 diff --git a/cog.yaml b/cog.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05bcbd581b9ee3a07babaa971e3deedcd16b161d --- /dev/null +++ b/cog.yaml @@ -0,0 +1,35 @@ +build: + gpu: true + cuda: "11.3" + python_version: "3.8" + system_packages: + - "ffmpeg" + - "libgl1-mesa-glx" + - "libglib2.0-0" + python_packages: + - "torch==1.12.1" + - "torchvision==0.13.1" + - "torchaudio==0.12.1" + - "joblib==1.1.0" + - "scikit-image==0.19.3" + - "basicsr==1.4.2" + - "facexlib==0.3.0" + - "resampy==0.3.1" + - "pydub==0.25.1" + - "scipy==1.10.1" + - "kornia==0.6.8" + - "face_alignment==1.3.5" + - "imageio==2.19.3" + - "imageio-ffmpeg==0.4.7" + - "librosa==0.9.2" # + - "tqdm==4.65.0" + - "yacs==0.1.8" + - "gfpgan==1.3.8" + - "dlib-bin==19.24.1" + - "av==10.0.0" + - "trimesh==3.9.20" + run: + - mkdir -p /root/.cache/torch/hub/checkpoints/ && wget --output-document "/root/.cache/torch/hub/checkpoints/s3fd-619a316812.pth" "https://www.adrianbulat.com/downloads/python-fan/s3fd-619a316812.pth" + - mkdir -p /root/.cache/torch/hub/checkpoints/ && wget --output-document "/root/.cache/torch/hub/checkpoints/2DFAN4-cd938726ad.zip" "https://www.adrianbulat.com/downloads/python-fan/2DFAN4-cd938726ad.zip" + +predict: "predict.py:Predictor" diff --git a/commons.py b/commons.py new file mode 100644 index 0000000000000000000000000000000000000000..fc384912618494475bda9d68fa76530f4fe2a27b --- /dev/null +++ b/commons.py @@ -0,0 +1,171 @@ +import math +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + + +def init_weights(m, mean=0.0, std=0.01): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + m.weight.data.normal_(mean, std) + + +def get_padding(kernel_size, dilation=1): + return int((kernel_size*dilation - dilation)/2) + + +def convert_pad_shape(pad_shape): + l = pad_shape[::-1] + pad_shape = [item for sublist in l for item in sublist] + return pad_shape + + +def intersperse(lst, item): + result = [item] * (len(lst) * 2 + 1) + result[1::2] = lst + return result + + +def kl_divergence(m_p, logs_p, m_q, logs_q): + """KL(P||Q)""" + kl = (logs_q - logs_p) - 0.5 + kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q) + return kl + + +def rand_gumbel(shape): + """Sample from the Gumbel distribution, protect from overflows.""" + uniform_samples = torch.rand(shape) * 0.99998 + 0.00001 + return -torch.log(-torch.log(uniform_samples)) + + +def rand_gumbel_like(x): + g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device) + return g + + +def slice_segments(x, ids_str, segment_size=4): + ret = torch.zeros_like(x[:, :, :segment_size]) + for i in range(x.size(0)): + idx_str = ids_str[i] + idx_end = idx_str + segment_size + ret[i] = x[i, :, idx_str:idx_end] + return ret + + +def rand_slice_segments(x, x_lengths=None, segment_size=4): + b, d, t = x.size() + if x_lengths is None: + x_lengths = t + ids_str_max = x_lengths - segment_size + 1 + ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long) + ret = slice_segments(x, ids_str, segment_size) + return ret, ids_str + + +def rand_spec_segments(x, x_lengths=None, segment_size=4): + b, d, t = x.size() + if x_lengths is None: + x_lengths = t + ids_str_max = x_lengths - segment_size + ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long) + ret = slice_segments(x, ids_str, segment_size) + return ret, ids_str + + +def get_timing_signal_1d( + length, channels, min_timescale=1.0, max_timescale=1.0e4): + position = torch.arange(length, dtype=torch.float) + num_timescales = channels // 2 + log_timescale_increment = ( + math.log(float(max_timescale) / float(min_timescale)) / + (num_timescales - 1)) + inv_timescales = min_timescale * torch.exp( + torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment) + scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1) + signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0) + signal = F.pad(signal, [0, 0, 0, channels % 2]) + signal = signal.view(1, channels, length) + return signal + + +def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4): + b, channels, length = x.size() + signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) + return x + signal.to(dtype=x.dtype, device=x.device) + + +def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1): + b, channels, length = x.size() + signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) + return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis) + + +def subsequent_mask(length): + mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0) + return mask + + +@torch.jit.script +def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): + n_channels_int = n_channels[0] + in_act = input_a + input_b + t_act = torch.tanh(in_act[:, :n_channels_int, :]) + s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) + acts = t_act * s_act + return acts + + +def convert_pad_shape(pad_shape): + l = pad_shape[::-1] + pad_shape = [item for sublist in l for item in sublist] + return pad_shape + + +def shift_1d(x): + x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1] + return x + + +def sequence_mask(length, max_length=None): + if max_length is None: + max_length = length.max() + x = torch.arange(max_length, dtype=length.dtype, device=length.device) + return x.unsqueeze(0) < length.unsqueeze(1) + + +def generate_path(duration, mask): + """ + duration: [b, 1, t_x] + mask: [b, 1, t_y, t_x] + """ + device = duration.device + + b, _, t_y, t_x = mask.shape + cum_duration = torch.cumsum(duration, -1) + + cum_duration_flat = cum_duration.view(b * t_x) + path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype) + path = path.view(b, t_x, t_y) + path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1] + path = path.unsqueeze(1).transpose(2,3) * mask + return path + + +def clip_grad_value_(parameters, clip_value, norm_type=2): + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = list(filter(lambda p: p.grad is not None, parameters)) + norm_type = float(norm_type) + if clip_value is not None: + clip_value = float(clip_value) + + total_norm = 0 + for p in parameters: + param_norm = p.grad.data.norm(norm_type) + total_norm += param_norm.item() ** norm_type + if clip_value is not None: + p.grad.data.clamp_(min=-clip_value, max=clip_value) + total_norm = total_norm ** (1. / norm_type) + return total_norm diff --git a/configs/.ipynb_checkpoints/freevc-24-checkpoint.json b/configs/.ipynb_checkpoints/freevc-24-checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..8454a2500a815ae82ea2db4de1ccd54d8c101887 --- /dev/null +++ b/configs/.ipynb_checkpoints/freevc-24-checkpoint.json @@ -0,0 +1,54 @@ +{ + "train": { + "log_interval": 200, + "eval_interval": 10000, + "seed": 1234, + "epochs": 10000, + "learning_rate": 2e-4, + "betas": [0.8, 0.99], + "eps": 1e-9, + "batch_size": 64, + "fp16_run": false, + "lr_decay": 0.999875, + "segment_size": 8640, + "init_lr_ratio": 1, + "warmup_epochs": 0, + "c_mel": 45, + "c_kl": 1.0, + "use_sr": true, + "max_speclen": 128, + "port": "8008" + }, + "data": { + "training_files":"filelists/train.txt", + "validation_files":"filelists/val.txt", + "max_wav_value": 32768.0, + "sampling_rate": 16000, + "filter_length": 1280, + "hop_length": 320, + "win_length": 1280, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": null + }, + "model": { + "inter_channels": 192, + "hidden_channels": 192, + "filter_channels": 768, + "n_heads": 2, + "n_layers": 6, + "kernel_size": 3, + "p_dropout": 0.1, + "resblock": "1", + "resblock_kernel_sizes": [3,7,11], + "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], + "upsample_rates": [10,6,4,2], + "upsample_initial_channel": 512, + "upsample_kernel_sizes": [16,16,4,4], + "n_layers_q": 3, + "use_spectral_norm": false, + "gin_channels": 256, + "ssl_dim": 1024, + "use_spk": true + } +} diff --git a/configs/freevc-24.json b/configs/freevc-24.json new file mode 100644 index 0000000000000000000000000000000000000000..8454a2500a815ae82ea2db4de1ccd54d8c101887 --- /dev/null +++ b/configs/freevc-24.json @@ -0,0 +1,54 @@ +{ + "train": { + "log_interval": 200, + "eval_interval": 10000, + "seed": 1234, + "epochs": 10000, + "learning_rate": 2e-4, + "betas": [0.8, 0.99], + "eps": 1e-9, + "batch_size": 64, + "fp16_run": false, + "lr_decay": 0.999875, + "segment_size": 8640, + "init_lr_ratio": 1, + "warmup_epochs": 0, + "c_mel": 45, + "c_kl": 1.0, + "use_sr": true, + "max_speclen": 128, + "port": "8008" + }, + "data": { + "training_files":"filelists/train.txt", + "validation_files":"filelists/val.txt", + "max_wav_value": 32768.0, + "sampling_rate": 16000, + "filter_length": 1280, + "hop_length": 320, + "win_length": 1280, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": null + }, + "model": { + "inter_channels": 192, + "hidden_channels": 192, + "filter_channels": 768, + "n_heads": 2, + "n_layers": 6, + "kernel_size": 3, + "p_dropout": 0.1, + "resblock": "1", + "resblock_kernel_sizes": [3,7,11], + "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], + "upsample_rates": [10,6,4,2], + "upsample_initial_channel": 512, + "upsample_kernel_sizes": [16,16,4,4], + "n_layers_q": 3, + "use_spectral_norm": false, + "gin_channels": 256, + "ssl_dim": 1024, + "use_spk": true + } +} diff --git a/docs/FAQ.md b/docs/FAQ.md new file mode 100644 index 0000000000000000000000000000000000000000..6451a226c2a80cd107d12b35f09e888086ec407d --- /dev/null +++ b/docs/FAQ.md @@ -0,0 +1,46 @@ + +## Frequency Asked Question + +**Q: `ffmpeg` is not recognized as an internal or external command** + +In Linux, you can install the ffmpeg via `conda install ffmpeg`. Or on Mac OS X, try to install ffmpeg via `brew install ffmpeg`. On windows, make sure you have `ffmpeg` in the `%PATH%` as suggested in [#54](https://github.com/Winfredy/SadTalker/issues/54), then, following [this](https://www.geeksforgeeks.org/how-to-install-ffmpeg-on-windows/) installation to install `ffmpeg`. + +**Q: Running Requirments.** + +Please refer to the discussion here: https://github.com/Winfredy/SadTalker/issues/124#issuecomment-1508113989 + + +**Q: ModuleNotFoundError: No module named 'ai'** + +please check the checkpoint's size of the `epoch_20.pth`. (https://github.com/Winfredy/SadTalker/issues/167, https://github.com/Winfredy/SadTalker/issues/113) + +**Q: Illegal Hardware Error: Mac M1** + +please reinstall the `dlib` by `pip install dlib` individually. (https://github.com/Winfredy/SadTalker/issues/129, https://github.com/Winfredy/SadTalker/issues/109) + + +**Q: FileNotFoundError: [Errno 2] No such file or directory: checkpoints\BFM_Fitting\similarity_Lm3D_all.mat** + +Make sure you have downloaded the checkpoints and gfpgan as [here](https://github.com/Winfredy/SadTalker#-2-download-trained-models) and placed them in the right place. + +**Q: RuntimeError: unexpected EOF, expected 237192 more bytes. The file might be corrupted.** + +The files are not automatically downloaded. Please update the code and download the gfpgan folders as [here](https://github.com/Winfredy/SadTalker#-2-download-trained-models). + +**Q: CUDA out of memory error** + +please refer to https://stackoverflow.com/questions/73747731/runtimeerror-cuda-out-of-memory-how-setting-max-split-size-mb + +``` +# windows +set PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128 +python inference.py ... + +# linux +export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128 +python inference.py ... +``` + +**Q: Error while decoding stream #0:0: Invalid data found when processing input [mp3float @ 0000015037628c00] Header missing** + +Our method only support wav or mp3 files as input, please make sure the feeded audios are in these formats. diff --git a/docs/best_practice.md b/docs/best_practice.md new file mode 100644 index 0000000000000000000000000000000000000000..3a738b217da216d41ff952593d8800851a7d2718 --- /dev/null +++ b/docs/best_practice.md @@ -0,0 +1,94 @@ +# Best Practice and Tips for configuration + +> Our model only works on REAL person's photo or the portrait image similar to REAL person. The anime talking head genreation method will be released in future. + +Advanced confiurations for `inference.py`: + +| Name | Configuration | default | Explaination | +|:------------- |:------------- |:----- | :------------- | +| Enhance Mode | `--enhancer` | None | Using `gfpgan` or `RestoreFormer` to enhance the generated face via face restoration network +| Background Enhancer | `--background_enhancer` | None | Using `realesrgan` to enhance the full video. +| Still Mode | ` --still` | False | Using the same pose parameters as the original image, fewer head motion. +| Expressive Mode | `--expression_scale` | 1.0 | a larger value will make the expression motion stronger. +| save path | `--result_dir` |`./results` | The file will be save in the newer location. +| preprocess | `--preprocess` | `crop` | Run and produce the results in the croped input image. Other choices: `resize`, where the images will be resized to the specific resolution. `full` Run the full image animation, use with `--still` to get better results. +| ref Mode (eye) | `--ref_eyeblink` | None | A video path, where we borrow the eyeblink from this reference video to provide more natural eyebrow movement. +| ref Mode (pose) | `--ref_pose` | None | A video path, where we borrow the pose from the head reference video. +| 3D Mode | `--face3dvis` | False | Need additional installation. More details to generate the 3d face can be founded [here](docs/face3d.md). +| free-view Mode | `--input_yaw`,
`--input_pitch`,
`--input_roll` | None | Genearting novel view or free-view 4D talking head from a single image. More details can be founded [here](https://github.com/Winfredy/SadTalker#generating-4d-free-view-talking-examples-from-audio-and-a-single-image). + + +### About `--preprocess` + +Our method automatically handle the input images via `crop`, `resize` and `full`. + + In `crop` mode, we only generate the croped image via the facial keypoints and generated the facial anime avator. The animation of both expression and head pose are realistic. + +> still mode will stop the eyeblink and head pose movement. + +| [input image @bagbag1815](https://twitter.com/bagbag1815/status/1642754319094108161) | crop | crop w/still | +|:--------------------: |:--------------------: | :----: | +| | ![full_body_2](example_crop.gif) | ![full_body_2](example_crop_still.gif) | + + + In `resize` mode, we resize the whole images to generate the fully talking head video. Thus, an image similar to the ID photo can be produced. ⚠️ It will produce bad results for full person images. + + + + +| | | +|:--------------------: |:--------------------: | +| ❌ not suitable for resize mode | ✅ good for resize mode | +| | | + +In `full` mode, our model will automatically process the croped region and paste back to the original image. Remember to use `--still` to keep the original head pose. + +| input | `--still` | `--still` & `enhancer` | +|:--------------------: |:--------------------: | :--:| +| | | + + +### About `--enhancer` + +For better facial quality, we intergate [gfpgan](https://github.com/TencentARC/GFPGAN) and [real-esrgan](https://github.com/xinntao/Real-ESRGAN) for different purpose. Just adding `--enhancer ` or `--background_enhancer ` for the enhancement of the face and the full image. + +```bash +# make sure above packages are available: +pip install gfpgan +pip install realesrgan +``` + +### About `--face3dvis` + +This flag indicate that we can generated the 3d-rendered face and it's 3d facial landmarks. More details can be founded [here](face3d.md). + +| Input | Animated 3d face | +|:-------------: | :-------------: | +| | | + +> Kindly ensure to activate the audio as the default audio playing is incompatible with GitHub. + + + +#### reference eye-link mode. + +| Input, w/ reference video , reference video | +|:-------------: | +| ![free_view](using_ref_video.gif)| +| If the reference video is shorter than the input audio, we will loop the reference video . + + + +#### Generating 4D free-view talking examples from audio and a single image + +We use `input_yaw`, `input_pitch`, `input_roll` to control head pose. For example, `--input_yaw -20 30 10` means the input head yaw degree changes from -20 to 30 and then changes from 30 to 10. +```bash +python inference.py --driven_audio \ + --source_image \ + --result_dir \ + --input_yaw -20 30 10 +``` + +| Results, Free-view results, Novel view results | +|:-------------: | +| ![free_view](free_view_result.gif)| diff --git a/docs/changlelog.md b/docs/changlelog.md new file mode 100644 index 0000000000000000000000000000000000000000..03d13851386e7acede4bdf54258bca86448ff463 --- /dev/null +++ b/docs/changlelog.md @@ -0,0 +1,29 @@ +## changelogs + + +- __[2023.04.06]__: stable-diffiusion webui extension is release. + +- __[2023.04.03]__: Enable TTS in huggingface and gradio local demo. + +- __[2023.03.30]__: Launch beta version of the full body mode. + +- __[2023.03.30]__: Launch new feature: through using reference videos, our algorithm can generate videos with more natural eye blinking and some eyebrow movement. + +- __[2023.03.29]__: `resize mode` is online by `python infererence.py --preprocess resize`! Where we can produce a larger crop of the image as discussed in https://github.com/Winfredy/SadTalker/issues/35. + +- __[2023.03.29]__: local gradio demo is online! `python app.py` to start the demo. New `requirments.txt` is used to avoid the bugs in `librosa`. + +- __[2023.03.28]__: Online demo is launched in [![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/vinthony/SadTalker), thanks AK! + +- __[2023.03.22]__: Launch new feature: generating the 3d face animation from a single image. New applications about it will be updated. + +- __[2023.03.22]__: Launch new feature: `still mode`, where only a small head pose will be produced via `python inference.py --still`. + +- __[2023.03.18]__: Support `expression intensity`, now you can change the intensity of the generated motion: `python inference.py --expression_scale 1.3 (some value > 1)`. + +- __[2023.03.18]__: Reconfig the data folders, now you can download the checkpoint automatically using `bash scripts/download_models.sh`. +- __[2023.03.18]__: We have offically integrate the [GFPGAN](https://github.com/TencentARC/GFPGAN) for face enhancement, using `python inference.py --enhancer gfpgan` for better visualization performance. +- __[2023.03.14]__: Specify the version of package `joblib` to remove the errors in using `librosa`, [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Winfredy/SadTalker/blob/main/quick_demo.ipynb) is online! +- __[2023.03.06]__: Solve some bugs in code and errors in installation +- __[2023.03.03]__: Release the test code for audio-driven single image animation! +- __[2023.02.28]__: SadTalker has been accepted by CVPR 2023! diff --git a/docs/example_crop.gif b/docs/example_crop.gif new file mode 100644 index 0000000000000000000000000000000000000000..9dd8ba03098d9e061f8b7018f2e24830f3e9e1ec --- /dev/null +++ b/docs/example_crop.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da08306e3e6355928887e74057ee4221f9d877d8536341d907e29fe35e078b45 +size 1548875 diff --git a/docs/example_crop_still.gif b/docs/example_crop_still.gif new file mode 100644 index 0000000000000000000000000000000000000000..860cb27505db75652469761d917e97bc49dab4e1 --- /dev/null +++ b/docs/example_crop_still.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:667c7531ed0a4d97a3ca9b15f79eea655b93dc40eda94498aa43b9e6a48c49aa +size 1247001 diff --git a/docs/example_full.gif b/docs/example_full.gif new file mode 100644 index 0000000000000000000000000000000000000000..b4854bc133e1aa7c2c107e0636602f84b1b48283 --- /dev/null +++ b/docs/example_full.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d1a2b8f5ed7b942a8625a5767828c1bc47568165a187079fbbb8492ed57301b +size 1460241 diff --git a/docs/example_full_crop.gif b/docs/example_full_crop.gif new file mode 100644 index 0000000000000000000000000000000000000000..e59c7d141bc42ce71b86e0dcb9129dffad543628 Binary files /dev/null and b/docs/example_full_crop.gif differ diff --git a/docs/example_full_enhanced.gif b/docs/example_full_enhanced.gif new file mode 100644 index 0000000000000000000000000000000000000000..0db44f7d8fb45da572ab62572afcaf9cf55b61ff --- /dev/null +++ b/docs/example_full_enhanced.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:906ca893e72854021c7715f784dc3fe219bbe67b73ff461e6ba8374f0d3b4712 +size 5782360 diff --git a/docs/face3d.md b/docs/face3d.md new file mode 100644 index 0000000000000000000000000000000000000000..b5a3f00b59834fc2de87baa08ed0ded6131ba647 --- /dev/null +++ b/docs/face3d.md @@ -0,0 +1,48 @@ +## 3D Face visualization + +We use pytorch3d to visualize the produced 3d face from a single image. + +Since it is not easy to install, we produce a new install guidence here: + +```bash +git clone https://github.com/Winfredy/SadTalker.git +cd SadTalker +conda create -n sadtalker3d python=3.8 +source activate sadtalker3d + +conda install ffmpeg +conda install -c fvcore -c iopath -c conda-forge fvcore iopath +conda install libgcc gmp + +pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu113 + +# insintall pytorch3d +pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py38_cu113_pyt1110/download.html + +pip install -r requirements3d.txt + +### install gpfgan for enhancer +pip install git+https://github.com/TencentARC/GFPGAN + + +### when occurs gcc version problem `from pytorch import _C` from pytorch3d, add the anaconda path to LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/$YOUR_ANACONDA_PATH/lib/ + +``` + + +Then, generating the result via: + +```bash + + +python inference.py --driven_audio \ + --source_image \ + --result_dir \ + --face3dvis + +``` + +Then, the results will be given in the folders with the file name of `face3d.mp4`. + +More applications about 3d face will be released. \ No newline at end of file diff --git a/docs/free_view_result.gif b/docs/free_view_result.gif new file mode 100644 index 0000000000000000000000000000000000000000..44c60e1c6495bfe250da45437fee16ed807d4df3 --- /dev/null +++ b/docs/free_view_result.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:035a7fba6800964254728f82fec47fe5c91458183e19a7506dd54d89940af40f +size 5611296 diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000000000000000000000000000000000000..31d1f9183e17c7f642fbf5d9c804bdf8fc623790 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,47 @@ + +### Mac (Tested on M1 Mac OS 13.3) + +``` +git clone https://github.com/Winfredy/SadTalker.git + +cd SadTalker + +conda create -n sadtalker python=3.8 + +conda activate sadtalker + +# install pytorch 2.0 +pip install torch torchvision torchaudio + +conda install ffmpeg + +pip install -r requirements.txt + +pip install dlib # mac need to install the original dlib. + +``` + + + +### Windows Native + +- Make sure you have `ffmpeg` in the `%PATH%` as suggested in [#54](https://github.com/Winfredy/SadTalker/issues/54), following [this](https://www.geeksforgeeks.org/how-to-install-ffmpeg-on-windows/) installation to install `ffmpeg`. + + +### Windows WSL +- Make sure the environment: `export LD_LIBRARY_PATH=/usr/lib/wsl/lib:$LD_LIBRARY_PATH` + + +### Docker installnation + +A dockerfile are also provided by [@thegenerativegeneration](https://github.com/thegenerativegeneration) in [docker hub](https://hub.docker.com/repository/docker/wawa9000/sadtalker), which can be used directly as: + +```bash +docker run --gpus "all" --rm -v $(pwd):/host_dir wawa9000/sadtalker \ + --driven_audio /host_dir/deyu.wav \ + --source_image /host_dir/image.jpg \ + --expression_scale 1.0 \ + --still \ + --result_dir /host_dir +``` + diff --git a/docs/resize_good.gif b/docs/resize_good.gif new file mode 100644 index 0000000000000000000000000000000000000000..a4bf5f565ea32db0468f42de327352593c44168d --- /dev/null +++ b/docs/resize_good.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ada6f2ea847e71c2a963882fd83f6b54193f4fe7c402f9f20698632b15bbdc0c +size 1728710 diff --git a/docs/resize_no.gif b/docs/resize_no.gif new file mode 100644 index 0000000000000000000000000000000000000000..34bf6b56597afa8b507732dae4f6dfa7255a0794 --- /dev/null +++ b/docs/resize_no.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7702f0be5c87c8977bf3c4a73ea4d27e90d0a5a3015816abb880cfd8f75c6ac +size 2138585 diff --git a/docs/sadtalker_logo.png b/docs/sadtalker_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..107aebfab5c2cdf842d7fc29ceed657e8c83ad56 Binary files /dev/null and b/docs/sadtalker_logo.png differ diff --git a/docs/using_ref_video.gif b/docs/using_ref_video.gif new file mode 100644 index 0000000000000000000000000000000000000000..e42dcd362b49004f5a59d54bddd6a33842e64029 --- /dev/null +++ b/docs/using_ref_video.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bb68ae077a6c009e7d30a36d34c30bf1310a073ab3c7d9cc1b5c9abe285e888 +size 8111572 diff --git a/docs/webui_extension.md b/docs/webui_extension.md new file mode 100644 index 0000000000000000000000000000000000000000..6e272e2d31ee912b59c7f7a87cbac7385b672827 --- /dev/null +++ b/docs/webui_extension.md @@ -0,0 +1,50 @@ + +## Run SadTalker as a Stable Diffusion WebUI Extension. + +1. Installing the lastest version of [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) and install the sadtalker via `extension`. +image + +2. Download the checkpoints manually, for Linux and Mac: + + ```bash + + cd SOMEWHERE_YOU_LIKE + + bash <(wget -qO- https://raw.githubusercontent.com/Winfredy/SadTalker/main/scripts/download_models.sh) + ``` + + For windows, you can download all the checkpoints from [google drive](https://drive.google.com/drive/folders/1Wd88VDoLhVzYsQ30_qDVluQr_Xm46yHT?usp=sharing) or [百度云盘](https://pan.baidu.com/s/1nXuVNd0exUl37ISwWqbFGA?pwd=sadt) 提取码: sadt. + +3.1. options 1: put the checkpoint in `stable-diffusion-webui/models/SadTalker` or `stable-diffusion-webui/extensions/SadTalker/checkpoints/`, the checkpoints will be detected automatically. + +3.2. Options 2: Set the path of `SADTALKTER_CHECKPOINTS` in `webui_user.sh`(linux) or `webui_user.bat`(windows) by: + + > only works if you are directly starting webui from `webui_user.sh` or `webui_user.bat`. + + ```bash + # windows (webui_user.bat) + set SADTALKER_CHECKPOINTS=D:\SadTalker\checkpoints + + # linux (webui_user.sh) + export SADTALKER_CHECKPOINTS=/path/to/SadTalker/checkpoints + ``` + +4. Then, starting the webui via `webui.sh or webui_user.sh(linux)` or `webui_user.bat(windows)` or any other methods, the SadTalker can be used in stable-diffusion-webui directly. + + image + +## Questsions + +1. if you are running on CPU, you need to specific `--disable-safe-unpickle` in `webui_user.sh` or `webui_user.bat`. + + ```bash + # windows (webui_user.bat) + set COMMANDLINE_ARGS="--disable-safe-unpickle" + + # linux (webui_user.sh) + export COMMANDLINE_ARGS="--disable-safe-unpickle" + ``` + + + +(Some [important discussion](https://github.com/Winfredy/SadTalker/issues/78) if you are unable to use `full` mode). diff --git a/examples/driven_audio/RD_Radio31_000.wav b/examples/driven_audio/RD_Radio31_000.wav new file mode 100644 index 0000000000000000000000000000000000000000..3b04940a0bff7481179c29bfc47553d9c4224bcf Binary files /dev/null and b/examples/driven_audio/RD_Radio31_000.wav differ diff --git a/examples/driven_audio/RD_Radio34_002.wav b/examples/driven_audio/RD_Radio34_002.wav new file mode 100644 index 0000000000000000000000000000000000000000..6813e812a8d1c57cb2f02eee3fece68a0864d96e Binary files /dev/null and b/examples/driven_audio/RD_Radio34_002.wav differ diff --git a/examples/driven_audio/RD_Radio36_000.wav b/examples/driven_audio/RD_Radio36_000.wav new file mode 100644 index 0000000000000000000000000000000000000000..c73adfed5f142886940bc249904d77f9e54befda Binary files /dev/null and b/examples/driven_audio/RD_Radio36_000.wav differ diff --git a/examples/driven_audio/RD_Radio40_000.wav b/examples/driven_audio/RD_Radio40_000.wav new file mode 100644 index 0000000000000000000000000000000000000000..88ce964e1734210451e3a364f87f8661db388b74 Binary files /dev/null and b/examples/driven_audio/RD_Radio40_000.wav differ diff --git a/examples/driven_audio/bus_chinese.wav b/examples/driven_audio/bus_chinese.wav new file mode 100644 index 0000000000000000000000000000000000000000..888647738d72dfaee99b8d40bb0ddf6f7a1872e7 Binary files /dev/null and b/examples/driven_audio/bus_chinese.wav differ diff --git a/examples/driven_audio/chinese_news.wav b/examples/driven_audio/chinese_news.wav new file mode 100644 index 0000000000000000000000000000000000000000..9232795586cbcb926cca70f90691a9e281d32ab9 --- /dev/null +++ b/examples/driven_audio/chinese_news.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b0f4d313a1ca671bc4831d60bcf0c12225efbffe6c0e93e54fbfe9bcd4021cb +size 1536078 diff --git a/examples/driven_audio/chinese_poem1.wav b/examples/driven_audio/chinese_poem1.wav new file mode 100644 index 0000000000000000000000000000000000000000..17c0871100d454bcd95b4281ab6b153c04724fe5 Binary files /dev/null and b/examples/driven_audio/chinese_poem1.wav differ diff --git a/examples/driven_audio/chinese_poem2.wav b/examples/driven_audio/chinese_poem2.wav new file mode 100644 index 0000000000000000000000000000000000000000..e3b294eceff5c5ee43124b7cfa42e4a70196a45f Binary files /dev/null and b/examples/driven_audio/chinese_poem2.wav differ diff --git a/examples/driven_audio/deyu.wav b/examples/driven_audio/deyu.wav new file mode 100644 index 0000000000000000000000000000000000000000..438cd45b36be0d7cec6732d1ffa1c396141a563e --- /dev/null +++ b/examples/driven_audio/deyu.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba1839c57770a2ab0b593ce814344bfd4d750da02acc9be9e8cf5b9113a0f88a +size 2694784 diff --git a/examples/driven_audio/eluosi.wav b/examples/driven_audio/eluosi.wav new file mode 100644 index 0000000000000000000000000000000000000000..336e85fe5cb8d7110fbade7684cce4a33fdffb98 --- /dev/null +++ b/examples/driven_audio/eluosi.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4a3593815dc7b68c256672baa61934c9479efa770af2065fb0886f02713606e +size 1786672 diff --git a/examples/driven_audio/fayu.wav b/examples/driven_audio/fayu.wav new file mode 100644 index 0000000000000000000000000000000000000000..bf5cb6e65b2f959174facc80e13ce145226991cc --- /dev/null +++ b/examples/driven_audio/fayu.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16ebd13626ae4171030b4ea05cceef06078483c352e4b68d469fc2a52bfffceb +size 1940428 diff --git a/examples/driven_audio/imagine.wav b/examples/driven_audio/imagine.wav new file mode 100644 index 0000000000000000000000000000000000000000..c02a95b80b8e2b5c4353a4047239c361e9e3d01a --- /dev/null +++ b/examples/driven_audio/imagine.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2db410217e074d91ae6011e1c5dc0b94f02d05d381c50af8e54253eeacad17d2 +size 1618510 diff --git a/examples/driven_audio/itosinger1.wav b/examples/driven_audio/itosinger1.wav new file mode 100644 index 0000000000000000000000000000000000000000..4937dbb264e2fc24d4752baf8b802b0bac41be24 Binary files /dev/null and b/examples/driven_audio/itosinger1.wav differ diff --git a/examples/driven_audio/japanese.wav b/examples/driven_audio/japanese.wav new file mode 100644 index 0000000000000000000000000000000000000000..63db9ffc287a9186f144b635f87bf352ba30ff22 --- /dev/null +++ b/examples/driven_audio/japanese.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3db5426d0b158799e2be4f609b11f75bfbd4affffe18e9a1c8e6f241fcdedcfc +size 2622712 diff --git a/examples/ref_video/WDA_AlexandriaOcasioCortez_000.mp4 b/examples/ref_video/WDA_AlexandriaOcasioCortez_000.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9d1219ed3f515765753a3c2e4bb97655781bcc4 --- /dev/null +++ b/examples/ref_video/WDA_AlexandriaOcasioCortez_000.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a85242c3fc4d50e2202cea393b9e7ee59019759b68e78e26a254d528c22615a7 +size 2257667 diff --git a/examples/ref_video/WDA_KatieHill_000.mp4 b/examples/ref_video/WDA_KatieHill_000.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61bf40411df95054efee238debd31c7d38ab6a3d --- /dev/null +++ b/examples/ref_video/WDA_KatieHill_000.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fbb4cfd64eedc49b170c441714a9c4fd5e2c2f8a11592070ad89fbd257f2817 +size 3548230 diff --git a/examples/source_image/art_0.png b/examples/source_image/art_0.png new file mode 100644 index 0000000000000000000000000000000000000000..d8d97645a4ecd9018bf2ad6d9094cf581f816f58 Binary files /dev/null and b/examples/source_image/art_0.png differ diff --git a/examples/source_image/art_1.png b/examples/source_image/art_1.png new file mode 100644 index 0000000000000000000000000000000000000000..4388abe026a5ba1f6c2e9f3a782564bb611f5781 Binary files /dev/null and b/examples/source_image/art_1.png differ diff --git a/examples/source_image/art_10.png b/examples/source_image/art_10.png new file mode 100644 index 0000000000000000000000000000000000000000..5f6568b30f063b09cef08c54df629dae7ff54360 Binary files /dev/null and b/examples/source_image/art_10.png differ diff --git a/examples/source_image/art_11.png b/examples/source_image/art_11.png new file mode 100644 index 0000000000000000000000000000000000000000..4caf17ca866fe54cc5c3af33fb0e93114da1bfb9 Binary files /dev/null and b/examples/source_image/art_11.png differ diff --git a/examples/source_image/art_12.png b/examples/source_image/art_12.png new file mode 100644 index 0000000000000000000000000000000000000000..e15306c30f09807f7df80504032cc39b1c265b6a Binary files /dev/null and b/examples/source_image/art_12.png differ diff --git a/examples/source_image/art_13.png b/examples/source_image/art_13.png new file mode 100644 index 0000000000000000000000000000000000000000..129374120f1f01580a9baa0f37d8bbbe904b2373 Binary files /dev/null and b/examples/source_image/art_13.png differ diff --git a/examples/source_image/art_14.png b/examples/source_image/art_14.png new file mode 100644 index 0000000000000000000000000000000000000000..0f0489bf7cebb41346f029421fdf41dc2e52519b Binary files /dev/null and b/examples/source_image/art_14.png differ diff --git a/examples/source_image/art_15.png b/examples/source_image/art_15.png new file mode 100644 index 0000000000000000000000000000000000000000..a0af242a4b3e962aef8ce5c10a5026646509bfc6 Binary files /dev/null and b/examples/source_image/art_15.png differ diff --git a/examples/source_image/art_16.png b/examples/source_image/art_16.png new file mode 100644 index 0000000000000000000000000000000000000000..afb659b641b564a3d850229c67d014483516af67 --- /dev/null +++ b/examples/source_image/art_16.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6d350055eea3abe35ee3fe9df80dcd99d8edae66ef4fc20bf06168bf189f25 +size 1480263 diff --git a/examples/source_image/art_17.png b/examples/source_image/art_17.png new file mode 100644 index 0000000000000000000000000000000000000000..875a3e3c2e985efe7407b6c8fff99faa591b9811 --- /dev/null +++ b/examples/source_image/art_17.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05747bb45dcf271d9bb24344bd1bce0e0746d24ce4e13545b27ad40b50c3bfe7 +size 2092096 diff --git a/examples/source_image/art_18.png b/examples/source_image/art_18.png new file mode 100644 index 0000000000000000000000000000000000000000..96358e0e542f66d1f4fd92acd092124e738fc6fe Binary files /dev/null and b/examples/source_image/art_18.png differ diff --git a/examples/source_image/art_19.png b/examples/source_image/art_19.png new file mode 100644 index 0000000000000000000000000000000000000000..4f477a1ab58994e3cb4140b1a8ca59dcc428f387 Binary files /dev/null and b/examples/source_image/art_19.png differ diff --git a/examples/source_image/art_2.png b/examples/source_image/art_2.png new file mode 100644 index 0000000000000000000000000000000000000000..9560673430d461ad94980731ee0b404fcda32084 Binary files /dev/null and b/examples/source_image/art_2.png differ diff --git a/examples/source_image/art_20.png b/examples/source_image/art_20.png new file mode 100644 index 0000000000000000000000000000000000000000..de1ea5c975dbed93ce80c1aa70f6298703acf70f Binary files /dev/null and b/examples/source_image/art_20.png differ diff --git a/examples/source_image/art_3.png b/examples/source_image/art_3.png new file mode 100644 index 0000000000000000000000000000000000000000..f2d3c117ed2d7074ec5427ebd1e68147e4476031 --- /dev/null +++ b/examples/source_image/art_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81be3a9cc605ab01cbf741330b406db5246e8bbbcb443ad43ffeca2ef161e005 +size 1353396 diff --git a/examples/source_image/art_4.png b/examples/source_image/art_4.png new file mode 100644 index 0000000000000000000000000000000000000000..ce5fda1d95dd1d6d497648fbfb95dc53380d367e --- /dev/null +++ b/examples/source_image/art_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab322220d8eab1bfefdaedea91ca5d08a34258c1ab1e585a9b1c85b32968f983 +size 3625669 diff --git a/examples/source_image/art_5.png b/examples/source_image/art_5.png new file mode 100644 index 0000000000000000000000000000000000000000..2726da0cb91b4ab9d54eef21efa653d2f8cda959 --- /dev/null +++ b/examples/source_image/art_5.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:199217b4c839ed849577aedcad32f2bce934628b9783ba4654a93756b25e7896 +size 1228028 diff --git a/examples/source_image/art_6.png b/examples/source_image/art_6.png new file mode 100644 index 0000000000000000000000000000000000000000..e9f6d8f272dc9bf971285667ecbe765ede41c967 Binary files /dev/null and b/examples/source_image/art_6.png differ diff --git a/examples/source_image/art_7.png b/examples/source_image/art_7.png new file mode 100644 index 0000000000000000000000000000000000000000..d8cc380aacb76a6ce9f5e41086bb1fb375a4e7db Binary files /dev/null and b/examples/source_image/art_7.png differ diff --git a/examples/source_image/art_8.png b/examples/source_image/art_8.png new file mode 100644 index 0000000000000000000000000000000000000000..169035fba5a1ab690564e661e2e5ea95a5a71e87 --- /dev/null +++ b/examples/source_image/art_8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d704497947c07ac16534299451fc0526acddf286c2ab4ceb48161ff6facc2af +size 3119298 diff --git a/examples/source_image/art_9.png b/examples/source_image/art_9.png new file mode 100644 index 0000000000000000000000000000000000000000..61a02dd4a57d382f215a73d635959ae45c208635 --- /dev/null +++ b/examples/source_image/art_9.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90f84739e2aa2388efaf0fac2b57a82df279b213a8dab9faa7af8ae7468b4e80 +size 1262963 diff --git a/examples/source_image/full3.png b/examples/source_image/full3.png new file mode 100644 index 0000000000000000000000000000000000000000..40cd6d6d3c5b95c29d6648c2ba7d7e27c9781970 Binary files /dev/null and b/examples/source_image/full3.png differ diff --git a/examples/source_image/full4.jpeg b/examples/source_image/full4.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..79f17f52123e8d173600e0df138a30e98ba2c6f3 Binary files /dev/null and b/examples/source_image/full4.jpeg differ diff --git a/examples/source_image/full_body_1.png b/examples/source_image/full_body_1.png new file mode 100644 index 0000000000000000000000000000000000000000..4fca65c949b7c7e7f7ed9459c473314a38be791f Binary files /dev/null and b/examples/source_image/full_body_1.png differ diff --git a/examples/source_image/full_body_2.png b/examples/source_image/full_body_2.png new file mode 100644 index 0000000000000000000000000000000000000000..b7bc6228cb2f4e8c01af8d2f52bbbf62540e2412 Binary files /dev/null and b/examples/source_image/full_body_2.png differ diff --git a/examples/source_image/happy.png b/examples/source_image/happy.png new file mode 100644 index 0000000000000000000000000000000000000000..9d194ba9a03dfda0867703d54ea6233819c46a73 Binary files /dev/null and b/examples/source_image/happy.png differ diff --git a/examples/source_image/happy1.png b/examples/source_image/happy1.png new file mode 100644 index 0000000000000000000000000000000000000000..b702974cca1a648ec70efee776e484284b527c90 Binary files /dev/null and b/examples/source_image/happy1.png differ diff --git a/examples/source_image/people_0.png b/examples/source_image/people_0.png new file mode 100644 index 0000000000000000000000000000000000000000..8895eeb07a3e300b9bcfa3bb53e7a6a552182bc3 Binary files /dev/null and b/examples/source_image/people_0.png differ diff --git a/examples/source_image/sad.png b/examples/source_image/sad.png new file mode 100644 index 0000000000000000000000000000000000000000..6584467fdac971207883cdcd84b31da1dbc4dfa6 Binary files /dev/null and b/examples/source_image/sad.png differ diff --git a/examples/source_image/sad1.png b/examples/source_image/sad1.png new file mode 100644 index 0000000000000000000000000000000000000000..341e0cb70886995ecf72eebb4b8a4474ab7d287b Binary files /dev/null and b/examples/source_image/sad1.png differ diff --git a/gfpgan/weights/GFPGANv1.4.pth b/gfpgan/weights/GFPGANv1.4.pth new file mode 100644 index 0000000000000000000000000000000000000000..afedb5c7e826056840c9cc183f2c6f0186fd17ba --- /dev/null +++ b/gfpgan/weights/GFPGANv1.4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2cd4703ab14f4d01fd1383a8a8b266f9a5833dacee8e6a79d3bf21a1b6be5ad +size 348632874 diff --git a/gfpgan/weights/alignment_WFLW_4HG.pth b/gfpgan/weights/alignment_WFLW_4HG.pth new file mode 100644 index 0000000000000000000000000000000000000000..3cfeef20123eb2e74b35a4319c2111ef65783c34 --- /dev/null +++ b/gfpgan/weights/alignment_WFLW_4HG.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbfd137307a4c7debd5c283b9b0ce539466cee417ac0a155e184d857f9f2899c +size 193670248 diff --git a/gfpgan/weights/detection_Resnet50_Final.pth b/gfpgan/weights/detection_Resnet50_Final.pth new file mode 100644 index 0000000000000000000000000000000000000000..16546738ce0a00a9fd47585e0fc52744d31cc117 --- /dev/null +++ b/gfpgan/weights/detection_Resnet50_Final.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d1de9c2944f2ccddca5f5e010ea5ae64a39845a86311af6fdf30841b0a5a16d +size 109497761 diff --git a/gfpgan/weights/parsing_parsenet.pth b/gfpgan/weights/parsing_parsenet.pth new file mode 100644 index 0000000000000000000000000000000000000000..1ac2efc50360a79c9905dbac57d9d99cbfbe863c --- /dev/null +++ b/gfpgan/weights/parsing_parsenet.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d558d8d0e42c20224f13cf5a29c79eba2d59913419f945545d8cf7b72920de2 +size 85331193 diff --git a/inference.py b/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..a0b007901c9848ef8b3409607a4a73cc5a3a5ab9 --- /dev/null +++ b/inference.py @@ -0,0 +1,145 @@ +from glob import glob +import shutil +import torch +from time import strftime +import os, sys, time +from argparse import ArgumentParser + +from src.utils.preprocess import CropAndExtract +from src.test_audio2coeff import Audio2Coeff +from src.facerender.animate import AnimateFromCoeff +from src.generate_batch import get_data +from src.generate_facerender_batch import get_facerender_data +from src.utils.init_path import init_path + +def main(args): + #torch.backends.cudnn.enabled = False + + pic_path = args.source_image + audio_path = args.driven_audio + save_dir = os.path.join(args.result_dir, strftime("%Y_%m_%d_%H.%M.%S")) + os.makedirs(save_dir, exist_ok=True) + pose_style = args.pose_style + device = args.device + batch_size = args.batch_size + input_yaw_list = args.input_yaw + input_pitch_list = args.input_pitch + input_roll_list = args.input_roll + ref_eyeblink = args.ref_eyeblink + ref_pose = args.ref_pose + + current_root_path = os.path.split(sys.argv[0])[0] + + sadtalker_paths = init_path(args.checkpoint_dir, os.path.join(current_root_path, 'src/config'), args.size, args.old_version, args.preprocess) + + #init model + preprocess_model = CropAndExtract(sadtalker_paths, device) + + audio_to_coeff = Audio2Coeff(sadtalker_paths, device) + + animate_from_coeff = AnimateFromCoeff(sadtalker_paths, device) + + #crop image and extract 3dmm from image + first_frame_dir = os.path.join(save_dir, 'first_frame_dir') + os.makedirs(first_frame_dir, exist_ok=True) + print('3DMM Extraction for source image') + first_coeff_path, crop_pic_path, crop_info = preprocess_model.generate(pic_path, first_frame_dir, args.preprocess,\ + source_image_flag=True, pic_size=args.size) + if first_coeff_path is None: + print("Can't get the coeffs of the input") + return + + if ref_eyeblink is not None: + ref_eyeblink_videoname = os.path.splitext(os.path.split(ref_eyeblink)[-1])[0] + ref_eyeblink_frame_dir = os.path.join(save_dir, ref_eyeblink_videoname) + os.makedirs(ref_eyeblink_frame_dir, exist_ok=True) + print('3DMM Extraction for the reference video providing eye blinking') + ref_eyeblink_coeff_path, _, _ = preprocess_model.generate(ref_eyeblink, ref_eyeblink_frame_dir, args.preprocess, source_image_flag=False) + else: + ref_eyeblink_coeff_path=None + + if ref_pose is not None: + if ref_pose == ref_eyeblink: + ref_pose_coeff_path = ref_eyeblink_coeff_path + else: + ref_pose_videoname = os.path.splitext(os.path.split(ref_pose)[-1])[0] + ref_pose_frame_dir = os.path.join(save_dir, ref_pose_videoname) + os.makedirs(ref_pose_frame_dir, exist_ok=True) + print('3DMM Extraction for the reference video providing pose') + ref_pose_coeff_path, _, _ = preprocess_model.generate(ref_pose, ref_pose_frame_dir, args.preprocess, source_image_flag=False) + else: + ref_pose_coeff_path=None + + #audio2ceoff + batch = get_data(first_coeff_path, audio_path, device, ref_eyeblink_coeff_path, still=args.still) + coeff_path = audio_to_coeff.generate(batch, save_dir, pose_style, ref_pose_coeff_path) + + # 3dface render + if args.face3dvis: + from src.face3d.visualize import gen_composed_video + gen_composed_video(args, device, first_coeff_path, coeff_path, audio_path, os.path.join(save_dir, '3dface.mp4')) + + #coeff2video + data = get_facerender_data(coeff_path, crop_pic_path, first_coeff_path, audio_path, + batch_size, input_yaw_list, input_pitch_list, input_roll_list, + expression_scale=args.expression_scale, still_mode=args.still, preprocess=args.preprocess, size=args.size) + + result = animate_from_coeff.generate(data, save_dir, pic_path, crop_info, \ + enhancer=args.enhancer, background_enhancer=args.background_enhancer, preprocess=args.preprocess, img_size=args.size) + + shutil.move(result, save_dir+'.mp4') + print('The generated video is named:', save_dir+'.mp4') + + if not args.verbose: + shutil.rmtree(save_dir) + + +if __name__ == '__main__': + + parser = ArgumentParser() + parser.add_argument("--driven_audio", default='./examples/driven_audio/bus_chinese.wav', help="path to driven audio") + parser.add_argument("--source_image", default='./examples/source_image/full_body_1.png', help="path to source image") + parser.add_argument("--ref_eyeblink", default=None, help="path to reference video providing eye blinking") + parser.add_argument("--ref_pose", default=None, help="path to reference video providing pose") + parser.add_argument("--checkpoint_dir", default='./checkpoints', help="path to output") + parser.add_argument("--result_dir", default='./results', help="path to output") + parser.add_argument("--pose_style", type=int, default=0, help="input pose style from [0, 46)") + parser.add_argument("--batch_size", type=int, default=2, help="the batch size of facerender") + parser.add_argument("--size", type=int, default=256, help="the image size of the facerender") + parser.add_argument("--expression_scale", type=float, default=1., help="the batch size of facerender") + parser.add_argument('--input_yaw', nargs='+', type=int, default=None, help="the input yaw degree of the user ") + parser.add_argument('--input_pitch', nargs='+', type=int, default=None, help="the input pitch degree of the user") + parser.add_argument('--input_roll', nargs='+', type=int, default=None, help="the input roll degree of the user") + parser.add_argument('--enhancer', type=str, default=None, help="Face enhancer, [gfpgan, RestoreFormer]") + parser.add_argument('--background_enhancer', type=str, default=None, help="background enhancer, [realesrgan]") + parser.add_argument("--cpu", dest="cpu", action="store_true") + parser.add_argument("--face3dvis", action="store_true", help="generate 3d face and 3d landmarks") + parser.add_argument("--still", action="store_true", help="can crop back to the original videos for the full body aniamtion") + parser.add_argument("--preprocess", default='crop', choices=['crop', 'extcrop', 'resize', 'full', 'extfull'], help="how to preprocess the images" ) + parser.add_argument("--verbose",action="store_true", help="saving the intermedia output or not" ) + parser.add_argument("--old_version",action="store_true", help="use the pth other than safetensor version" ) + + + # net structure and parameters + parser.add_argument('--net_recon', type=str, default='resnet50', choices=['resnet18', 'resnet34', 'resnet50'], help='useless') + parser.add_argument('--init_path', type=str, default=None, help='Useless') + parser.add_argument('--use_last_fc',default=False, help='zero initialize the last fc') + parser.add_argument('--bfm_folder', type=str, default='./checkpoints/BFM_Fitting/') + parser.add_argument('--bfm_model', type=str, default='BFM_model_front.mat', help='bfm model') + + # default renderer parameters + parser.add_argument('--focal', type=float, default=1015.) + parser.add_argument('--center', type=float, default=112.) + parser.add_argument('--camera_d', type=float, default=10.) + parser.add_argument('--z_near', type=float, default=5.) + parser.add_argument('--z_far', type=float, default=15.) + + args = parser.parse_args() + + if torch.cuda.is_available() and not args.cpu: + args.device = "cuda" + else: + args.device = "cpu" + + main(args) + diff --git a/launcher.py b/launcher.py new file mode 100644 index 0000000000000000000000000000000000000000..17ce9f1a18c3d563333bbb0eacc2922fb8524e3f --- /dev/null +++ b/launcher.py @@ -0,0 +1,204 @@ +# this scripts installs necessary requirements and launches main program in webui.py +# borrow from : https://github.com/AUTOMATIC1111/stable-diffusion-webui/blob/master/launch.py +import subprocess +import os +import sys +import importlib.util +import shlex +import platform +import json + +python = sys.executable +git = os.environ.get('GIT', "git") +index_url = os.environ.get('INDEX_URL', "") +stored_commit_hash = None +skip_install = False +dir_repos = "repositories" +script_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + +if 'GRADIO_ANALYTICS_ENABLED' not in os.environ: + os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False' + + +def check_python_version(): + is_windows = platform.system() == "Windows" + major = sys.version_info.major + minor = sys.version_info.minor + micro = sys.version_info.micro + + if is_windows: + supported_minors = [10] + else: + supported_minors = [7, 8, 9, 10, 11] + + if not (major == 3 and minor in supported_minors): + + raise (f""" +INCOMPATIBLE PYTHON VERSION +This program is tested with 3.10.6 Python, but you have {major}.{minor}.{micro}. +If you encounter an error with "RuntimeError: Couldn't install torch." message, +or any other error regarding unsuccessful package (library) installation, +please downgrade (or upgrade) to the latest version of 3.10 Python +and delete current Python and "venv" folder in WebUI's directory. +You can download 3.10 Python from here: https://www.python.org/downloads/release/python-3109/ +{"Alternatively, use a binary release of WebUI: https://github.com/AUTOMATIC1111/stable-diffusion-webui/releases" if is_windows else ""} +Use --skip-python-version-check to suppress this warning. +""") + + +def commit_hash(): + global stored_commit_hash + + if stored_commit_hash is not None: + return stored_commit_hash + + try: + stored_commit_hash = run(f"{git} rev-parse HEAD").strip() + except Exception: + stored_commit_hash = "" + + return stored_commit_hash + + +def run(command, desc=None, errdesc=None, custom_env=None, live=False): + if desc is not None: + print(desc) + + if live: + result = subprocess.run(command, shell=True, env=os.environ if custom_env is None else custom_env) + if result.returncode != 0: + raise RuntimeError(f"""{errdesc or 'Error running command'}. +Command: {command} +Error code: {result.returncode}""") + + return "" + + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ if custom_env is None else custom_env) + + if result.returncode != 0: + + message = f"""{errdesc or 'Error running command'}. +Command: {command} +Error code: {result.returncode} +stdout: {result.stdout.decode(encoding="utf8", errors="ignore") if len(result.stdout)>0 else ''} +stderr: {result.stderr.decode(encoding="utf8", errors="ignore") if len(result.stderr)>0 else ''} +""" + raise RuntimeError(message) + + return result.stdout.decode(encoding="utf8", errors="ignore") + + +def check_run(command): + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + return result.returncode == 0 + + +def is_installed(package): + try: + spec = importlib.util.find_spec(package) + except ModuleNotFoundError: + return False + + return spec is not None + + +def repo_dir(name): + return os.path.join(script_path, dir_repos, name) + + +def run_python(code, desc=None, errdesc=None): + return run(f'"{python}" -c "{code}"', desc, errdesc) + + +def run_pip(args, desc=None): + if skip_install: + return + + index_url_line = f' --index-url {index_url}' if index_url != '' else '' + return run(f'"{python}" -m pip {args} --prefer-binary{index_url_line}', desc=f"Installing {desc}", errdesc=f"Couldn't install {desc}") + + +def check_run_python(code): + return check_run(f'"{python}" -c "{code}"') + + +def git_clone(url, dir, name, commithash=None): + # TODO clone into temporary dir and move if successful + + if os.path.exists(dir): + if commithash is None: + return + + current_hash = run(f'"{git}" -C "{dir}" rev-parse HEAD', None, f"Couldn't determine {name}'s hash: {commithash}").strip() + if current_hash == commithash: + return + + run(f'"{git}" -C "{dir}" fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}") + run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}") + return + + run(f'"{git}" clone "{url}" "{dir}"', f"Cloning {name} into {dir}...", f"Couldn't clone {name}") + + if commithash is not None: + run(f'"{git}" -C "{dir}" checkout {commithash}', None, "Couldn't checkout {name}'s hash: {commithash}") + + +def git_pull_recursive(dir): + for subdir, _, _ in os.walk(dir): + if os.path.exists(os.path.join(subdir, '.git')): + try: + output = subprocess.check_output([git, '-C', subdir, 'pull', '--autostash']) + print(f"Pulled changes for repository in '{subdir}':\n{output.decode('utf-8').strip()}\n") + except subprocess.CalledProcessError as e: + print(f"Couldn't perform 'git pull' on repository in '{subdir}':\n{e.output.decode('utf-8').strip()}\n") + + +def run_extension_installer(extension_dir): + path_installer = os.path.join(extension_dir, "install.py") + if not os.path.isfile(path_installer): + return + + try: + env = os.environ.copy() + env['PYTHONPATH'] = os.path.abspath(".") + + print(run(f'"{python}" "{path_installer}"', errdesc=f"Error running install.py for extension {extension_dir}", custom_env=env)) + except Exception as e: + print(e, file=sys.stderr) + + +def prepare_environment(): + global skip_install + + torch_command = os.environ.get('TORCH_COMMAND', "pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113") + + ## check windows + if sys.platform != 'win32': + requirements_file = os.environ.get('REQS_FILE', "req.txt") + else: + requirements_file = os.environ.get('REQS_FILE', "requirements.txt") + + commit = commit_hash() + + print(f"Python {sys.version}") + print(f"Commit hash: {commit}") + + if not is_installed("torch") or not is_installed("torchvision"): + run(f'"{python}" -m {torch_command}', "Installing torch and torchvision", "Couldn't install torch", live=True) + + run_pip(f"install -r \"{requirements_file}\"", "requirements for SadTalker WebUI (may take longer time in first time)") + + if sys.platform != 'win32' and not is_installed('tts'): + run_pip(f"install TTS", "install TTS individually in SadTalker, which might not work on windows.") + + +def start(): + print(f"Launching SadTalker Web UI") + from app_sadtalker import sadtalker_demo + demo = sadtalker_demo() + demo.queue() + demo.launch() + +if __name__ == "__main__": + prepare_environment() + start() \ No newline at end of file diff --git a/mel_processing.py b/mel_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..99c5b35beb83f3b288af0fac5b49ebf2c69f062c --- /dev/null +++ b/mel_processing.py @@ -0,0 +1,112 @@ +import math +import os +import random +import torch +from torch import nn +import torch.nn.functional as F +import torch.utils.data +import numpy as np +import librosa +import librosa.util as librosa_util +from librosa.util import normalize, pad_center, tiny +from scipy.signal import get_window +from scipy.io.wavfile import read +from librosa.filters import mel as librosa_mel_fn + +MAX_WAV_VALUE = 32768.0 + + +def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): + """ + PARAMS + ------ + C: compression factor + """ + return torch.log(torch.clamp(x, min=clip_val) * C) + + +def dynamic_range_decompression_torch(x, C=1): + """ + PARAMS + ------ + C: compression factor used to compress + """ + return torch.exp(x) / C + + +def spectral_normalize_torch(magnitudes): + output = dynamic_range_compression_torch(magnitudes) + return output + + +def spectral_de_normalize_torch(magnitudes): + output = dynamic_range_decompression_torch(magnitudes) + return output + + +mel_basis = {} +hann_window = {} + + +def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False): + if torch.min(y) < -1.: + print('min value is ', torch.min(y)) + if torch.max(y) > 1.: + print('max value is ', torch.max(y)) + + global hann_window + dtype_device = str(y.dtype) + '_' + str(y.device) + wnsize_dtype_device = str(win_size) + '_' + dtype_device + if wnsize_dtype_device not in hann_window: + hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) + + y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') + y = y.squeeze(1) + + spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], + center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=False) + + spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) + return spec + + +def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax): + global mel_basis + dtype_device = str(spec.dtype) + '_' + str(spec.device) + fmax_dtype_device = str(fmax) + '_' + dtype_device + if fmax_dtype_device not in mel_basis: + mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax) + mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device) + spec = torch.matmul(mel_basis[fmax_dtype_device], spec) + spec = spectral_normalize_torch(spec) + return spec + + +def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False): + if torch.min(y) < -1.: + print('min value is ', torch.min(y)) + if torch.max(y) > 1.: + print('max value is ', torch.max(y)) + + global mel_basis, hann_window + dtype_device = str(y.dtype) + '_' + str(y.device) + fmax_dtype_device = str(fmax) + '_' + dtype_device + wnsize_dtype_device = str(win_size) + '_' + dtype_device + if fmax_dtype_device not in mel_basis: + mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax) + mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device) + if wnsize_dtype_device not in hann_window: + hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) + + y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') + y = y.squeeze(1) + + spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], + center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=False) + + spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) + + spec = torch.matmul(mel_basis[fmax_dtype_device], spec) + spec = spectral_normalize_torch(spec) + + return spec diff --git a/models.py b/models.py new file mode 100644 index 0000000000000000000000000000000000000000..46b8aacb1bef18f6fad4c20c968b19125626799c --- /dev/null +++ b/models.py @@ -0,0 +1,351 @@ +import copy +import math +import torch +from torch import nn +from torch.nn import functional as F + +import commons +import modules + +from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d +from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm +from commons import init_weights, get_padding + + +class ResidualCouplingBlock(nn.Module): + def __init__(self, + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + n_flows=4, + gin_channels=0): + super().__init__() + self.channels = channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.n_flows = n_flows + self.gin_channels = gin_channels + + self.flows = nn.ModuleList() + for i in range(n_flows): + self.flows.append(modules.ResidualCouplingLayer(channels, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels, mean_only=True)) + self.flows.append(modules.Flip()) + + def forward(self, x, x_mask, g=None, reverse=False): + if not reverse: + for flow in self.flows: + x, _ = flow(x, x_mask, g=g, reverse=reverse) + else: + for flow in reversed(self.flows): + x = flow(x, x_mask, g=g, reverse=reverse) + return x + + +class Encoder(nn.Module): + def __init__(self, + in_channels, + out_channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + gin_channels=0): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + + self.pre = nn.Conv1d(in_channels, hidden_channels, 1) + self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels) + self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) + + def forward(self, x, x_lengths, g=None): + x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype) + x = self.pre(x) * x_mask + x = self.enc(x, x_mask, g=g) + stats = self.proj(x) * x_mask + m, logs = torch.split(stats, self.out_channels, dim=1) + z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask + return z, m, logs, x_mask + + +class Generator(torch.nn.Module): + def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=0): + super(Generator, self).__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + self.conv_pre = Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3) + resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2 + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append(weight_norm( + ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)), + k, u, padding=(k-u)//2))) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel//(2**(i+1)) + for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)): + self.resblocks.append(resblock(ch, k, d)) + + self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) + self.ups.apply(init_weights) + + if gin_channels != 0: + self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) + + def forward(self, x, g=None): + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, modules.LRELU_SLOPE) + x = self.ups[i](x) + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i*self.num_kernels+j](x) + else: + xs += self.resblocks[i*self.num_kernels+j](x) + x = xs / self.num_kernels + x = F.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) + + return x + + def remove_weight_norm(self): + print('Removing weight norm...') + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + + +class DiscriminatorP(torch.nn.Module): + def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False): + super(DiscriminatorP, self).__init__() + self.period = period + self.use_spectral_norm = use_spectral_norm + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList([ + norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), + norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), + norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), + norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), + norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))), + ]) + self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) + + def forward(self, x): + fmap = [] + + # 1d to 2d + b, c, t = x.shape + if t % self.period != 0: # pad first + n_pad = self.period - (t % self.period) + x = F.pad(x, (0, n_pad), "reflect") + t = t + n_pad + x = x.view(b, c, t // self.period, self.period) + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class DiscriminatorS(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(DiscriminatorS, self).__init__() + norm_f = weight_norm if use_spectral_norm == False else spectral_norm + self.convs = nn.ModuleList([ + norm_f(Conv1d(1, 16, 15, 1, padding=7)), + norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)), + norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)), + norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)), + norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)), + norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), + ]) + self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) + + def forward(self, x): + fmap = [] + + for l in self.convs: + x = l(x) + x = F.leaky_relu(x, modules.LRELU_SLOPE) + fmap.append(x) + x = self.conv_post(x) + fmap.append(x) + x = torch.flatten(x, 1, -1) + + return x, fmap + + +class MultiPeriodDiscriminator(torch.nn.Module): + def __init__(self, use_spectral_norm=False): + super(MultiPeriodDiscriminator, self).__init__() + periods = [2,3,5,7,11] + + discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] + discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods] + self.discriminators = nn.ModuleList(discs) + + def forward(self, y, y_hat): + y_d_rs = [] + y_d_gs = [] + fmap_rs = [] + fmap_gs = [] + for i, d in enumerate(self.discriminators): + y_d_r, fmap_r = d(y) + y_d_g, fmap_g = d(y_hat) + y_d_rs.append(y_d_r) + y_d_gs.append(y_d_g) + fmap_rs.append(fmap_r) + fmap_gs.append(fmap_g) + + return y_d_rs, y_d_gs, fmap_rs, fmap_gs + + +class SpeakerEncoder(torch.nn.Module): + def __init__(self, mel_n_channels=80, model_num_layers=3, model_hidden_size=256, model_embedding_size=256): + super(SpeakerEncoder, self).__init__() + self.lstm = nn.LSTM(mel_n_channels, model_hidden_size, model_num_layers, batch_first=True) + self.linear = nn.Linear(model_hidden_size, model_embedding_size) + self.relu = nn.ReLU() + + def forward(self, mels): + self.lstm.flatten_parameters() + _, (hidden, _) = self.lstm(mels) + embeds_raw = self.relu(self.linear(hidden[-1])) + return embeds_raw / torch.norm(embeds_raw, dim=1, keepdim=True) + + def compute_partial_slices(self, total_frames, partial_frames, partial_hop): + mel_slices = [] + for i in range(0, total_frames-partial_frames, partial_hop): + mel_range = torch.arange(i, i+partial_frames) + mel_slices.append(mel_range) + + return mel_slices + + def embed_utterance(self, mel, partial_frames=128, partial_hop=64): + mel_len = mel.size(1) + last_mel = mel[:,-partial_frames:] + + if mel_len > partial_frames: + mel_slices = self.compute_partial_slices(mel_len, partial_frames, partial_hop) + mels = list(mel[:,s] for s in mel_slices) + mels.append(last_mel) + mels = torch.stack(tuple(mels), 0).squeeze(1) + + with torch.no_grad(): + partial_embeds = self(mels) + embed = torch.mean(partial_embeds, axis=0).unsqueeze(0) + #embed = embed / torch.linalg.norm(embed, 2) + else: + with torch.no_grad(): + embed = self(last_mel) + + return embed + + +class SynthesizerTrn(nn.Module): + """ + Synthesizer for Training + """ + + def __init__(self, + spec_channels, + segment_size, + inter_channels, + hidden_channels, + filter_channels, + n_heads, + n_layers, + kernel_size, + p_dropout, + resblock, + resblock_kernel_sizes, + resblock_dilation_sizes, + upsample_rates, + upsample_initial_channel, + upsample_kernel_sizes, + gin_channels, + ssl_dim, + use_spk, + **kwargs): + + super().__init__() + self.spec_channels = spec_channels + self.inter_channels = inter_channels + self.hidden_channels = hidden_channels + self.filter_channels = filter_channels + self.n_heads = n_heads + self.n_layers = n_layers + self.kernel_size = kernel_size + self.p_dropout = p_dropout + self.resblock = resblock + self.resblock_kernel_sizes = resblock_kernel_sizes + self.resblock_dilation_sizes = resblock_dilation_sizes + self.upsample_rates = upsample_rates + self.upsample_initial_channel = upsample_initial_channel + self.upsample_kernel_sizes = upsample_kernel_sizes + self.segment_size = segment_size + self.gin_channels = gin_channels + self.ssl_dim = ssl_dim + self.use_spk = use_spk + + self.enc_p = Encoder(ssl_dim, inter_channels, hidden_channels, 5, 1, 16) + self.dec = Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=gin_channels) + self.enc_q = Encoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels) + self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4, gin_channels=gin_channels) + + if not self.use_spk: + self.enc_spk = SpeakerEncoder(model_hidden_size=gin_channels, model_embedding_size=gin_channels) + + def forward(self, c, spec, g=None, mel=None, c_lengths=None, spec_lengths=None): + if c_lengths == None: + c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device) + if spec_lengths == None: + spec_lengths = (torch.ones(spec.size(0)) * spec.size(-1)).to(spec.device) + + if not self.use_spk: + g = self.enc_spk(mel.transpose(1,2)) + g = g.unsqueeze(-1) + + _, m_p, logs_p, _ = self.enc_p(c, c_lengths) + z, m_q, logs_q, spec_mask = self.enc_q(spec, spec_lengths, g=g) + z_p = self.flow(z, spec_mask, g=g) + + z_slice, ids_slice = commons.rand_slice_segments(z, spec_lengths, self.segment_size) + o = self.dec(z_slice, g=g) + + return o, ids_slice, spec_mask, (z, z_p, m_p, logs_p, m_q, logs_q) + + def infer(self, c, g=None, mel=None, c_lengths=None): + if c_lengths == None: + c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device) + if not self.use_spk: + g = self.enc_spk.embed_utterance(mel.transpose(1,2)) + g = g.unsqueeze(-1) + + z_p, m_p, logs_p, c_mask = self.enc_p(c, c_lengths) + z = self.flow(z_p, c_mask, g=g, reverse=True) + o = self.dec(z * c_mask, g=g) + + return o diff --git a/modules.py b/modules.py new file mode 100644 index 0000000000000000000000000000000000000000..52ee14e41a5b6d67d875d1b694aecd2a51244897 --- /dev/null +++ b/modules.py @@ -0,0 +1,342 @@ +import copy +import math +import numpy as np +import scipy +import torch +from torch import nn +from torch.nn import functional as F + +from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d +from torch.nn.utils import weight_norm, remove_weight_norm + +import commons +from commons import init_weights, get_padding + + +LRELU_SLOPE = 0.1 + + +class LayerNorm(nn.Module): + def __init__(self, channels, eps=1e-5): + super().__init__() + self.channels = channels + self.eps = eps + + self.gamma = nn.Parameter(torch.ones(channels)) + self.beta = nn.Parameter(torch.zeros(channels)) + + def forward(self, x): + x = x.transpose(1, -1) + x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps) + return x.transpose(1, -1) + + +class ConvReluNorm(nn.Module): + def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout): + super().__init__() + self.in_channels = in_channels + self.hidden_channels = hidden_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.n_layers = n_layers + self.p_dropout = p_dropout + assert n_layers > 1, "Number of layers should be larger than 0." + + self.conv_layers = nn.ModuleList() + self.norm_layers = nn.ModuleList() + self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size//2)) + self.norm_layers.append(LayerNorm(hidden_channels)) + self.relu_drop = nn.Sequential( + nn.ReLU(), + nn.Dropout(p_dropout)) + for _ in range(n_layers-1): + self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size//2)) + self.norm_layers.append(LayerNorm(hidden_channels)) + self.proj = nn.Conv1d(hidden_channels, out_channels, 1) + self.proj.weight.data.zero_() + self.proj.bias.data.zero_() + + def forward(self, x, x_mask): + x_org = x + for i in range(self.n_layers): + x = self.conv_layers[i](x * x_mask) + x = self.norm_layers[i](x) + x = self.relu_drop(x) + x = x_org + self.proj(x) + return x * x_mask + + +class DDSConv(nn.Module): + """ + Dialted and Depth-Separable Convolution + """ + def __init__(self, channels, kernel_size, n_layers, p_dropout=0.): + super().__init__() + self.channels = channels + self.kernel_size = kernel_size + self.n_layers = n_layers + self.p_dropout = p_dropout + + self.drop = nn.Dropout(p_dropout) + self.convs_sep = nn.ModuleList() + self.convs_1x1 = nn.ModuleList() + self.norms_1 = nn.ModuleList() + self.norms_2 = nn.ModuleList() + for i in range(n_layers): + dilation = kernel_size ** i + padding = (kernel_size * dilation - dilation) // 2 + self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size, + groups=channels, dilation=dilation, padding=padding + )) + self.convs_1x1.append(nn.Conv1d(channels, channels, 1)) + self.norms_1.append(LayerNorm(channels)) + self.norms_2.append(LayerNorm(channels)) + + def forward(self, x, x_mask, g=None): + if g is not None: + x = x + g + for i in range(self.n_layers): + y = self.convs_sep[i](x * x_mask) + y = self.norms_1[i](y) + y = F.gelu(y) + y = self.convs_1x1[i](y) + y = self.norms_2[i](y) + y = F.gelu(y) + y = self.drop(y) + x = x + y + return x * x_mask + + +class WN(torch.nn.Module): + def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0): + super(WN, self).__init__() + assert(kernel_size % 2 == 1) + self.hidden_channels =hidden_channels + self.kernel_size = kernel_size, + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.gin_channels = gin_channels + self.p_dropout = p_dropout + + self.in_layers = torch.nn.ModuleList() + self.res_skip_layers = torch.nn.ModuleList() + self.drop = nn.Dropout(p_dropout) + + if gin_channels != 0: + cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1) + self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') + + for i in range(n_layers): + dilation = dilation_rate ** i + padding = int((kernel_size * dilation - dilation) / 2) + in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size, + dilation=dilation, padding=padding) + in_layer = torch.nn.utils.weight_norm(in_layer, name='weight') + self.in_layers.append(in_layer) + + # last one is not necessary + if i < n_layers - 1: + res_skip_channels = 2 * hidden_channels + else: + res_skip_channels = hidden_channels + + res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1) + res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') + self.res_skip_layers.append(res_skip_layer) + + def forward(self, x, x_mask, g=None, **kwargs): + output = torch.zeros_like(x) + n_channels_tensor = torch.IntTensor([self.hidden_channels]) + + if g is not None: + g = self.cond_layer(g) + + for i in range(self.n_layers): + x_in = self.in_layers[i](x) + if g is not None: + cond_offset = i * 2 * self.hidden_channels + g_l = g[:,cond_offset:cond_offset+2*self.hidden_channels,:] + else: + g_l = torch.zeros_like(x_in) + + acts = commons.fused_add_tanh_sigmoid_multiply( + x_in, + g_l, + n_channels_tensor) + acts = self.drop(acts) + + res_skip_acts = self.res_skip_layers[i](acts) + if i < self.n_layers - 1: + res_acts = res_skip_acts[:,:self.hidden_channels,:] + x = (x + res_acts) * x_mask + output = output + res_skip_acts[:,self.hidden_channels:,:] + else: + output = output + res_skip_acts + return output * x_mask + + def remove_weight_norm(self): + if self.gin_channels != 0: + torch.nn.utils.remove_weight_norm(self.cond_layer) + for l in self.in_layers: + torch.nn.utils.remove_weight_norm(l) + for l in self.res_skip_layers: + torch.nn.utils.remove_weight_norm(l) + + +class ResBlock1(torch.nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): + super(ResBlock1, self).__init__() + self.convs1 = nn.ModuleList([ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2], + padding=get_padding(kernel_size, dilation[2]))) + ]) + self.convs1.apply(init_weights) + + self.convs2 = nn.ModuleList([ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, + padding=get_padding(kernel_size, 1))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, + padding=get_padding(kernel_size, 1))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, + padding=get_padding(kernel_size, 1))) + ]) + self.convs2.apply(init_weights) + + def forward(self, x, x_mask=None): + for c1, c2 in zip(self.convs1, self.convs2): + xt = F.leaky_relu(x, LRELU_SLOPE) + if x_mask is not None: + xt = xt * x_mask + xt = c1(xt) + xt = F.leaky_relu(xt, LRELU_SLOPE) + if x_mask is not None: + xt = xt * x_mask + xt = c2(xt) + x = xt + x + if x_mask is not None: + x = x * x_mask + return x + + def remove_weight_norm(self): + for l in self.convs1: + remove_weight_norm(l) + for l in self.convs2: + remove_weight_norm(l) + + +class ResBlock2(torch.nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3)): + super(ResBlock2, self).__init__() + self.convs = nn.ModuleList([ + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], + padding=get_padding(kernel_size, dilation[0]))), + weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], + padding=get_padding(kernel_size, dilation[1]))) + ]) + self.convs.apply(init_weights) + + def forward(self, x, x_mask=None): + for c in self.convs: + xt = F.leaky_relu(x, LRELU_SLOPE) + if x_mask is not None: + xt = xt * x_mask + xt = c(xt) + x = xt + x + if x_mask is not None: + x = x * x_mask + return x + + def remove_weight_norm(self): + for l in self.convs: + remove_weight_norm(l) + + +class Log(nn.Module): + def forward(self, x, x_mask, reverse=False, **kwargs): + if not reverse: + y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask + logdet = torch.sum(-y, [1, 2]) + return y, logdet + else: + x = torch.exp(x) * x_mask + return x + + +class Flip(nn.Module): + def forward(self, x, *args, reverse=False, **kwargs): + x = torch.flip(x, [1]) + if not reverse: + logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device) + return x, logdet + else: + return x + + +class ElementwiseAffine(nn.Module): + def __init__(self, channels): + super().__init__() + self.channels = channels + self.m = nn.Parameter(torch.zeros(channels,1)) + self.logs = nn.Parameter(torch.zeros(channels,1)) + + def forward(self, x, x_mask, reverse=False, **kwargs): + if not reverse: + y = self.m + torch.exp(self.logs) * x + y = y * x_mask + logdet = torch.sum(self.logs * x_mask, [1,2]) + return y, logdet + else: + x = (x - self.m) * torch.exp(-self.logs) * x_mask + return x + + +class ResidualCouplingLayer(nn.Module): + def __init__(self, + channels, + hidden_channels, + kernel_size, + dilation_rate, + n_layers, + p_dropout=0, + gin_channels=0, + mean_only=False): + assert channels % 2 == 0, "channels should be divisible by 2" + super().__init__() + self.channels = channels + self.hidden_channels = hidden_channels + self.kernel_size = kernel_size + self.dilation_rate = dilation_rate + self.n_layers = n_layers + self.half_channels = channels // 2 + self.mean_only = mean_only + + self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1) + self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels) + self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) + self.post.weight.data.zero_() + self.post.bias.data.zero_() + + def forward(self, x, x_mask, g=None, reverse=False): + x0, x1 = torch.split(x, [self.half_channels]*2, 1) + h = self.pre(x0) * x_mask + h = self.enc(h, x_mask, g=g) + stats = self.post(h) * x_mask + if not self.mean_only: + m, logs = torch.split(stats, [self.half_channels]*2, 1) + else: + m = stats + logs = torch.zeros_like(m) + + if not reverse: + x1 = m + x1 * torch.exp(logs) * x_mask + x = torch.cat([x0, x1], 1) + logdet = torch.sum(logs, [1,2]) + return x, logdet + else: + x1 = (x1 - m) * torch.exp(-logs) * x_mask + x = torch.cat([x0, x1], 1) + return x diff --git a/predict.py b/predict.py new file mode 100644 index 0000000000000000000000000000000000000000..1bfcd28ee3e23c4977af5319f95d817bebefeed0 --- /dev/null +++ b/predict.py @@ -0,0 +1,192 @@ +"""run bash scripts/download_models.sh first to prepare the weights file""" +import os +import shutil +from argparse import Namespace +from src.utils.preprocess import CropAndExtract +from src.test_audio2coeff import Audio2Coeff +from src.facerender.animate import AnimateFromCoeff +from src.generate_batch import get_data +from src.generate_facerender_batch import get_facerender_data +from src.utils.init_path import init_path +from cog import BasePredictor, Input, Path + +checkpoints = "checkpoints" + + +class Predictor(BasePredictor): + def setup(self): + """Load the model into memory to make running multiple predictions efficient""" + device = "cuda" + + + sadtalker_paths = init_path(checkpoints,os.path.join("src","config")) + + # init model + self.preprocess_model = CropAndExtract(sadtalker_paths, device + ) + + self.audio_to_coeff = Audio2Coeff( + sadtalker_paths, + device, + ) + + self.animate_from_coeff = { + "full": AnimateFromCoeff( + sadtalker_paths, + device, + ), + "others": AnimateFromCoeff( + sadtalker_paths, + device, + ), + } + + def predict( + self, + source_image: Path = Input( + description="Upload the source image, it can be video.mp4 or picture.png", + ), + driven_audio: Path = Input( + description="Upload the driven audio, accepts .wav and .mp4 file", + ), + enhancer: str = Input( + description="Choose a face enhancer", + choices=["gfpgan", "RestoreFormer"], + default="gfpgan", + ), + preprocess: str = Input( + description="how to preprocess the images", + choices=["crop", "resize", "full"], + default="full", + ), + ref_eyeblink: Path = Input( + description="path to reference video providing eye blinking", + default=None, + ), + ref_pose: Path = Input( + description="path to reference video providing pose", + default=None, + ), + still: bool = Input( + description="can crop back to the original videos for the full body aniamtion when preprocess is full", + default=True, + ), + ) -> Path: + """Run a single prediction on the model""" + + animate_from_coeff = ( + self.animate_from_coeff["full"] + if preprocess == "full" + else self.animate_from_coeff["others"] + ) + + args = load_default() + args.pic_path = str(source_image) + args.audio_path = str(driven_audio) + device = "cuda" + args.still = still + args.ref_eyeblink = None if ref_eyeblink is None else str(ref_eyeblink) + args.ref_pose = None if ref_pose is None else str(ref_pose) + + # crop image and extract 3dmm from image + results_dir = "results" + if os.path.exists(results_dir): + shutil.rmtree(results_dir) + os.makedirs(results_dir) + first_frame_dir = os.path.join(results_dir, "first_frame_dir") + os.makedirs(first_frame_dir) + + print("3DMM Extraction for source image") + first_coeff_path, crop_pic_path, crop_info = self.preprocess_model.generate( + args.pic_path, first_frame_dir, preprocess, source_image_flag=True + ) + if first_coeff_path is None: + print("Can't get the coeffs of the input") + return + + if ref_eyeblink is not None: + ref_eyeblink_videoname = os.path.splitext(os.path.split(ref_eyeblink)[-1])[ + 0 + ] + ref_eyeblink_frame_dir = os.path.join(results_dir, ref_eyeblink_videoname) + os.makedirs(ref_eyeblink_frame_dir, exist_ok=True) + print("3DMM Extraction for the reference video providing eye blinking") + ref_eyeblink_coeff_path, _, _ = self.preprocess_model.generate( + ref_eyeblink, ref_eyeblink_frame_dir + ) + else: + ref_eyeblink_coeff_path = None + + if ref_pose is not None: + if ref_pose == ref_eyeblink: + ref_pose_coeff_path = ref_eyeblink_coeff_path + else: + ref_pose_videoname = os.path.splitext(os.path.split(ref_pose)[-1])[0] + ref_pose_frame_dir = os.path.join(results_dir, ref_pose_videoname) + os.makedirs(ref_pose_frame_dir, exist_ok=True) + print("3DMM Extraction for the reference video providing pose") + ref_pose_coeff_path, _, _ = self.preprocess_model.generate( + ref_pose, ref_pose_frame_dir + ) + else: + ref_pose_coeff_path = None + + # audio2ceoff + batch = get_data( + first_coeff_path, + args.audio_path, + device, + ref_eyeblink_coeff_path, + still=still, + ) + coeff_path = self.audio_to_coeff.generate( + batch, results_dir, args.pose_style, ref_pose_coeff_path + ) + # coeff2video + print("coeff2video") + data = get_facerender_data( + coeff_path, + crop_pic_path, + first_coeff_path, + args.audio_path, + args.batch_size, + args.input_yaw, + args.input_pitch, + args.input_roll, + expression_scale=args.expression_scale, + still_mode=still, + preprocess=preprocess, + ) + animate_from_coeff.generate( + data, results_dir, args.pic_path, crop_info, + enhancer=enhancer, background_enhancer=args.background_enhancer, + preprocess=preprocess) + + output = "/tmp/out.mp4" + mp4_path = os.path.join(results_dir, [f for f in os.listdir(results_dir) if "enhanced.mp4" in f][0]) + shutil.copy(mp4_path, output) + + return Path(output) + + +def load_default(): + return Namespace( + pose_style=0, + batch_size=2, + expression_scale=1.0, + input_yaw=None, + input_pitch=None, + input_roll=None, + background_enhancer=None, + face3dvis=False, + net_recon="resnet50", + init_path=None, + use_last_fc=False, + bfm_folder="./src/config/", + bfm_model="BFM_model_front.mat", + focal=1015.0, + center=112.0, + camera_d=10.0, + z_near=5.0, + z_far=15.0, + ) diff --git a/quick_demo.ipynb b/quick_demo.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..8b9767d073779093782815c8f9a9b3e4d0a5653d --- /dev/null +++ b/quick_demo.ipynb @@ -0,0 +1,213 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "M74Gs_TjYl_B" + }, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Winfredy/SadTalker/blob/main/quick_demo.ipynb)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "view-in-github" + }, + "source": [ + "### SadTalker：Learning Realistic 3D Motion Coefficients for Stylized Audio-Driven Single Image Talking Face Animation \n", + "\n", + "[arxiv](https://arxiv.org/abs/2211.12194) | [project](https://sadtalker.github.io) | [Github](https://github.com/Winfredy/SadTalker)\n", + "\n", + "Wenxuan Zhang, Xiaodong Cun, Xuan Wang, Yong Zhang, Xi Shen, Yu Guo, Ying Shan, Fei Wang.\n", + "\n", + "Xi'an Jiaotong University, Tencent AI Lab, Ant Group\n", + "\n", + "CVPR 2023\n", + "\n", + "TL;DR: A realistic and stylized talking head video generation method from a single image and audio\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "kA89DV-sKS4i" + }, + "source": [ + "Installation (around 5 mins)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qJ4CplXsYl_E" + }, + "outputs": [], + "source": [ + "### make sure that CUDA is available in Edit -> Nootbook settings -> GPU\n", + "!nvidia-smi --query-gpu=name,memory.total,memory.free --format=csv,noheader" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Mdq6j4E5KQAR" + }, + "outputs": [], + "source": [ + "!update-alternatives --install /usr/local/bin/python3 python3 /usr/bin/python3.8 2 \n", + "!update-alternatives --install /usr/local/bin/python3 python3 /usr/bin/python3.9 1 \n", + "!python --version \n", + "!apt-get update\n", + "!apt install software-properties-common\n", + "!sudo dpkg --remove --force-remove-reinstreq python3-pip python3-setuptools python3-wheel\n", + "!apt-get install python3-pip\n", + "\n", + "print('Git clone project and install requirements...')\n", + "!git clone https://github.com/Winfredy/SadTalker &> /dev/null\n", + "%cd SadTalker \n", + "!export PYTHONPATH=/content/SadTalker:$PYTHONPATH \n", + "!python3.8 -m pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113\n", + "!apt update\n", + "!apt install ffmpeg &> /dev/null \n", + "!python3.8 -m pip install -r requirements.txt" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "DddcKB_nKsnk" + }, + "source": [ + "Download models (1 mins)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eDw3_UN8K2xa" + }, + "outputs": [], + "source": [ + "print('Download pre-trained models...')\n", + "!rm -rf checkpoints\n", + "!bash scripts/download_models.sh" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kK7DYeo7Yl_H" + }, + "outputs": [], + "source": [ + "# borrow from makeittalk\n", + "import ipywidgets as widgets\n", + "import glob\n", + "import matplotlib.pyplot as plt\n", + "print(\"Choose the image name to animate: (saved in folder 'examples/')\")\n", + "img_list = glob.glob1('examples/source_image', '*.png')\n", + "img_list.sort()\n", + "img_list = [item.split('.')[0] for item in img_list]\n", + "default_head_name = widgets.Dropdown(options=img_list, value='full3')\n", + "def on_change(change):\n", + " if change['type'] == 'change' and change['name'] == 'value':\n", + " plt.imshow(plt.imread('examples/source_image/{}.png'.format(default_head_name.value)))\n", + " plt.axis('off')\n", + " plt.show()\n", + "default_head_name.observe(on_change)\n", + "display(default_head_name)\n", + "plt.imshow(plt.imread('examples/source_image/{}.png'.format(default_head_name.value)))\n", + "plt.axis('off')\n", + "plt.show()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "-khNZcnGK4UK" + }, + "source": [ + "Animation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ToBlDusjK5sS" + }, + "outputs": [], + "source": [ + "# selected audio from exmaple/driven_audio\n", + "img = 'examples/source_image/{}.png'.format(default_head_name.value)\n", + "print(img)\n", + "!python3.8 inference.py --driven_audio ./examples/driven_audio/RD_Radio31_000.wav \\\n", + " --source_image {img} \\\n", + " --result_dir ./results --still --preprocess full --enhancer gfpgan" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fAjwGmKKYl_I" + }, + "outputs": [], + "source": [ + "# visualize code from makeittalk\n", + "from IPython.display import HTML\n", + "from base64 import b64encode\n", + "import os, sys\n", + "\n", + "# get the last from results\n", + "\n", + "results = sorted(os.listdir('./results/'))\n", + "\n", + "mp4_name = glob.glob('./results/*.mp4')[0]\n", + "\n", + "mp4 = open('{}'.format(mp4_name),'rb').read()\n", + "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n", + "\n", + "print('Display animation: {}'.format(mp4_name), file=sys.stderr)\n", + "display(HTML(\"\"\"\n", + " \n", + " \"\"\" % data_url))\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [] + }, + "gpuClass": "standard", + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.9.7" + }, + "vscode": { + "interpreter": { + "hash": "db5031b3636a3f037ea48eb287fd3d023feb9033aefc2a9652a92e470fb0851b" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/req.txt b/req.txt new file mode 100644 index 0000000000000000000000000000000000000000..55dd548dba40e8672ab09171936dde32b2094af9 --- /dev/null +++ b/req.txt @@ -0,0 +1,22 @@ +llvmlite==0.38.1 +numpy==1.21.6 +face_alignment==1.3.5 +imageio==2.19.3 +imageio-ffmpeg==0.4.7 +librosa==0.10.0.post2 +numba==0.55.1 +resampy==0.3.1 +pydub==0.25.1 +scipy==1.10.1 +kornia==0.6.8 +tqdm +yacs==0.1.8 +pyyaml +joblib==1.1.0 +scikit-image==0.19.3 +basicsr==1.4.2 +facexlib==0.3.0 +gradio +gfpgan +av +safetensors diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b6505a54eb96698aa1ef05b0715a5cd989c93775 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +numpy==1.23.4 +face_alignment==1.3.5 +imageio==2.19.3 +imageio-ffmpeg==0.4.7 +librosa==0.9.2 # +numba +resampy==0.3.1 +pydub==0.25.1 +scipy==1.10.1 +kornia==0.6.8 +tqdm +yacs==0.1.8 +pyyaml +joblib==1.1.0 +scikit-image==0.19.3 +basicsr==1.4.2 +facexlib==0.3.0 +gradio +gfpgan +av +safetensors diff --git a/requirements3d.txt b/requirements3d.txt new file mode 100644 index 0000000000000000000000000000000000000000..7ad8d9f412e1c7647d8d914b19bb40a658148416 --- /dev/null +++ b/requirements3d.txt @@ -0,0 +1,21 @@ +numpy==1.23.4 +face_alignment==1.3.5 +imageio==2.19.3 +imageio-ffmpeg==0.4.7 +librosa==0.9.2 # +numba +resampy==0.3.1 +pydub==0.25.1 +scipy==1.5.3 +kornia==0.6.8 +tqdm +yacs==0.1.8 +pyyaml +joblib==1.1.0 +scikit-image==0.19.3 +basicsr==1.4.2 +facexlib==0.3.0 +trimesh==3.9.20 +gradio +gfpgan +safetensors \ No newline at end of file diff --git a/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/first_frame_dir/image.mat b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/first_frame_dir/image.mat new file mode 100644 index 0000000000000000000000000000000000000000..2eedcec1d0917097b4a022ee8ac338434e7644ed Binary files /dev/null and b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/first_frame_dir/image.mat differ diff --git a/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/first_frame_dir/image.png b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/first_frame_dir/image.png new file mode 100644 index 0000000000000000000000000000000000000000..befdaf7452fd92d72d6d899bd55a608f83e06cc4 Binary files /dev/null and b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/first_frame_dir/image.png differ diff --git a/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/first_frame_dir/image_landmarks.txt b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/first_frame_dir/image_landmarks.txt new file mode 100644 index 0000000000000000000000000000000000000000..8e4088d870e65e9b195c5f5efbd6e33bf90a57d0 --- /dev/null +++ b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/first_frame_dir/image_landmarks.txt @@ -0,0 +1,136 @@ +5.869531250000000000e+01 +1.211171875000000000e+02 +5.869531250000000000e+01 +1.500703125000000000e+02 +5.869531250000000000e+01 +1.688046875000000000e+02 +6.249218750000000000e+01 +1.926484375000000000e+02 +7.261718750000000000e+01 +2.113828125000000000e+02 +8.527343750000000000e+01 +2.267109375000000000e+02 +1.004609375000000000e+02 +2.386328125000000000e+02 +1.194453125000000000e+02 +2.437421875000000000e+02 +1.384296875000000000e+02 +2.437421875000000000e+02 +1.561484375000000000e+02 +2.437421875000000000e+02 +1.713359375000000000e+02 +2.386328125000000000e+02 +1.839921875000000000e+02 +2.267109375000000000e+02 +1.953828125000000000e+02 +2.113828125000000000e+02 +2.029765625000000000e+02 +1.943515625000000000e+02 +2.093046875000000000e+02 +1.739140625000000000e+02 +2.105703125000000000e+02 +1.534765625000000000e+02 +2.105703125000000000e+02 +1.296328125000000000e+02 +7.514843750000000000e+01 +1.074921875000000000e+02 +8.717187500000000000e+01 +1.015312500000000000e+02 +1.004609375000000000e+02 +1.015312500000000000e+02 +1.143828125000000000e+02 +1.032343750000000000e+02 +1.264062500000000000e+02 +1.074921875000000000e+02 +1.599453125000000000e+02 +1.083437500000000000e+02 +1.719687500000000000e+02 +1.049375000000000000e+02 +1.827265625000000000e+02 +1.040859375000000000e+02 +1.934843750000000000e+02 +1.049375000000000000e+02 +2.042421875000000000e+02 +1.126015625000000000e+02 +1.434921875000000000e+02 +1.279296875000000000e+02 +1.434921875000000000e+02 +1.415546875000000000e+02 +1.434921875000000000e+02 +1.568828125000000000e+02 +1.422265625000000000e+02 +1.705078125000000000e+02 +1.245078125000000000e+02 +1.790234375000000000e+02 +1.321015625000000000e+02 +1.807265625000000000e+02 +1.409609375000000000e+02 +1.824296875000000000e+02 +1.498203125000000000e+02 +1.824296875000000000e+02 +1.574140625000000000e+02 +1.807265625000000000e+02 +8.780468750000000000e+01 +1.245234375000000000e+02 +9.539843750000000000e+01 +1.194140625000000000e+02 +1.131171875000000000e+02 +1.228203125000000000e+02 +1.194453125000000000e+02 +1.296328125000000000e+02 +1.118515625000000000e+02 +1.330390625000000000e+02 +9.413281250000000000e+01 +1.296328125000000000e+02 +1.624765625000000000e+02 +1.330390625000000000e+02 +1.700703125000000000e+02 +1.245234375000000000e+02 +1.865234375000000000e+02 +1.245234375000000000e+02 +1.928515625000000000e+02 +1.296328125000000000e+02 +1.865234375000000000e+02 +1.347421875000000000e+02 +1.700703125000000000e+02 +1.347421875000000000e+02 +1.080546875000000000e+02 +2.011640625000000000e+02 +1.219765625000000000e+02 +1.994609375000000000e+02 +1.358984375000000000e+02 +1.977578125000000000e+02 +1.409609375000000000e+02 +1.977578125000000000e+02 +1.460234375000000000e+02 +1.977578125000000000e+02 +1.574140625000000000e+02 +2.011640625000000000e+02 +1.688046875000000000e+02 +2.045703125000000000e+02 +1.599453125000000000e+02 +2.113828125000000000e+02 +1.510859375000000000e+02 +2.181953125000000000e+02 +1.396953125000000000e+02 +2.181953125000000000e+02 +1.283046875000000000e+02 +2.147890625000000000e+02 +1.181796875000000000e+02 +2.096796875000000000e+02 +1.105859375000000000e+02 +2.011640625000000000e+02 +1.257734375000000000e+02 +2.045703125000000000e+02 +1.409609375000000000e+02 +2.062734375000000000e+02 +1.536171875000000000e+02 +2.062734375000000000e+02 +1.662734375000000000e+02 +2.045703125000000000e+02 +1.536171875000000000e+02 +2.045703125000000000e+02 +1.409609375000000000e+02 +2.062734375000000000e+02 +1.245078125000000000e+02 +2.045703125000000000e+02 diff --git a/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/image##nana_speech_cut_4sec 1-0-100.mat b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/image##nana_speech_cut_4sec 1-0-100.mat new file mode 100644 index 0000000000000000000000000000000000000000..5a3fadf3c66bf541e1cd29fa28eebfd77246d6ba Binary files /dev/null and b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/image##nana_speech_cut_4sec 1-0-100.mat differ diff --git a/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/image##nana_speech_cut_4sec 1-0-100.mp4 b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/image##nana_speech_cut_4sec 1-0-100.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..925f6e3efa0804313cfbb31858277533ece876bc Binary files /dev/null and b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/image##nana_speech_cut_4sec 1-0-100.mp4 differ diff --git a/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/image##nana_speech_cut_4sec 1-0-100.txt b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/image##nana_speech_cut_4sec 1-0-100.txt new file mode 100644 index 0000000000000000000000000000000000000000..6fe9041dc6ff468a5616646386ca3833fe336988 --- /dev/null +++ b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/image##nana_speech_cut_4sec 1-0-100.txt @@ -0,0 +1,92 @@ +-0.4278 0.11022 0.14870 -0.2325 0.01787 -0.4796 -0.0643 -0.2972 0.25088 0.13831 0.30814 0.36796 -0.4101 -0.9264 -0.2901 0.48847 0.39547 0.29986 0.48494 0.48839 -0.2810 -0.2537 0.15106 0.33192 -0.0136 0.32432 -0.2574 -0.3313 -0.2323 -0.5627 -0.2871 0.00675 -0.1613 -0.0024 0.15430 0.04477 0.12900 -0.0454 -0.1306 0.26167 0.46351 -0.0125 0.01944 0.18737 0.10583 0.00578 0.06965 -0.0416 -0.0402 0.17248 0.04880 0.03180 -0.0671 0.00404 0.00068 -0.0339 0.03742 -0.0102 -0.0330 0.01098 -0.0253 -0.0932 0.08337 -0.0208 0.09115 0.08247 -0.0630 -0.0219 0.01700 0.45128 +-0.6047 0.08414 0.05667 -0.4514 -0.1011 -0.5610 -0.1128 -0.3657 0.19877 0.18448 0.32096 0.31758 -0.4484 -0.9779 -0.1762 0.30501 0.43134 0.29158 0.48953 0.39711 -0.2334 -0.2356 0.25520 0.29294 -0.0044 0.34578 -0.2350 -0.4119 -0.2798 -0.5582 -0.2647 -0.0197 -0.1652 0.00229 0.11013 0.00794 0.15928 -0.0454 -0.1168 0.24610 0.44520 -0.0152 -0.0053 0.21020 0.12585 0.01247 0.06212 -0.0647 -0.0547 0.18453 0.04046 0.03551 -0.0641 0.01368 -0.0024 -0.0338 0.02966 -0.0058 -0.0372 0.00953 -0.0240 -0.0891 0.07743 -0.0278 0.09449 0.08686 -0.0614 -0.0229 0.01754 0.43938 +-0.5478 0.09706 0.10812 -0.3731 -0.0679 -0.5359 -0.0821 -0.3461 0.21009 0.07680 0.31611 0.31124 -0.4905 -1.0944 -0.2447 0.38574 0.42531 0.29591 0.49722 0.38059 -0.2290 -0.2376 0.31737 0.28803 -0.1066 0.35436 -0.2078 -0.3836 -0.2861 -0.5624 -0.2746 0.00229 -0.1760 8.16397 0.09381 -0.0008 0.16676 -0.0294 -0.1687 0.26102 0.48414 -0.0164 0.01131 0.22299 0.11796 0.00411 0.06174 -0.0621 -0.0559 0.17926 0.03615 0.03289 -0.0689 0.01182 -0.0011 -0.0257 0.02959 -0.0027 -0.0421 0.00774 -0.0214 -0.0895 0.08284 -0.0317 0.09736 0.09032 -0.0601 -0.0239 0.01794 0.42958 +-0.2675 0.11166 0.22797 -0.1856 0.02059 -0.3494 -0.0880 -0.3074 0.23520 0.07450 0.15885 0.41287 -0.4531 -1.1041 -0.2621 0.56769 0.42512 0.31168 0.48984 0.42658 -0.3393 -0.2632 0.25678 0.27073 -0.0900 0.35873 -0.2487 -0.2793 -0.2187 -0.5406 -0.2944 0.00695 -0.1854 0.02065 0.08260 0.02136 0.12730 -0.0443 -0.1332 0.28243 0.47592 -0.0318 0.02569 0.23533 0.11238 0.00536 0.06400 -0.0457 -0.0387 0.18362 0.04547 0.03201 -0.0718 0.00610 0.00523 -0.0352 0.03362 0.00150 -0.0425 -0.0039 -0.0310 -0.0881 0.09511 -0.0246 0.09976 0.09285 -0.0590 -0.0247 0.01820 0.42189 +-0.1524 0.18810 0.33753 -0.0271 0.13939 -0.3338 -0.0752 -0.3214 0.23749 0.17323 0.17151 0.46752 -0.4210 -1.0832 -0.2713 0.68544 0.37131 0.34510 0.47823 0.48616 -0.3563 -0.2457 0.14579 0.25126 -0.0386 0.33201 -0.2915 -0.2563 -0.1989 -0.5445 -0.3092 0.00764 -0.1873 0.00805 0.10022 0.05740 0.10738 -0.0381 -0.1097 0.29417 0.45072 -0.0178 0.04653 0.23254 0.11465 0.01452 0.06548 -0.0297 -0.0369 0.17299 0.06006 0.03024 -0.0660 -0.0015 0.00642 -0.0344 0.03692 -0.0014 -0.0369 0.00648 -0.0326 -0.1018 0.08740 -0.0223 0.10169 0.09445 -0.0583 -0.0256 0.01831 0.41629 +-0.1408 0.15545 0.30327 0.04509 0.21360 -0.2921 -0.0529 -0.3419 0.20999 0.14100 0.14939 0.43828 -0.4322 -1.0968 -0.2276 0.71601 0.36478 0.32621 0.48872 0.46921 -0.4002 -0.2232 0.08338 0.28667 0.04843 0.29937 -0.1731 -0.2345 -0.1800 -0.4951 -0.3413 0.04855 -0.1989 0.01741 0.14823 0.10881 0.10743 -0.0248 -0.1086 0.29052 0.42423 -0.0025 0.02763 0.22532 0.11852 0.00562 0.05018 -0.0315 -0.0223 0.16076 0.06299 0.03202 -0.0611 0.01039 0.00874 -0.0282 0.03472 -0.0024 -0.0351 0.00815 -0.0304 -0.1007 0.08838 -0.0255 0.10315 0.09513 -0.0578 -0.0264 0.01828 0.41281 +-0.1851 0.15016 0.23173 -0.0017 0.14779 -0.2681 -0.0758 -0.3332 0.20381 0.06855 0.11942 0.40373 -0.4558 -1.0899 -0.2136 0.64226 0.40913 0.29731 0.50293 0.46653 -0.3374 -0.2392 0.10374 0.30071 0.05212 0.30046 -0.1659 -0.2428 -0.2335 -0.4752 -0.3349 0.07078 -0.2007 0.01031 0.14133 0.08745 0.13601 -0.0223 -0.1197 0.27354 0.40321 -0.0228 0.02289 0.23224 0.11198 0.01144 0.04222 -0.0441 -0.0332 0.16660 0.06003 0.03288 -0.0641 0.00879 0.00732 -0.0264 0.03606 0.00148 -0.0348 0.00575 -0.0304 -0.1026 0.09057 -0.0289 0.10415 0.09488 -0.0576 -0.0271 0.01811 0.41142 +-0.2925 0.14622 0.15957 0.02663 0.18046 -0.3412 -0.0969 -0.2966 0.21508 0.03252 0.08580 0.47675 -0.4338 -0.9522 -0.0751 0.50279 0.40658 0.29461 0.49896 0.48020 -0.2676 -0.2107 0.17760 0.31977 0.00666 0.31242 -0.1709 -0.2655 -0.2154 -0.5071 -0.3236 0.00494 -0.1795 0.02192 0.14150 0.02319 0.13342 -0.0439 -0.1022 0.25994 0.40601 -0.0206 -0.0097 0.22563 0.11108 0.00582 0.05014 -0.0439 -0.0359 0.16612 0.06320 0.03061 -0.0674 0.00238 0.00219 -0.0319 0.03854 0.00399 -0.0341 0.00367 -0.0299 -0.0954 0.08650 -0.0180 0.10479 0.09298 -0.0576 -0.0272 0.01684 0.41583 +-0.1271 0.19498 0.22263 0.08480 0.23885 -0.2846 -0.0719 -0.1606 0.27068 -0.0059 0.13887 0.50770 -0.3255 -0.7612 0.06380 0.54435 0.39413 0.25914 0.46002 0.45377 -0.1306 -0.1915 0.17051 0.34794 0.01742 0.30071 -0.1042 -0.3540 -0.2782 -0.5036 -0.3076 -0.0258 -0.1282 0.01798 0.22497 -0.1021 0.17035 -0.0332 -0.1285 0.27023 0.37360 0.00200 -0.0093 0.21561 0.10799 0.00110 0.05492 -0.0480 -0.0456 0.16576 0.03992 0.02569 -0.0744 0.01004 -0.0116 -0.0278 0.03087 -0.0055 -0.0440 -0.0032 -0.0245 -0.0858 0.09107 -0.0225 0.10361 0.09069 -0.0580 -0.0279 0.01669 0.41892 +-0.4264 0.16537 0.09336 -0.1224 0.16498 -0.4622 -0.0981 -0.2447 0.24155 0.06092 0.18259 0.45880 -0.3874 -0.9176 0.03100 0.43226 0.40750 0.28312 0.48072 0.43622 -0.1931 -0.2583 0.20142 0.30962 -0.0455 0.31758 -0.1711 -0.3393 -0.2400 -0.5405 -0.3128 -0.0151 -0.1512 0.01912 0.18398 -0.0623 0.15801 -0.0462 -0.1274 0.28796 0.39080 0.00143 -0.0300 0.21285 0.11937 0.00918 0.05628 -0.0605 -0.0405 0.16888 0.04447 0.03317 -0.0782 0.01220 0.00210 -0.0228 0.03151 -0.0047 -0.0350 0.00965 -0.0248 -0.0911 0.08712 -0.0266 0.10335 0.08732 -0.0586 -0.0288 0.01663 0.42039 +-0.4551 0.15696 0.13984 -0.3316 -0.0378 -0.4574 -0.1140 -0.2889 0.21186 0.04093 0.26414 0.34365 -0.4518 -1.0417 -0.1114 0.38321 0.41892 0.26930 0.51217 0.41605 -0.1359 -0.2205 0.32142 0.33678 -0.0883 0.33333 -0.1637 -0.4346 -0.3047 -0.5639 -0.2864 -0.0016 -0.1627 0.00193 0.12415 -0.0524 0.20288 -0.0255 -0.1604 0.28161 0.44703 -0.0183 0.00820 0.23442 0.12266 0.01473 0.06515 -0.0740 -0.0530 0.17889 0.03844 0.02911 -0.0729 0.01219 -0.0039 -0.0301 0.02915 -0.0018 -0.0315 0.00826 -0.0240 -0.0897 0.07670 -0.0289 0.10310 0.08388 -0.0594 -0.0297 0.01605 0.43075 +-0.2059 0.18628 0.27607 -0.1854 0.03176 -0.2923 -0.1429 -0.2518 0.24091 0.02850 0.16540 0.38111 -0.4288 -1.0206 -0.1443 0.53315 0.40819 0.30278 0.52216 0.42676 -0.2194 -0.2478 0.28951 0.27807 -0.1096 0.34494 -0.2380 -0.3521 -0.2571 -0.5534 -0.3047 0.01155 -0.1502 0.01345 0.11922 -0.0225 0.20452 -0.0192 -0.1437 0.25167 0.47113 -0.0348 0.03857 0.23954 0.11511 0.00398 0.06001 -0.0436 -0.0492 0.17407 0.03657 0.02724 -0.0737 0.00347 -0.0004 -0.0273 0.03246 -0.0030 -0.0344 -0.0010 -0.0302 -0.0888 0.08353 -0.0262 0.10309 0.07905 -0.0603 -0.0305 0.01551 0.43698 +-0.1739 0.18221 0.36656 -0.2517 -0.0406 -0.3412 -0.1286 -0.2609 0.26470 0.13812 0.26674 0.45282 -0.4335 -1.0174 -0.2697 0.63494 0.37678 0.32056 0.50090 0.47926 -0.2793 -0.2794 0.21741 0.23237 -0.1244 0.33794 -0.3423 -0.3031 -0.2615 -0.5739 -0.2812 0.01686 -0.1453 0.00800 0.13714 -0.0097 0.15979 -0.0151 -0.1441 0.25745 0.49888 -0.0305 0.06904 0.22840 0.09342 0.00544 0.05968 -0.0293 -0.0378 0.15911 0.05699 0.03009 -0.0655 0.00581 0.00511 -0.0333 0.03641 -0.0023 -0.0426 0.00189 -0.0244 -0.0867 0.09260 -0.0217 0.10300 0.07640 -0.0616 -0.0309 0.01523 0.43704 +-0.3693 0.12700 0.19849 -0.2380 -0.0024 -0.4207 -0.1158 -0.3207 0.22051 0.18818 0.25876 0.43669 -0.4496 -0.9918 -0.3207 0.54615 0.38364 0.31791 0.50976 0.47373 -0.3235 -0.2607 0.21014 0.25997 -0.0543 0.31930 -0.2827 -0.2876 -0.1998 -0.5540 -0.2874 0.02212 -0.1692 -0.0005 0.11720 0.03522 0.12654 -0.0368 -0.1227 0.27521 0.49937 -0.0244 0.04363 0.22815 0.11238 0.00976 0.06138 -0.0432 -0.0488 0.15936 0.04571 0.03522 -0.0667 0.01205 0.00786 -0.0347 0.03610 -0.0013 -0.0376 0.00528 -0.0174 -0.0972 0.09006 -0.0234 0.10282 0.07363 -0.0629 -0.0315 0.01549 0.42565 +-0.5624 0.10881 0.12938 -0.4219 -0.1064 -0.5310 -0.1160 -0.3761 0.24174 0.18273 0.27401 0.37444 -0.4561 -1.0359 -0.2382 0.43382 0.39954 0.31458 0.50087 0.44361 -0.2326 -0.2527 0.24356 0.27985 -0.0275 0.32014 -0.2464 -0.3926 -0.2925 -0.5647 -0.2680 0.00705 -0.1616 0.00567 0.07899 0.01652 0.17168 -0.0229 -0.1487 0.25758 0.46129 -0.0310 0.00312 0.21425 0.11209 0.00993 0.05917 -0.0498 -0.0621 0.19902 0.03973 0.02972 -0.0640 0.01287 -0.0012 -0.0318 0.03059 -0.0014 -0.0428 0.00915 -0.0240 -0.0825 0.09015 -0.0200 0.10246 0.07244 -0.0639 -0.0318 0.01526 0.41764 +-0.6071 0.09940 0.10883 -0.5073 -0.1400 -0.5057 -0.1220 -0.3700 0.19464 0.19938 0.29688 0.30018 -0.4778 -1.0880 -0.2789 0.36340 0.41318 0.30132 0.49899 0.40491 -0.2430 -0.2701 0.28837 0.22825 -0.0739 0.35796 -0.2735 -0.3812 -0.2656 -0.5770 -0.2432 -0.0186 -0.1688 0.00809 0.05270 0.02433 0.15369 -0.0398 -0.1218 0.24572 0.47652 -0.0293 0.02468 0.22057 0.11973 0.00829 0.06097 -0.0628 -0.0614 0.18730 0.03384 0.03631 -0.0677 0.00877 -0.0026 -0.0352 0.03176 0.00069 -0.0422 0.00760 -0.0221 -0.0897 0.09105 -0.0235 0.10138 0.07109 -0.0647 -0.0322 0.01469 0.41954 +-0.5365 0.12490 0.14866 -0.4650 -0.0998 -0.4865 -0.0931 -0.3349 0.22426 0.12252 0.23958 0.29015 -0.4586 -1.1193 -0.2339 0.41595 0.41871 0.28909 0.51463 0.38167 -0.2670 -0.2406 0.35884 0.25657 -0.1077 0.36047 -0.2424 -0.3805 -0.2350 -0.5624 -0.2610 -0.0098 -0.1738 0.00645 0.04479 -0.0088 0.15696 -0.0313 -0.1340 0.27044 0.50637 -0.0250 0.03705 0.24679 0.11049 0.00670 0.06172 -0.0517 -0.0511 0.18339 0.03640 0.03573 -0.0707 0.00996 -0.0038 -0.0307 0.02585 -0.0024 -0.0370 0.00187 -0.0203 -0.0873 0.08631 -0.0286 0.09846 0.07115 -0.0653 -0.0322 0.01500 0.41096 +-0.4424 0.14245 0.21138 -0.3464 -0.0599 -0.4173 -0.1167 -0.2990 0.24154 0.06038 0.20042 0.34015 -0.4769 -1.0879 -0.1899 0.46388 0.42708 0.27275 0.50190 0.37831 -0.2475 -0.2174 0.35390 0.28440 -0.1242 0.35415 -0.2381 -0.3730 -0.2593 -0.5652 -0.2607 0.00273 -0.1663 0.00288 0.05000 -0.0357 0.17930 -0.0209 -0.1403 0.25026 0.51882 -0.0387 0.02466 0.24651 0.10577 0.00691 0.06206 -0.0511 -0.0540 0.17748 0.03421 0.02895 -0.0683 0.00450 -0.0027 -0.0370 0.02990 0.00077 -0.0406 0.00527 -0.0209 -0.0839 0.09313 -0.0179 0.09558 0.07001 -0.0656 -0.0322 0.01560 0.39771 +-0.2512 0.15307 0.23998 -0.1717 0.03049 -0.2933 -0.1204 -0.2839 0.23053 0.05831 0.16079 0.44192 -0.4216 -1.0289 -0.1581 0.51696 0.41011 0.30935 0.48770 0.42335 -0.2758 -0.2368 0.29953 0.27955 -0.1198 0.35379 -0.2612 -0.3313 -0.2308 -0.5695 -0.2714 0.00473 -0.1518 0.00318 0.10216 -0.0111 0.15717 -0.0383 -0.1346 0.27371 0.48005 -0.0339 0.04564 0.23789 0.10591 0.00318 0.06078 -0.0442 -0.0415 0.15981 0.03362 0.03145 -0.0677 0.00364 0.00264 -0.0345 0.03101 -0.0018 -0.0379 -0.0023 -0.0231 -0.0925 0.08654 -0.0198 0.09178 0.06944 -0.0658 -0.0326 0.01579 0.40346 +-0.1813 0.20084 0.33178 -0.1319 0.07677 -0.2856 -0.0974 -0.2760 0.26124 0.17995 0.23163 0.47058 -0.4395 -1.0274 -0.2315 0.65521 0.36408 0.30679 0.48379 0.48419 -0.2954 -0.2415 0.19962 0.26145 -0.1246 0.33830 -0.2821 -0.3070 -0.2163 -0.5752 -0.2880 0.01502 -0.1558 0.01405 0.13489 0.03269 0.14293 -0.0425 -0.1253 0.28631 0.46395 -0.0339 0.04042 0.23914 0.10832 0.00562 0.06225 -0.0328 -0.0315 0.16434 0.05106 0.02817 -0.0656 -0.0031 0.00651 -0.0333 0.03201 -0.0035 -0.0388 0.00019 -0.0275 -0.0942 0.09330 -0.0166 0.08856 0.06771 -0.0654 -0.0329 0.01587 0.40349 +-0.2174 0.15765 0.30799 -0.1567 0.02414 -0.3448 -0.1083 -0.3256 0.22397 0.18176 0.25010 0.44758 -0.4429 -1.0640 -0.3157 0.64885 0.35809 0.35642 0.48127 0.49478 -0.3536 -0.2455 0.14041 0.24553 -0.0874 0.31985 -0.2990 -0.2777 -0.1942 -0.5566 -0.2963 0.02995 -0.1601 0.01650 0.13946 0.06790 0.11102 -0.0475 -0.1136 0.28841 0.47066 -0.0305 0.04265 0.22132 0.11458 0.00411 0.06062 -0.0409 -0.0258 0.17078 0.05630 0.02653 -0.0658 0.00031 0.00792 -0.0300 0.03430 -0.0006 -0.0407 0.00504 -0.0291 -0.0938 0.09507 -0.0221 0.08600 0.06757 -0.0645 -0.0330 0.01639 0.40026 +-0.4239 0.10750 0.15411 -0.2467 -0.0255 -0.4226 -0.1009 -0.3687 0.20937 0.08254 0.25043 0.38236 -0.4739 -1.1307 -0.2774 0.52513 0.42220 0.29422 0.48618 0.45788 -0.3116 -0.2451 0.21603 0.28592 -0.0622 0.33978 -0.2247 -0.2726 -0.2286 -0.5316 -0.2949 0.01773 -0.1946 0.01802 0.12702 0.06645 0.15332 -0.0533 -0.1375 0.25927 0.45526 -0.0201 0.01942 0.22179 0.11722 0.00667 0.05948 -0.0543 -0.0412 0.17858 0.05167 0.03143 -0.0696 0.00372 0.00337 -0.0271 0.03224 0.00058 -0.0362 0.00368 -0.0281 -0.0945 0.09019 -0.0248 0.08542 0.06657 -0.0637 -0.0334 0.01671 0.40151 +-0.4056 0.12798 0.17823 -0.2546 -0.0068 -0.4530 -0.1387 -0.3222 0.23229 0.08520 0.22629 0.39306 -0.4721 -1.0170 -0.1830 0.45541 0.42442 0.29133 0.49979 0.37890 -0.2323 -0.2475 0.31429 0.27910 -0.1250 0.33462 -0.2529 -0.3400 -0.2562 -0.5455 -0.2862 0.00649 -0.1891 0.01232 0.10530 -0.0254 0.14587 -0.0371 -0.1430 0.23801 0.47298 -0.0183 0.02117 0.22661 0.11955 0.00735 0.06228 -0.0504 -0.0462 0.17883 0.04751 0.03203 -0.0639 0.00960 0.00098 -0.0320 0.03566 0.00389 -0.0331 0.00127 -0.0217 -0.0932 0.08503 -0.0241 0.08687 0.06666 -0.0630 -0.0337 0.01580 0.41013 +-0.2553 0.18608 0.25455 -0.1620 0.07017 -0.2950 -0.1091 -0.2359 0.26779 0.02561 0.10412 0.47189 -0.4098 -0.9068 -0.0623 0.45619 0.42932 0.28751 0.48666 0.40229 -0.2239 -0.2189 0.31506 0.27670 -0.1303 0.35651 -0.2937 -0.3062 -0.2179 -0.5673 -0.2726 -0.0297 -0.1477 0.01933 0.09591 -0.0655 0.12171 -0.0393 -0.1107 0.26964 0.48774 -0.0414 0.04251 0.23567 0.09863 0.00301 0.06130 -0.0466 -0.0438 0.17344 0.04784 0.03252 -0.0783 -0.0001 -0.0029 -0.0324 0.03180 -0.0040 -0.0381 0.00140 -0.0249 -0.0881 0.09040 -0.0232 0.08977 0.06715 -0.0625 -0.0342 0.01490 0.41926 +-0.2737 0.18401 0.26461 -0.1796 0.09481 -0.2896 -0.1062 -0.2565 0.25462 0.07530 0.12636 0.47335 -0.4033 -0.9249 -0.1121 0.49646 0.41493 0.30432 0.48114 0.45196 -0.2667 -0.1959 0.29579 0.25255 -0.1557 0.37068 -0.3704 -0.2480 -0.1993 -0.5935 -0.2596 -0.0348 -0.1479 0.01410 0.03987 -0.0527 0.10154 -0.0509 -0.0958 0.27474 0.51860 -0.0449 0.05797 0.23778 0.09051 -0.0004 0.06414 -0.0430 -0.0431 0.17707 0.04371 0.03266 -0.0794 0.00020 -0.0014 -0.0343 0.03199 -0.0048 -0.0398 -0.0033 -0.0213 -0.0860 0.08476 -0.0234 0.09305 0.07030 -0.0623 -0.0342 0.01451 0.40872 +-0.3623 0.16821 0.22216 -0.1522 0.10808 -0.4145 -0.1187 -0.3112 0.22564 0.10828 0.21872 0.46398 -0.4569 -0.9872 -0.1431 0.47575 0.40768 0.32767 0.48052 0.47825 -0.2300 -0.2239 0.24473 0.28339 -0.1424 0.34417 -0.3087 -0.2779 -0.2143 -0.5620 -0.2677 -0.0163 -0.1605 0.01533 0.10592 -0.0171 0.11813 -0.0562 -0.1134 0.27462 0.46984 -0.0265 0.03199 0.23493 0.10770 0.00303 0.06701 -0.0420 -0.0428 0.18010 0.05177 0.02863 -0.0659 0.00883 0.00034 -0.0327 0.03371 0.00074 -0.0381 0.00818 -0.0305 -0.0875 0.08568 -0.0236 0.09667 0.07297 -0.0620 -0.0342 0.01450 0.38812 +-0.4669 0.11417 0.09248 -0.0324 0.18022 -0.4511 -0.1197 -0.2658 0.23098 0.05701 0.20067 0.52876 -0.3993 -0.8972 -0.0377 0.41769 0.38132 0.30436 0.47103 0.48379 -0.2755 -0.2406 0.19609 0.26281 -0.1164 0.34191 -0.2658 -0.2214 -0.1751 -0.5350 -0.2750 -0.0178 -0.1569 0.02524 0.13879 -0.0229 0.04708 -0.0769 -0.1057 0.27832 0.41236 -0.0193 -0.0027 0.21852 0.13220 0.01169 0.06471 -0.0521 -0.0434 0.17511 0.05235 0.03175 -0.0732 0.00490 0.00093 -0.0359 0.03784 -0.0001 -0.0368 0.00336 -0.0240 -0.0866 0.08695 -0.0224 0.09920 0.07657 -0.0622 -0.0339 0.01398 0.38588 +-0.2363 0.20379 0.23276 0.16472 0.25789 -0.2894 -0.0931 -0.1882 0.28261 -0.0081 0.01075 0.63079 -0.3404 -0.6996 0.17862 0.54810 0.42108 0.33312 0.47687 0.51390 -0.2705 -0.2470 0.20166 0.25305 -0.1672 0.35379 -0.3612 -0.1622 -0.1719 -0.5586 -0.2824 -0.0438 -0.1524 0.02024 0.12729 -0.0614 0.02531 -0.0632 -0.0593 0.29124 0.43858 -0.0347 0.03099 0.22458 0.11336 0.01973 0.06125 -0.0227 -0.0460 0.16938 0.04839 0.02817 -0.0740 0.00787 -0.0031 -0.0271 0.02579 -0.0135 -0.0314 0.00941 -0.0265 -0.0833 0.07584 -0.0203 0.09814 0.07899 -0.0622 -0.0338 0.01328 0.39567 +-0.1172 0.28111 0.26640 0.27336 0.28858 -0.1458 -0.0668 -0.1651 0.27683 -0.0570 -0.1257 0.57224 -0.2900 -0.6676 0.32683 0.51150 0.43298 0.28531 0.50563 0.44293 -0.2678 -0.1684 0.18398 0.24654 -0.0816 0.33949 -0.2514 -0.2789 -0.1683 -0.4769 -0.2890 -0.0403 -0.1455 0.01972 0.13157 -0.0783 0.06693 -0.0396 -0.0815 0.27533 0.43132 -0.0279 0.02104 0.22295 0.10848 0.00094 0.05602 -0.0276 -0.0299 0.16934 0.03928 0.02978 -0.0732 -0.0022 -0.0075 -0.0318 0.04011 -0.0013 -0.0461 -0.0035 -0.0249 -0.0879 0.09076 -0.0205 0.09509 0.08117 -0.0620 -0.0330 0.01376 0.39947 +-0.0742 0.22965 0.29051 0.21930 0.18844 -0.0426 -0.1166 -0.1782 0.29306 -0.0438 -0.0987 0.49245 -0.3470 -0.8266 0.22794 0.57200 0.45869 0.22833 0.53903 0.37301 -0.1860 -0.1822 0.26961 0.31214 -0.0457 0.31921 -0.1907 -0.3822 -0.2660 -0.5157 -0.3048 0.02541 -0.1595 0.02244 0.17318 -0.0888 0.17812 -0.0073 -0.1329 0.26141 0.42855 -0.0290 0.02792 0.23666 0.10249 0.00552 0.05304 -0.0254 -0.0315 0.17419 0.02385 0.03337 -0.0760 0.00399 -0.0010 -0.0234 0.03873 0.00234 -0.0293 0.00196 -0.0243 -0.0848 0.08720 -0.0320 0.09273 0.08160 -0.0619 -0.0320 0.01485 0.40137 +-0.0354 0.23795 0.31755 0.03577 0.17068 -0.1652 -0.1005 -0.2719 0.26607 0.03298 0.07017 0.43961 -0.3966 -0.9771 -0.0262 0.62253 0.36767 0.28845 0.50863 0.45682 -0.2405 -0.2268 0.17077 0.30474 -0.0752 0.30945 -0.2158 -0.3604 -0.2831 -0.5230 -0.3143 0.03655 -0.1403 0.02536 0.18433 -0.0220 0.17198 -0.0196 -0.1315 0.27980 0.44052 -0.0241 0.03421 0.22212 0.11029 0.01313 0.04911 -0.0407 -0.0292 0.16188 0.05201 0.03303 -0.0700 -0.0008 -0.0040 -0.0197 0.03587 -0.0021 -0.0339 0.00421 -0.0289 -0.1019 0.08568 -0.0288 0.09021 0.08245 -0.0616 -0.0311 0.01586 0.42872 +-0.1092 0.18521 0.31886 -0.0994 0.09587 -0.2703 -0.0779 -0.2925 0.22594 0.12071 0.14991 0.41062 -0.4282 -1.0612 -0.2226 0.69370 0.35763 0.33411 0.50030 0.46030 -0.2677 -0.2203 0.16141 0.25645 -0.0392 0.31351 -0.2158 -0.3018 -0.2356 -0.5378 -0.3045 0.02665 -0.1693 0.01095 0.15807 0.04644 0.16261 -0.0433 -0.1134 0.27829 0.43283 -0.0108 0.01653 0.23550 0.11708 0.01711 0.05531 -0.0297 -0.0334 0.17244 0.05510 0.03342 -0.0601 -0.0011 0.00515 -0.0254 0.03466 -0.0027 -0.0353 0.00685 -0.0255 -0.1063 0.09143 -0.0240 0.08920 0.08228 -0.0609 -0.0298 0.01693 0.43731 +-0.3664 0.09350 0.14949 -0.1060 0.07741 -0.4506 -0.0785 -0.3382 0.21836 0.06716 0.18636 0.39382 -0.4350 -1.0621 -0.1936 0.53348 0.39502 0.29580 0.50781 0.46745 -0.3032 -0.2295 0.14315 0.32530 -0.0243 0.32465 -0.2240 -0.2509 -0.2037 -0.5235 -0.3153 0.02115 -0.1952 0.02756 0.12924 0.07174 0.15561 -0.0480 -0.1241 0.28958 0.44210 -0.0238 0.00812 0.22353 0.11177 -0.0006 0.05469 -0.0464 -0.0386 0.17192 0.04963 0.03499 -0.0756 0.00099 0.00280 -0.0303 0.03609 -0.0035 -0.0356 0.00693 -0.0247 -0.0983 0.08733 -0.0241 0.08886 0.08302 -0.0599 -0.0284 0.01821 0.45009 +-0.2754 0.14652 0.13083 0.18732 0.32005 -0.3316 -0.1026 -0.2434 0.26775 0.00502 0.08195 0.56858 -0.3346 -0.8004 0.16627 0.45958 0.39346 0.26848 0.45124 0.50364 -0.2926 -0.2237 0.14316 0.28191 -0.0777 0.34321 -0.2937 -0.1818 -0.1859 -0.5225 -0.3039 -0.0219 -0.1478 0.03981 0.20020 -0.0611 0.07762 -0.0699 -0.0983 0.26541 0.38575 -0.0094 -0.0079 0.19989 0.13111 0.00969 0.06371 -0.0533 -0.0433 0.16935 0.04709 0.03061 -0.0707 0.00680 0.00107 -0.0277 0.03598 -0.0051 -0.0282 0.01101 -0.0236 -0.0894 0.08245 -0.0220 0.08950 0.08320 -0.0593 -0.0275 0.01882 0.47367 +-0.0726 0.25032 0.23531 0.24845 0.29652 -0.2910 -0.0597 -0.1577 0.28034 -0.0034 -0.0156 0.54561 -0.2812 -0.6423 0.24744 0.51690 0.37448 0.26885 0.44626 0.46903 -0.2389 -0.1620 0.11133 0.27950 -0.1130 0.32575 -0.2523 -0.3057 -0.2052 -0.4929 -0.3152 -0.0528 -0.1320 0.02918 0.20599 -0.0643 0.07607 -0.0653 -0.0977 0.28898 0.39590 -0.0217 0.01662 0.22038 0.11150 -0.0035 0.06711 -0.0457 -0.0211 0.16960 0.04466 0.02308 -0.0784 0.00494 -0.0039 -0.0317 0.03650 0.00118 -0.0462 -0.0017 -0.0202 -0.0973 0.07992 -0.0205 0.09130 0.08462 -0.0587 -0.0265 0.01838 0.48822 +-0.2357 0.20115 0.16657 -0.0206 0.18462 -0.4243 -0.0389 -0.2092 0.29213 0.06364 0.17581 0.47290 -0.3587 -0.7546 -0.0420 0.48750 0.37025 0.32437 0.47234 0.50525 -0.2386 -0.2355 0.12329 0.32374 -0.0550 0.33659 -0.2701 -0.3112 -0.2626 -0.5712 -0.2974 -0.0368 -0.1139 0.00527 0.19260 -0.0481 0.12918 -0.0431 -0.1092 0.28651 0.44395 -0.0063 0.01545 0.21420 0.09882 0.01541 0.06169 -0.0298 -0.0414 0.19490 0.04457 0.02685 -0.0787 0.00911 -0.0098 -0.0225 0.02374 -0.0043 -0.0364 0.00525 -0.0228 -0.0831 0.07325 -0.0321 0.09392 0.08565 -0.0583 -0.0259 0.01749 0.50383 +-0.5365 0.09865 0.05659 -0.1929 0.12884 -0.4923 -0.0509 -0.2651 0.24771 0.07336 0.26792 0.44726 -0.4033 -0.9302 -0.0845 0.42126 0.38984 0.29981 0.46817 0.49095 -0.2237 -0.2409 0.20188 0.33072 -0.1428 0.34455 -0.2490 -0.3103 -0.2273 -0.5620 -0.2851 -0.0277 -0.1387 0.03462 0.17072 -0.0423 0.09609 -0.0596 -0.1226 0.28117 0.41377 0.00612 -0.0246 0.19211 0.11648 0.00599 0.05831 -0.0563 -0.0522 0.17799 0.04706 0.03036 -0.0710 0.00604 -0.0053 -0.0245 0.03214 -0.0026 -0.0367 0.00702 -0.0208 -0.0901 0.07759 -0.0290 0.09596 0.08744 -0.0579 -0.0254 0.01712 0.49306 +-0.4934 0.14048 0.14095 -0.4281 -0.0379 -0.5936 -0.1271 -0.2069 0.19809 0.08742 0.29252 0.35646 -0.3630 -1.0128 -0.1140 0.23589 0.52382 0.10795 0.56801 0.10902 -0.1263 -0.4619 0.07261 0.34429 -0.1684 0.44305 -0.1672 -0.5079 -0.2272 -0.5855 -0.1546 -0.0849 -0.2541 -0.1663 0.04168 -0.0690 0.07084 -0.0461 -0.2458 0.18559 0.51687 0.13548 -0.1188 0.29145 0.05414 0.02779 0.03503 -0.0350 -0.0692 0.16711 0.04158 0.03687 -0.0790 0.02973 0.00031 -0.0393 0.02882 -0.0140 -0.0359 0.01266 -0.0189 -0.0876 0.06625 -0.0270 0.09871 0.08745 -0.0583 -0.0253 0.01665 0.48261 +-0.2075 0.24472 0.38509 -0.1729 0.05083 -0.3549 -0.1662 -0.0932 0.24581 0.02167 0.12692 0.43739 -0.2654 -0.9166 0.08354 0.36117 0.60016 -0.0271 0.61790 -0.0849 -0.0236 -0.6619 -0.1495 0.36346 -0.2804 0.53498 -0.2305 -0.6281 -0.1932 -0.6168 -0.0262 -0.0991 -0.2925 -0.2941 0.03145 -0.1727 0.02184 -0.0445 -0.3624 0.11187 0.60219 0.20528 -0.1850 0.36783 -0.0106 0.05570 0.01976 -0.0139 -0.0809 0.16936 0.05904 0.03829 -0.0748 0.02734 -0.0006 -0.0296 0.02084 -0.0255 -0.0413 0.02078 -0.0102 -0.0773 0.05344 -0.0239 0.10065 0.08641 -0.0601 -0.0255 0.01532 0.48417 +0.25732 0.35982 0.65124 0.16777 0.16200 -0.0535 -0.1412 0.03784 0.27999 -0.0144 -0.1999 0.39855 -0.0692 -0.5781 0.55408 0.29766 0.61539 -0.1753 0.63479 -0.3092 0.02108 -0.5787 -0.1223 0.34661 -0.1833 0.52220 -0.0413 -0.8179 -0.1992 -0.5344 -0.0217 -0.0803 -0.3251 -0.3337 0.08957 -0.2231 0.07760 -0.0041 -0.3688 0.07064 0.55906 0.23997 -0.1702 0.40864 -0.0217 0.04789 -0.0031 0.04237 -0.0525 0.16103 0.04291 0.04302 -0.0695 0.03256 0.01077 -0.0179 0.00146 -0.0281 -0.0435 0.02778 -0.0077 -0.0877 0.05321 -0.0245 0.10159 0.08380 -0.0623 -0.0260 0.01340 0.47661 +-0.0144 0.29360 0.40050 0.11714 0.20686 -0.2856 -0.1026 -0.0779 0.20375 -0.0225 -0.0341 0.30972 -0.1725 -0.7500 0.37858 0.24277 0.60041 -0.1433 0.61563 -0.1936 0.00791 -0.5725 -0.1535 0.39628 -0.0952 0.48936 -0.0167 -0.7676 -0.2251 -0.5516 -0.0689 -0.0728 -0.3201 -0.2886 0.15685 -0.1806 0.08596 -0.0158 -0.3829 0.14987 0.52171 0.24165 -0.1873 0.37072 -0.0102 0.05017 0.00820 0.00021 -0.0699 0.15886 0.05178 0.04261 -0.0763 0.03380 0.00828 -0.0271 0.02355 -0.0279 -0.0427 0.01832 -0.0054 -0.0865 0.05486 -0.0291 0.10261 0.08116 -0.0648 -0.0261 0.01206 0.47020 +-0.3406 0.16048 0.24025 -0.2231 0.04953 -0.4098 -0.1318 -0.1901 0.24577 0.01657 0.23906 0.34830 -0.3686 -1.0588 0.03983 0.41157 0.50691 0.07739 0.56849 0.12022 -0.0285 -0.4561 0.01791 0.35152 -0.1841 0.43497 -0.1596 -0.5598 -0.2625 -0.5829 -0.1822 -0.0361 -0.2550 -0.1632 0.14004 -0.1191 0.16472 -0.0255 -0.3117 0.19755 0.48528 0.12289 -0.1203 0.28699 0.05454 0.02643 0.03990 -0.0425 -0.0622 0.15366 0.04604 0.03927 -0.0757 0.02877 0.00302 -0.0295 0.02441 -0.0217 -0.0309 0.01853 -0.0205 -0.0925 0.06829 -0.0287 0.10430 0.07835 -0.0674 -0.0265 0.01084 0.47008 +-0.4159 0.14780 0.13797 -0.1960 0.03225 -0.4003 -0.1029 -0.2826 0.22846 0.04713 0.14033 0.41000 -0.3762 -0.9807 -0.0552 0.41419 0.42594 0.31176 0.48231 0.41857 -0.2947 -0.2061 0.31796 0.25362 -0.0994 0.34384 -0.2935 -0.3126 -0.1965 -0.5276 -0.2615 -0.0404 -0.1717 0.00863 0.06246 -0.0447 0.12039 -0.0408 -0.1099 0.26658 0.48253 -0.0297 0.04255 0.23162 0.10719 0.01199 0.05782 -0.0462 -0.0405 0.17318 0.03692 0.03282 -0.0733 0.00732 -0.0077 -0.0314 0.02713 -0.0083 -0.0353 0.00381 -0.0213 -0.0870 0.08276 -0.0296 0.10577 0.07670 -0.0697 -0.0270 0.00904 0.48497 +-0.2757 0.19059 0.20111 -0.0388 0.10536 -0.3292 -0.1296 -0.2751 0.26290 -0.0178 -0.0101 0.50925 -0.3812 -0.9228 0.07841 0.42115 0.43038 0.30740 0.48260 0.40406 -0.2989 -0.2122 0.29665 0.21849 -0.0998 0.34208 -0.3118 -0.2617 -0.1647 -0.5118 -0.2730 -0.0318 -0.1713 0.04693 0.08882 -0.0461 0.08348 -0.0298 -0.0949 0.26555 0.48197 -0.0542 0.03305 0.23637 0.10200 0.00710 0.06639 -0.0408 -0.0435 0.17827 0.03412 0.03172 -0.0623 0.00354 0.00075 -0.0362 0.02887 0.00107 -0.0380 -0.0020 -0.0219 -0.0923 0.09650 -0.0243 0.10794 0.07488 -0.0715 -0.0275 0.00723 0.49451 +-0.2266 0.17762 0.27054 -0.0027 0.12175 -0.2541 -0.1499 -0.2825 0.25600 0.00926 0.01762 0.46941 -0.4024 -0.9818 0.05652 0.47153 0.43584 0.29553 0.51777 0.44257 -0.2715 -0.2508 0.26069 0.27481 -0.0636 0.34016 -0.2652 -0.3136 -0.2428 -0.5244 -0.2951 0.00024 -0.1447 0.03651 0.14268 -0.0520 0.13771 -0.0184 -0.1223 0.26863 0.45118 -0.0375 0.02701 0.23577 0.10405 0.00415 0.05621 -0.0455 -0.0379 0.16955 0.05022 0.02966 -0.0727 0.00382 -0.0021 -0.0273 0.03612 0.00188 -0.0376 0.00290 -0.0268 -0.0881 0.09009 -0.0258 0.10966 0.07470 -0.0728 -0.0280 0.00565 0.51284 +-0.1829 0.16202 0.22948 -0.0273 0.10941 -0.2759 -0.1214 -0.3067 0.22827 0.06287 0.10696 0.44938 -0.4066 -1.0336 -0.1074 0.57199 0.38735 0.31303 0.48771 0.46848 -0.3142 -0.2514 0.20392 0.27566 -0.0683 0.33313 -0.2645 -0.2872 -0.2197 -0.5350 -0.3085 0.01133 -0.1574 0.02476 0.14312 0.02156 0.12236 -0.0279 -0.1264 0.28211 0.44369 -0.0276 0.03260 0.22556 0.10242 0.00887 0.05802 -0.0465 -0.0393 0.17219 0.04631 0.03008 -0.0705 0.00526 0.00153 -0.0315 0.03291 0.00246 -0.0345 -0.0015 -0.0348 -0.0998 0.09365 -0.0255 0.11062 0.07564 -0.0734 -0.0286 0.00468 0.52631 +-0.1296 0.15572 0.28276 -0.0025 0.14193 -0.2975 -0.0783 -0.3161 0.23159 0.11647 0.13501 0.44001 -0.4155 -1.0589 -0.1827 0.61746 0.38332 0.33009 0.48138 0.46766 -0.3355 -0.2527 0.17887 0.26297 -0.0395 0.31735 -0.2539 -0.2622 -0.2091 -0.5265 -0.3023 0.01180 -0.1871 0.00853 0.13072 0.05088 0.13162 -0.0424 -0.1285 0.28543 0.44163 -0.0297 0.03353 0.23171 0.11418 0.00904 0.06205 -0.0381 -0.0344 0.17062 0.05448 0.02957 -0.0626 -0.0020 0.00362 -0.0311 0.03627 8.07957 -0.0352 0.00771 -0.0307 -0.1010 0.09420 -0.0252 0.11146 0.07800 -0.0732 -0.0291 0.00352 0.53302 +-0.1387 0.17023 0.32072 -0.0514 0.14450 -0.2779 -0.0431 -0.3517 0.22236 0.13675 0.19283 0.40500 -0.4458 -1.0966 -0.2662 0.70402 0.36201 0.33219 0.48126 0.43040 -0.3657 -0.2373 0.13735 0.26666 -0.0206 0.30407 -0.2111 -0.2608 -0.1866 -0.5172 -0.3249 0.04046 -0.1917 0.01906 0.14623 0.09860 0.12476 -0.0342 -0.1134 0.29054 0.44531 -0.0122 0.02927 0.22934 0.12095 0.00710 0.04748 -0.0373 -0.0208 0.16716 0.05397 0.03297 -0.0620 0.00629 0.00959 -0.0301 0.03173 -0.0019 -0.0375 0.00546 -0.0296 -0.0970 0.09403 -0.0231 0.11204 0.07999 -0.0724 -0.0295 0.00233 0.54389 +-0.2677 0.16760 0.25253 -0.1751 0.04013 -0.3169 -0.0763 -0.3241 0.23172 0.12215 0.19109 0.34482 -0.4626 -1.0765 -0.2467 0.57767 0.39343 0.32375 0.48575 0.45242 -0.2832 -0.2224 0.17173 0.26981 -0.0366 0.31125 -0.2132 -0.2712 -0.2011 -0.5377 -0.3102 0.03722 -0.1892 0.01046 0.12733 0.06093 0.15072 -0.0484 -0.1216 0.28116 0.44757 -0.0285 0.01627 0.23401 0.11803 0.00310 0.04979 -0.0503 -0.0299 0.17981 0.05616 0.02812 -0.0599 0.00817 0.00822 -0.0268 0.03258 -0.0007 -0.0388 0.00980 -0.0252 -0.0959 0.09318 -0.0193 0.11034 0.08221 -0.0713 -0.0299 0.00249 0.54044 +-0.5472 0.08043 0.11046 -0.2852 0.00306 -0.5183 -0.0507 -0.3559 0.19251 0.12489 0.26585 0.38127 -0.4812 -1.0588 -0.2272 0.46671 0.39767 0.33074 0.49420 0.44915 -0.2778 -0.2372 0.23029 0.31809 -0.0545 0.32757 -0.2253 -0.3187 -0.2184 -0.5506 -0.2769 -0.0052 -0.1927 0.01726 0.10797 0.03672 0.13211 -0.0570 -0.1383 0.26189 0.45158 -0.0125 0.00089 0.22112 0.12028 0.00311 0.05083 -0.0597 -0.0501 0.17955 0.04938 0.03167 -0.0654 0.00442 0.00915 -0.0280 0.03233 -0.0002 -0.0287 0.00618 -0.0247 -0.0813 0.08642 -0.0191 0.10830 0.08276 -0.0704 -0.0304 0.00337 0.53565 +-0.5704 0.08443 0.09487 -0.2329 0.04390 -0.4743 -0.1295 -0.2686 0.21463 0.08873 0.28771 0.44584 -0.4405 -0.9285 -0.0929 0.37413 0.40314 0.28697 0.48231 0.42532 -0.2210 -0.2768 0.25692 0.32190 -0.1277 0.33891 -0.1967 -0.3440 -0.2596 -0.5498 -0.2618 -0.0086 -0.1671 0.01819 0.15200 -0.0480 0.13014 -0.0662 -0.1216 0.24972 0.40467 -0.0070 -0.0142 0.21232 0.13406 0.00872 0.05717 -0.0619 -0.0575 0.19142 0.04502 0.03081 -0.0702 0.00859 -3.3807 -0.0344 0.03246 -0.0016 -0.0354 0.00740 -0.0217 -0.0878 0.09182 -0.0192 0.10576 0.08192 -0.0702 -0.0306 0.00360 0.53239 +-0.2654 0.18509 0.23632 -0.0453 0.11826 -0.3594 -0.1018 -0.2099 0.26776 -0.0294 0.14492 0.55129 -0.3596 -0.7594 0.09265 0.42406 0.41829 0.28587 0.47277 0.45348 -0.1903 -0.2539 0.27772 0.30769 -0.1640 0.33286 -0.2679 -0.3115 -0.2262 -0.5382 -0.2730 -0.0266 -0.1463 0.01453 0.13408 -0.0817 0.12273 -0.0535 -0.1199 0.28020 0.43430 -0.0285 0.00847 0.21688 0.12239 0.01345 0.05815 -0.0477 -0.0494 0.17880 0.05155 0.03155 -0.0790 0.00163 -0.0142 -0.0209 0.02787 -0.0080 -0.0383 0.00667 -0.0179 -0.0864 0.07317 -0.0307 0.10400 0.07960 -0.0702 -0.0305 0.00370 0.51797 +0.01144 0.26520 0.34379 0.17016 0.18172 -0.1153 -0.0828 -0.1383 0.32211 -0.0328 -0.1113 0.49942 -0.2367 -0.6144 0.33671 0.50704 0.41952 0.22491 0.47602 0.38593 -0.1847 -0.1561 0.25489 0.32865 -0.1374 0.32838 -0.1987 -0.4337 -0.3168 -0.5516 -0.2423 -0.0041 -0.1093 0.01329 0.19084 -0.1412 0.16333 -0.0417 -0.1137 0.28638 0.43636 -0.0404 0.03982 0.23698 0.09969 0.00720 0.06211 -0.0191 -0.0283 0.17791 0.03946 0.03333 -0.0775 -0.0007 -0.0047 -0.0210 0.02063 -0.0080 -0.0436 0.00186 -0.0189 -0.0750 0.07867 -0.0210 0.10151 0.07758 -0.0697 -0.0303 0.00414 0.50458 +0.13119 0.27599 0.41494 0.32871 0.26738 -0.0390 -0.0498 -0.1444 0.33776 -0.0508 -0.1754 0.46004 -0.2377 -0.6442 0.36349 0.53977 0.41942 0.27463 0.46859 0.41021 -0.1929 -0.1763 0.17279 0.30146 -0.0276 0.31394 -0.2263 -0.3834 -0.1917 -0.5041 -0.2927 0.00454 -0.1171 0.02521 0.17092 -0.1152 0.13246 -0.0292 -0.0828 0.30419 0.42419 -0.0241 0.03871 0.22532 0.09886 0.01339 0.05523 -0.0222 -0.0230 0.16951 0.04461 0.03097 -0.0716 -0.0015 -0.0012 -0.0175 0.02256 -0.0035 -0.0406 0.00447 -0.0269 -0.0792 0.08131 -0.0230 0.09967 0.07504 -0.0696 -0.0302 0.00479 0.50020 +0.16692 0.27596 0.36214 0.44921 0.33443 -0.0826 -0.0723 -0.1687 0.29175 -0.0576 -0.1875 0.44469 -0.2515 -0.6570 0.37768 0.51370 0.39351 0.23842 0.46322 0.39227 -0.2038 -0.1750 0.15878 0.31725 0.00507 0.28712 -0.1554 -0.3385 -0.1700 -0.4648 -0.3306 0.00504 -0.1295 0.02276 0.22387 -0.0971 0.11978 -0.0574 -0.0954 0.30919 0.37601 -0.0083 0.01078 0.21934 0.10768 0.00904 0.05162 -0.0192 -0.0189 0.16840 0.05340 0.03089 -0.0740 0.00191 -0.0015 -0.0180 0.02664 -0.0016 -0.0342 0.00489 -0.0320 -0.0811 0.08786 -0.0313 0.09907 0.07311 -0.0701 -0.0299 0.00442 0.49936 +0.11367 0.27331 0.30999 0.52759 0.41063 -0.0702 -0.0635 -0.1747 0.26726 -0.0357 -0.2186 0.46759 -0.2582 -0.5649 0.47336 0.47074 0.39506 0.20292 0.46928 0.39000 -0.2317 -0.1144 0.13006 0.31621 0.05465 0.28854 -0.1088 -0.3398 -0.1564 -0.4294 -0.3596 0.00959 -0.1587 0.02292 0.24234 -0.0763 0.11214 -0.0607 -0.0935 0.28388 0.35764 0.00753 0.00711 0.21486 0.11776 0.01180 0.05348 -0.0275 -0.0133 0.15613 0.04992 0.03539 -0.0733 0.00079 -0.0027 -0.0268 0.02321 0.00062 -0.0350 0.00130 -0.0308 -0.0904 0.08406 -0.0291 0.09982 0.07174 -0.0705 -0.0300 0.00415 0.49999 +0.09579 0.26393 0.18956 0.67212 0.51505 -0.0998 -0.0551 -0.2069 0.26827 -0.0507 -0.2537 0.49614 -0.2325 -0.5005 0.52661 0.44977 0.38556 0.15991 0.45135 0.35943 -0.3067 -0.1001 0.09574 0.31643 0.08291 0.27745 -0.0991 -0.3097 -0.1170 -0.3945 -0.3692 0.00433 -0.1677 0.03229 0.29270 -0.0579 0.06444 -0.0768 -0.0932 0.30137 0.33568 0.01699 0.00036 0.21523 0.11654 0.00833 0.05756 -0.0366 -0.0182 0.15311 0.05035 0.03418 -0.0728 -0.0023 -0.0071 -0.0245 0.02860 0.00129 -0.0314 0.00224 -0.0288 -0.0969 0.07650 -0.0278 0.10051 0.07247 -0.0706 -0.0300 0.00426 0.49569 +0.02541 0.22141 0.12908 0.53424 0.49005 -0.1977 -0.0372 -0.1937 0.26851 -0.0633 -0.1323 0.56222 -0.1885 -0.5194 0.39862 0.50472 0.35679 0.21167 0.44325 0.43317 -0.2631 -0.1296 0.05611 0.31343 0.02792 0.30442 -0.1521 -0.2600 -0.1446 -0.4200 -0.3292 -0.0098 -0.1286 0.03120 0.27957 -0.0940 0.08587 -0.0853 -0.0991 0.28990 0.34266 0.01722 -0.0035 0.21208 0.11743 0.01700 0.05741 -0.0312 -0.0290 0.16806 0.04336 0.03021 -0.0764 -0.0034 -0.0044 -0.0165 0.02477 -0.0054 -0.0289 0.00578 -0.0210 -0.0924 0.06108 -0.0370 0.10080 0.07392 -0.0697 -0.0299 0.00521 0.47654 +-0.1037 0.20924 0.12582 0.38837 0.43369 -0.2692 -0.0390 -0.2362 0.26478 -0.0032 -0.0809 0.60824 -0.2667 -0.6546 0.25819 0.53589 0.35781 0.27250 0.47459 0.41914 -0.2877 -0.1831 0.10863 0.33967 -0.0223 0.31552 -0.1964 -0.2625 -0.1834 -0.4628 -0.3260 -0.0127 -0.1326 0.03642 0.23948 -0.0764 0.08430 -0.0738 -0.1009 0.32657 0.34673 0.02193 -0.0268 0.23316 0.10675 0.00664 0.05899 -0.0332 -0.0308 0.16960 0.04456 0.02810 -0.0770 0.00578 -0.0106 -0.0204 0.03269 -0.0005 -0.0358 -0.0016 -0.0253 -0.0926 0.06515 -0.0312 0.09990 0.07586 -0.0690 -0.0294 0.00555 0.47065 +-0.3495 0.15414 0.07892 0.07778 0.29910 -0.3577 -0.0453 -0.2222 0.26830 0.06822 0.11692 0.57100 -0.3718 -0.8647 0.11185 0.48532 0.36050 0.31323 0.48395 0.52169 -0.2324 -0.2356 0.16070 0.30435 -0.0992 0.32117 -0.2736 -0.2315 -0.2082 -0.5589 -0.3032 -0.0220 -0.1426 0.02439 0.17186 -0.0717 0.07219 -0.0622 -0.1080 0.28818 0.40378 0.00627 -0.0021 0.19336 0.12252 0.01517 0.06327 -0.0452 -0.0396 0.17478 0.04627 0.03363 -0.0807 0.00088 -0.0016 -0.0318 0.03926 -0.0018 -0.0369 0.00212 -0.0243 -0.0941 0.07465 -0.0292 0.09760 0.07709 -0.0681 -0.0284 0.00529 0.47968 +-0.3635 0.13578 0.14701 -0.0145 0.18909 -0.4322 -0.0988 -0.1757 0.23589 0.03368 0.19207 0.52149 -0.3525 -0.9689 0.05135 0.39873 0.46378 0.15952 0.52723 0.23460 -0.1624 -0.4813 -0.0607 0.26195 -0.2000 0.43942 -0.3077 -0.3582 -0.1218 -0.5684 -0.1590 -0.0712 -0.2408 -0.1425 0.08996 -0.0858 -0.0037 -0.0586 -0.2192 0.19031 0.50809 0.12616 -0.1063 0.28377 0.06527 0.02521 0.03888 -0.0398 -0.0583 0.16412 0.04730 0.03666 -0.0755 0.01959 0.00751 -0.0329 0.03103 -0.0110 -0.0348 0.01319 -0.0185 -0.0898 0.06699 -0.0215 0.09383 0.07852 -0.0666 -0.0277 0.00627 0.48224 +-0.3551 0.16922 0.21803 -0.0504 0.13703 -0.4533 -0.1452 -0.1457 0.21741 0.00785 0.18247 0.50846 -0.3155 -1.0606 0.09562 0.34770 0.56297 0.03175 0.62539 -0.0130 -0.1475 -0.6817 -0.1989 0.22766 -0.2575 0.52182 -0.3376 -0.4098 -0.0433 -0.5761 -0.0459 -0.1169 -0.3103 -0.2660 -0.0196 -0.1064 -0.0813 -0.0527 -0.3091 0.12323 0.60926 0.20387 -0.1766 0.37108 0.01100 0.04777 0.01538 -0.0151 -0.0700 0.15139 0.05625 0.04131 -0.0699 0.03288 0.00786 -0.0371 0.02796 -0.0187 -0.0436 0.01796 -0.0134 -0.0913 0.05640 -0.0235 0.09000 0.07992 -0.0645 -0.0273 0.00799 0.48355 +-0.2564 0.19825 0.32838 -0.1142 0.07403 -0.3992 -0.1732 -0.1250 0.23124 0.00288 0.18544 0.45378 -0.3201 -1.0762 0.02009 0.36130 0.58828 -0.0337 0.63529 -0.0720 -0.0954 -0.7149 -0.2126 0.24795 -0.2796 0.54991 -0.3091 -0.4653 -0.0680 -0.6073 -0.0265 -0.1112 -0.3471 -0.3054 -0.0427 -0.1020 -0.0533 -0.0423 -0.3476 0.09003 0.63995 0.21792 -0.1892 0.39771 -0.0074 0.04825 0.01642 -0.0042 -0.0751 0.15271 0.05067 0.04154 -0.0684 0.04083 0.01184 -0.0425 0.02787 -0.0232 -0.0389 0.01946 -0.0131 -0.0945 0.05845 -0.0224 0.08723 0.08037 -0.0626 -0.0267 0.00885 0.49591 +-0.1276 0.22360 0.34867 -0.1033 0.08175 -0.3071 -0.1785 -0.1409 0.21368 0.01285 0.18183 0.46389 -0.3280 -1.1003 -0.0305 0.48574 0.55077 -0.0104 0.63087 -0.0426 -0.1139 -0.6786 -0.2119 0.25793 -0.2604 0.51374 -0.2555 -0.5111 -0.1458 -0.6045 -0.0752 -0.0767 -0.3166 -0.2849 0.03054 -0.0756 0.01557 -0.0139 -0.3474 0.12144 0.60691 0.21173 -0.1636 0.36573 -0.0089 0.04613 0.01425 -0.0093 -0.0716 0.15664 0.05219 0.04510 -0.0735 0.04005 0.00852 -0.0327 0.02614 -0.0245 -0.0362 0.01491 -0.0217 -0.0920 0.05989 -0.0245 0.08560 0.08065 -0.0607 -0.0261 0.01023 0.48911 +-0.1161 0.20291 0.35571 0.00062 0.11839 -0.2939 -0.1214 -0.2187 0.22677 0.07453 0.17851 0.43143 -0.3429 -1.0537 -0.1219 0.56727 0.47858 0.13376 0.57119 0.17534 -0.2004 -0.4716 -0.0944 0.28445 -0.1223 0.40711 -0.2170 -0.4033 -0.1613 -0.5428 -0.1983 -0.0234 -0.2668 -0.1581 0.09797 0.00205 0.07780 -0.0392 -0.2519 0.18698 0.51206 0.11626 -0.0952 0.30400 0.05350 0.03129 0.03072 -0.0184 -0.0442 0.15809 0.05964 0.03692 -0.0633 0.01845 0.00738 -0.0272 0.03150 -0.0148 -0.0341 0.01434 -0.0245 -0.1029 0.07449 -0.0245 0.08542 0.08057 -0.0593 -0.0258 0.01203 0.48606 +-0.1792 0.16410 0.27658 -0.0524 0.08228 -0.2593 -0.0785 -0.3046 0.23486 0.04377 0.17454 0.38692 -0.4355 -0.9953 -0.1130 0.58530 0.39348 0.28286 0.49471 0.43538 -0.2180 -0.2257 0.18368 0.31912 -0.0398 0.30288 -0.1675 -0.3415 -0.2688 -0.5297 -0.3097 0.03475 -0.1863 0.01090 0.20319 0.03546 0.20428 -0.0315 -0.1616 0.25101 0.41741 -0.0118 0.01479 0.22225 0.11749 0.00694 0.05662 -0.0483 -0.0337 0.17379 0.05851 0.03154 -0.0641 0.00576 0.00550 -0.0225 0.03757 0.00135 -0.0352 0.00799 -0.0295 -0.1006 0.09273 -0.0248 0.08659 0.07996 -0.0595 -0.0258 0.01324 0.48949 +-0.1724 0.16500 0.27505 0.04932 0.12162 -0.2047 -0.1032 -0.2972 0.21177 -0.0108 0.09410 0.37549 -0.4343 -0.9587 -0.0078 0.55134 0.41779 0.24564 0.50602 0.40154 -0.1876 -0.2104 0.22779 0.35690 -0.0518 0.30852 -0.1362 -0.4149 -0.3009 -0.5169 -0.3215 0.04278 -0.1886 0.01951 0.21487 0.01087 0.23106 -0.0037 -0.1742 0.25967 0.39683 -0.0224 0.00439 0.22914 0.11516 0.00811 0.05476 -0.0492 -0.0391 0.17071 0.05405 0.03182 -0.0639 0.00431 0.00475 -0.0252 0.03345 0.00057 -0.0352 0.00657 -0.0288 -0.1077 0.08747 -0.0312 0.08997 0.07921 -0.0605 -0.0262 0.01323 0.48197 +-0.1802 0.17876 0.32732 -0.1320 0.03039 -0.2466 -0.0809 -0.2720 0.26519 0.07083 0.22797 0.41033 -0.4255 -0.9609 -0.1834 0.60832 0.40201 0.29822 0.50560 0.43588 -0.2206 -0.2407 0.22845 0.29290 -0.0799 0.32225 -0.2224 -0.3656 -0.2916 -0.5694 -0.2900 0.01242 -0.1576 0.01140 0.16002 0.00333 0.20744 -0.0168 -0.1639 0.25839 0.44881 -0.0212 0.02310 0.23018 0.11284 0.00958 0.06235 -0.0411 -0.0348 0.17144 0.04910 0.03358 -0.0695 0.00976 0.00436 -0.0263 0.02717 -0.0019 -0.0402 0.01085 -0.0258 -0.0948 0.08729 -0.0223 0.09335 0.07886 -0.0618 -0.0270 0.01312 0.48189 +-0.2834 0.14692 0.26586 -0.2133 -0.0221 -0.3050 -0.0899 -0.3237 0.22708 0.09556 0.23234 0.37240 -0.4590 -1.0147 -0.2305 0.57105 0.40193 0.29140 0.49869 0.44937 -0.2672 -0.2541 0.20477 0.30194 -0.0581 0.32785 -0.2386 -0.3292 -0.2425 -0.5622 -0.2838 0.01510 -0.1881 -0.0025 0.13267 0.03273 0.17835 -0.0462 -0.1512 0.26887 0.46319 -0.0079 0.00933 0.22092 0.10631 0.01352 0.05254 -0.0473 -0.0405 0.17819 0.05260 0.03118 -0.0679 0.00539 0.00407 -0.0276 0.03346 -0.0025 -0.0397 0.00349 -0.0266 -0.0922 0.08957 -0.0203 0.09636 0.07885 -0.0633 -0.0280 0.01320 0.47564 +-0.2997 0.12538 0.26643 -0.1628 0.02277 -0.3016 -0.1208 -0.2998 0.22279 0.04324 0.18435 0.36998 -0.4529 -1.0516 -0.1179 0.55067 0.41264 0.27155 0.52100 0.42492 -0.2359 -0.2648 0.22355 0.34959 -0.0601 0.32449 -0.1734 -0.3665 -0.2906 -0.5591 -0.2901 0.04352 -0.1747 0.00517 0.17263 0.01730 0.20060 -0.0197 -0.1788 0.26575 0.44673 -0.0214 0.01343 0.22804 0.10874 0.00788 0.05648 -0.0540 -0.0477 0.17187 0.04452 0.03316 -0.0615 0.00567 0.00199 -0.0333 0.03670 -0.0034 -0.0323 0.00385 -0.0237 -0.0981 0.08605 -0.0225 0.09874 0.07852 -0.0649 -0.0289 0.01249 0.47404 +-0.3581 0.09559 0.14066 -0.0260 0.19021 -0.4095 -0.0739 -0.3136 0.21179 0.06727 0.17819 0.43811 -0.3878 -0.9473 -0.0293 0.45205 0.40008 0.31029 0.44690 0.47500 -0.2835 -0.2209 0.20011 0.29832 -0.0685 0.33712 -0.2603 -0.2582 -0.2102 -0.5380 -0.2871 -0.0090 -0.1586 0.02215 0.12185 -0.0239 0.08344 -0.0624 -0.1077 0.28849 0.40304 -0.0137 -0.0072 0.20425 0.12103 0.00978 0.05600 -0.0483 -0.0487 0.16924 0.05829 0.03176 -0.0701 0.00289 -0.0044 -0.0341 0.03710 -0.0026 -0.0303 0.00739 -0.0253 -0.0957 0.08587 -0.0190 0.10027 0.07773 -0.0671 -0.0294 0.01083 0.49219 +-0.4225 0.12965 0.03897 0.01589 0.24236 -0.4255 -0.0963 -0.2367 0.23394 0.01231 0.21699 0.48498 -0.3338 -0.8253 0.01274 0.38308 0.40016 0.27198 0.44052 0.53003 -0.2419 -0.2622 0.14223 0.32425 -0.1038 0.34979 -0.2763 -0.2259 -0.1616 -0.5414 -0.2868 -0.0146 -0.1406 0.02883 0.17392 -0.0392 0.06992 -0.0655 -0.1002 0.28547 0.41914 -0.0170 -0.0223 0.18926 0.12551 0.00039 0.07072 -0.0534 -0.0405 0.17060 0.04391 0.02978 -0.0806 0.00272 1.04419 -0.0388 0.03589 -0.0049 -0.0363 0.00371 -0.0249 -0.0794 0.08316 -0.0219 0.10203 0.07745 -0.0687 -0.0298 0.00870 0.50774 +-0.4783 0.15308 0.05010 -0.0176 0.18882 -0.4830 -0.0989 -0.2023 0.25243 0.06362 0.19162 0.54515 -0.3493 -0.8389 0.02782 0.43614 0.37237 0.31601 0.49952 0.51812 -0.2676 -0.2382 0.18180 0.25788 -0.1608 0.34877 -0.2984 -0.2128 -0.1755 -0.5531 -0.2735 -0.0347 -0.1148 0.03342 0.14257 -0.0628 0.06493 -0.0569 -0.0814 0.28858 0.43346 -0.0101 -0.0126 0.19898 0.12125 0.00706 0.06652 -0.0415 -0.0531 0.17151 0.03505 0.03519 -0.0714 0.00933 -0.0048 -0.0392 0.03243 -0.0014 -0.0311 0.01056 -0.0199 -0.0916 0.08238 -0.0273 0.10327 0.07764 -0.0693 -0.0304 0.00803 0.51196 +-0.4563 0.14000 0.14083 -0.3368 -0.0328 -0.4942 -0.0877 -0.2403 0.23149 0.16122 0.19720 0.39101 -0.3921 -0.9350 -0.1446 0.37894 0.41363 0.32037 0.49027 0.42832 -0.2877 -0.2364 0.27003 0.22022 -0.1028 0.33178 -0.2906 -0.3172 -0.1934 -0.5576 -0.2526 -0.0208 -0.1526 0.01955 0.05092 -0.0428 0.09586 -0.0313 -0.0812 0.27748 0.48911 -0.0386 0.03964 0.23485 0.11428 0.01104 0.05791 -0.0432 -0.0461 0.18112 0.03760 0.02910 -0.0726 0.00333 -0.0004 -0.0416 0.03489 0.00478 -0.0427 0.00408 -0.0167 -0.0874 0.09070 -0.0220 0.10329 0.07872 -0.0689 -0.0310 0.00736 0.51513 +-0.4594 0.11133 0.18337 -0.3811 -0.0728 -0.4046 -0.0841 -0.3162 0.25487 0.12730 0.17880 0.33081 -0.4469 -1.1215 -0.2589 0.43138 0.41162 0.32260 0.50488 0.44638 -0.3375 -0.2245 0.31956 0.20983 -0.1122 0.37381 -0.3064 -0.2589 -0.1669 -0.5792 -0.2474 -0.0212 -0.1696 0.02370 -0.0283 -0.0090 0.08363 -0.0333 -0.0863 0.27673 0.51828 -0.0332 0.06728 0.24529 0.10693 0.00598 0.05948 -0.0285 -0.0422 0.17806 0.03383 0.03176 -0.0725 0.00894 0.00321 -0.0419 0.03130 -0.0019 -0.0411 0.00546 -0.0202 -0.0870 0.09073 -0.0121 0.10219 0.07910 -0.0681 -0.0314 0.00660 0.52167 +-0.3555 0.16566 0.22803 -0.3155 -0.0504 -0.3655 -0.1161 -0.3381 0.24321 0.07076 0.21549 0.32869 -0.4696 -1.1398 -0.2856 0.49630 0.42864 0.29360 0.51417 0.42584 -0.2489 -0.2632 0.32190 0.24460 -0.1038 0.34128 -0.2613 -0.3284 -0.2267 -0.5585 -0.2764 -0.0020 -0.1889 0.00871 0.05122 -0.0026 0.15784 -0.0276 -0.1270 0.26706 0.50542 -0.0274 0.04733 0.24377 0.10286 0.00383 0.05977 -0.0397 -0.0456 0.18132 0.04734 0.03545 -0.0678 0.00667 0.00354 -0.0308 0.02995 0.00067 -0.0368 0.00216 -0.0251 -0.0864 0.08764 -0.0212 0.09885 0.07993 -0.0671 -0.0316 0.00705 0.51447 +-0.5836 0.13965 0.15271 -0.4839 -0.1168 -0.5351 -0.0741 -0.3893 0.20829 0.18466 0.24978 0.32329 -0.4588 -1.1833 -0.3693 0.46472 0.39746 0.36441 0.50206 0.44685 -0.3596 -0.2557 0.32203 0.20187 -0.1614 0.35461 -0.3440 -0.2576 -0.1868 -0.5922 -0.2384 -0.0239 -0.1771 0.03282 -0.0014 0.02270 0.08217 -0.0555 -0.1101 0.24285 0.53143 -0.0286 0.04844 0.24929 0.10952 0.00147 0.06296 -0.0436 -0.0356 0.17862 0.04642 0.03259 -0.0762 0.00576 0.00311 -0.0325 0.03030 0.00136 -0.0332 -0.0010 -0.0191 -0.0763 0.08943 -0.0188 0.09560 0.07935 -0.0664 -0.0318 0.00820 0.50639 +-0.7530 0.07403 0.07600 -0.5537 -0.0919 -0.7144 -0.0806 -0.4135 0.17242 0.25093 0.36206 0.33543 -0.4540 -1.1444 -0.3464 0.40452 0.35195 0.33281 0.47009 0.42931 -0.3621 -0.1979 0.30691 0.24505 -0.1614 0.37129 -0.3332 -0.3287 -0.2149 -0.5864 -0.2192 -0.0346 -0.1611 0.02735 0.02796 0.01936 0.07613 -0.0449 -0.1176 0.25929 0.52870 -0.0184 0.02447 0.22805 0.11327 0.00121 0.06329 -0.0574 -0.0493 0.18553 0.03037 0.03257 -0.0678 0.00162 -0.0043 -0.0452 0.02206 -0.0027 -0.0377 0.00808 -0.0179 -0.0789 0.09224 -0.0257 0.09213 0.07827 -0.0664 -0.0321 0.00864 0.50419 +-0.6859 0.10302 0.13395 -0.5183 -0.0964 -0.6988 -0.0820 -0.3446 0.20964 0.22381 0.36345 0.30038 -0.4553 -1.0696 -0.2648 0.35938 0.39258 0.27711 0.45862 0.41651 -0.2540 -0.1900 0.29700 0.30027 -0.1712 0.36666 -0.2716 -0.4158 -0.2846 -0.5916 -0.2419 -0.0236 -0.1651 0.02844 0.06293 -0.0184 0.14057 -0.0371 -0.1206 0.23973 0.50854 -0.0200 0.01218 0.20658 0.10770 0.00614 0.06065 -0.0501 -0.0474 0.18296 0.03013 0.03393 -0.0677 0.00783 -0.0029 -0.0426 0.02455 -0.0033 -0.0362 0.00419 -0.0204 -0.0806 0.08954 -0.0224 0.09002 0.07637 -0.0666 -0.0321 0.00878 0.49109 +-0.5299 0.13469 0.21400 -0.4571 -0.0963 -0.5578 -0.1219 -0.2919 0.22945 0.12017 0.28310 0.33434 -0.4520 -1.0164 -0.2738 0.42595 0.42325 0.26848 0.51507 0.37212 -0.2233 -0.2226 0.38435 0.30014 -0.1472 0.36903 -0.2357 -0.4319 -0.2815 -0.5727 -0.2473 -0.0205 -0.1632 0.00514 0.06919 -0.0468 0.18699 -0.0384 -0.1430 0.24264 0.51531 -0.0325 0.02420 0.23887 0.11913 0.00451 0.06002 -0.0488 -0.0504 0.18034 0.03259 0.03077 -0.0674 0.00622 -0.0091 -0.0363 0.02833 -0.0011 -0.0388 0.00232 -0.0174 -0.0822 0.09064 -0.0245 0.08791 0.07505 -0.0667 -0.0320 0.00922 0.48162 +-0.3118 0.15036 0.25851 -0.2311 -0.0093 -0.3385 -0.1375 -0.2680 0.25344 0.06429 0.19042 0.41998 -0.4450 -1.0586 -0.2188 0.50812 0.42703 0.29416 0.49667 0.41720 -0.2441 -0.2637 0.30934 0.27781 -0.1522 0.35824 -0.2653 -0.3390 -0.2272 -0.5757 -0.2798 0.00380 -0.1552 0.00948 0.09126 -0.0322 0.15910 -0.0387 -0.1385 0.26701 0.48883 -0.0388 0.03424 0.23413 0.10879 0.00678 0.06531 -0.0495 -0.0463 0.17674 0.03935 0.02929 -0.0740 0.00151 -0.0020 -0.0345 0.03375 -0.0009 -0.0376 -0.0045 -0.0262 -0.0821 0.09179 -0.0246 0.08690 0.07317 -0.0671 -0.0321 0.00973 0.48185 +-0.1602 0.16897 0.27402 -0.0575 0.11406 -0.2963 -0.0908 -0.3012 0.23559 0.12568 0.15444 0.47616 -0.4106 -1.0457 -0.2042 0.62389 0.38202 0.33300 0.47310 0.48997 -0.3203 -0.2600 0.17582 0.25534 -0.0932 0.32058 -0.2769 -0.2528 -0.1943 -0.5501 -0.2977 0.01222 -0.1781 0.01381 0.11484 0.04934 0.12109 -0.0448 -0.1203 0.29161 0.45770 -0.0473 0.03681 0.23290 0.11098 0.00499 0.06209 -0.0406 -0.0310 0.16986 0.05208 0.02795 -0.0632 -0.0016 0.00279 -0.0328 0.03663 -0.0013 -0.0365 0.00322 -0.0328 -0.0955 0.09269 -0.0229 0.08696 0.07192 -0.0679 -0.0320 0.00898 0.48856 +-0.1097 0.17305 0.31919 0.04986 0.17188 -0.3073 -0.0682 -0.2890 0.24205 0.16078 0.19267 0.44750 -0.4269 -1.0390 -0.1867 0.67474 0.35153 0.33198 0.48806 0.50926 -0.3441 -0.2239 0.10632 0.27256 0.01422 0.30140 -0.2173 -0.2579 -0.2143 -0.5236 -0.3412 0.03775 -0.1801 0.01057 0.16918 0.07493 0.14047 -0.0174 -0.1264 0.27480 0.43240 -0.0148 0.03595 0.22717 0.11019 0.00654 0.05575 -0.0273 -0.0293 0.16472 0.06141 0.03164 -0.0628 0.00396 0.00913 -0.0260 0.03703 -0.0006 -0.0375 0.00710 -0.0299 -0.1015 0.08397 -0.0258 0.08866 0.07100 -0.0687 -0.0322 0.00824 0.49694 +-0.0898 0.16887 0.29845 0.07800 0.20636 -0.2694 -0.0669 -0.3108 0.22254 0.15100 0.15629 0.43242 -0.4343 -1.0420 -0.2006 0.67921 0.36292 0.32233 0.48405 0.47690 -0.3715 -0.2155 0.09762 0.28763 0.04290 0.29293 -0.1913 -0.2503 -0.1980 -0.5045 -0.3501 0.05272 -0.1893 0.01646 0.17407 0.09767 0.12595 -0.0190 -0.1100 0.29060 0.42671 -0.0120 0.02405 0.23577 0.10955 0.01006 0.04909 -0.0235 -0.0299 0.16383 0.06234 0.03373 -0.0571 0.00479 0.01109 -0.0243 0.03436 -0.0014 -0.0363 0.00960 -0.0277 -0.1014 0.07978 -0.0277 0.09046 0.07237 -0.0690 -0.0322 0.00819 0.49227 +-0.1015 0.15653 0.27844 0.07669 0.20077 -0.2099 -0.0510 -0.3332 0.22056 0.08906 0.12629 0.39604 -0.4207 -1.0578 -0.1490 0.63636 0.38741 0.27436 0.48398 0.43479 -0.3597 -0.2168 0.13855 0.30097 0.07107 0.29088 -0.1792 -0.2514 -0.1939 -0.5029 -0.3340 0.05418 -0.2092 0.02544 0.14829 0.08502 0.12148 -0.0165 -0.0934 0.28767 0.43502 -0.0152 0.01771 0.23999 0.10805 0.01037 0.04774 -0.0266 -0.0298 0.16544 0.05935 0.03534 -0.0566 0.00728 0.01173 -0.0306 0.03488 -0.0006 -0.0345 0.01013 -0.0310 -0.0986 0.08868 -0.0270 0.09167 0.07391 -0.0683 -0.0321 0.00895 0.47476 +-0.1740 0.14410 0.21871 -0.0479 0.11039 -0.2121 -0.0573 -0.3401 0.22540 0.04205 0.14797 0.36761 -0.4145 -1.0528 -0.1198 0.57750 0.40221 0.29651 0.48140 0.42989 -0.3184 -0.2114 0.17003 0.25993 0.02934 0.30997 -0.2226 -0.2348 -0.1991 -0.5115 -0.3094 0.03278 -0.2052 0.02113 0.13140 0.06729 0.12695 -0.0173 -0.1154 0.26272 0.44053 -0.0141 0.02352 0.22052 0.10685 0.00831 0.05208 -0.0441 -0.0399 0.17432 0.05893 0.03615 -0.0578 0.00771 0.01157 -0.0316 0.03516 0.00041 -0.0323 0.00939 -0.0298 -0.0983 0.08738 -0.0261 0.09072 0.07600 -0.0676 -0.0317 0.00910 0.47275 +-0.2279 0.15222 0.21033 -0.1377 0.06937 -0.2746 -0.0601 -0.3575 0.21338 0.04875 0.14991 0.37188 -0.4267 -1.0639 -0.2038 0.57404 0.40192 0.31840 0.47144 0.46390 -0.3466 -0.2391 0.18205 0.23893 -0.0261 0.32561 -0.2993 -0.2058 -0.1889 -0.5238 -0.3074 0.01198 -0.1855 0.02514 0.10208 0.06317 0.10413 -0.0355 -0.1177 0.27259 0.46534 -0.0158 0.03168 0.22205 0.10819 0.01373 0.04866 -0.0494 -0.0473 0.17573 0.05691 0.03472 -0.0691 0.00642 0.00898 -0.0306 0.03707 -0.0029 -0.0339 0.00030 -0.0301 -0.0956 0.08655 -0.0235 0.08971 0.07663 -0.0669 -0.0310 0.00926 0.47462 +-0.4054 0.12408 0.19691 -0.3007 -0.0550 -0.4125 -0.0820 -0.3599 0.21574 0.05471 0.18093 0.30331 -0.4640 -1.1312 -0.2223 0.47925 0.42732 0.32031 0.48152 0.44341 -0.3239 -0.2461 0.28079 0.24663 -0.0847 0.35208 -0.2740 -0.2656 -0.1959 -0.5819 -0.2729 -0.0028 -0.1896 0.02676 0.06494 0.01558 0.13213 -0.0377 -0.1277 0.25954 0.50271 -0.0210 0.03802 0.24345 0.11007 0.00520 0.05983 -0.0616 -0.0494 0.18641 0.04899 0.03368 -0.0658 0.00119 0.00243 -0.0293 0.02526 -0.0001 -0.0311 0.00524 -0.0264 -0.0837 0.08040 -0.0264 0.08797 0.07717 -0.0660 -0.0302 0.00950 0.47876 +-0.4695 0.12556 0.17223 -0.3085 -0.0332 -0.4395 -0.1058 -0.3176 0.22436 0.09335 0.18300 0.32881 -0.4892 -1.0957 -0.1771 0.47449 0.41965 0.30093 0.49778 0.37645 -0.3120 -0.2362 0.32132 0.25848 -0.1242 0.3615 -0.2515 -0.3481 -0.2073 -0.5586 -0.2667 0.00066 -0.1823 0.01998 0.04488 -0.0171 0.13667 -0.0199 -0.1278 0.26467 0.51167 -0.0217 0.03637 0.24306 0.10794 0.00596 0.06184 -0.0629 -0.0430 0.17412 0.04070 0.03390 -0.0707 0.00744 -0.0027 -0.0380 0.02831 -0.0019 -0.0366 0.00255 -0.0191 -0.0827 0.08534 -0.0212 0.08551 0.07764 -0.0649 -0.0293 0.00981 0.48516 +-0.4561 0.14108 0.20322 -0.3100 -0.0351 -0.4242 -0.1126 -0.2852 0.23709 0.07616 0.15799 0.35321 -0.4462 -1.0350 -0.1432 0.44964 0.44364 0.27727 0.49717 0.35777 -0.2987 -0.2006 0.36354 0.26436 -0.1460 0.37256 -0.2713 -0.3506 -0.2250 -0.5760 -0.2680 -0.0128 -0.1685 0.01246 0.03600 -0.0526 0.13446 -0.0268 -0.1203 0.26123 0.52134 -0.0286 0.05162 0.24305 0.10072 0.00169 0.06769 -0.0452 -0.0387 0.17308 0.04159 0.03415 -0.0775 0.00776 -0.0016 -0.0377 0.02771 -0.0044 -0.0377 0.00277 -0.0223 -0.0811 0.08736 -0.0212 0.08231 0.07803 -0.0635 -0.0281 0.01020 0.49382 +-0.2915 0.16764 0.25559 -0.1779 0.02950 -0.3338 -0.1270 -0.2261 0.28260 0.04037 0.12508 0.40198 -0.3937 -0.9578 -0.1033 0.47747 0.43340 0.28516 0.49871 0.42647 -0.2622 -0.2200 0.31975 0.24898 -0.1818 0.36699 -0.3157 -0.3268 -0.2266 -0.5898 -0.2588 -0.0139 -0.1311 0.02101 0.06474 -0.0661 0.15628 -0.0196 -0.1309 0.25855 0.50692 -0.0472 0.05460 0.23717 0.10298 0.00187 0.06394 -0.0483 -0.0448 0.17836 0.03777 0.03076 -0.0754 0.00135 -0.0023 -0.0335 0.02424 -0.0058 -0.0420 0.00299 -0.0228 -0.0815 0.08905 -0.0245 0.07839 0.07835 -0.0619 -0.0268 0.01066 0.50475 +-0.2303 0.19406 0.24771 -0.1573 0.04175 -0.3110 -0.1174 -0.1987 0.30107 0.03073 0.13025 0.39950 -0.3776 -0.9211 -0.0519 0.48160 0.41014 0.28314 0.49382 0.43821 -0.2373 -0.2190 0.29889 0.20638 -0.2054 0.37286 -0.3512 -0.3133 -0.2339 -0.6017 -0.2527 -0.0310 -0.1120 0.02568 0.08815 -0.0791 0.15938 -0.0178 -0.1406 0.24988 0.52439 -0.0464 0.06063 0.23221 0.10133 0.00722 0.06689 -0.0504 -0.0469 0.18111 0.03700 0.03084 -0.0707 0.00381 -0.0007 -0.0306 0.02345 -0.0059 -0.0470 0.00316 -0.0238 -0.0828 0.08659 -0.0253 0.07375 0.07858 -0.0601 -0.0253 0.01120 0.51794 diff --git a/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/input/image.png b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/input/image.png new file mode 100644 index 0000000000000000000000000000000000000000..9b3328e3e36cbf33024c9e17902fda8f627f0c0f Binary files /dev/null and b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/input/image.png differ diff --git a/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/input/nana_speech_cut_4sec 1-0-100.wav b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/input/nana_speech_cut_4sec 1-0-100.wav new file mode 100644 index 0000000000000000000000000000000000000000..b84d0ec2cae941fa9b4d828df76340818bda4d13 Binary files /dev/null and b/results/82b4ccf3-6380-4e87-adb4-fe0a1067afa9/input/nana_speech_cut_4sec 1-0-100.wav differ diff --git a/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/first_frame_dir/image.mat b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/first_frame_dir/image.mat new file mode 100644 index 0000000000000000000000000000000000000000..142fc26b1ef351b424ec637f4d1bdce23d7c609f Binary files /dev/null and b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/first_frame_dir/image.mat differ diff --git a/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/first_frame_dir/image.png b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/first_frame_dir/image.png new file mode 100644 index 0000000000000000000000000000000000000000..93118c4ee1169fa614550273d84b34941915daa7 Binary files /dev/null and b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/first_frame_dir/image.png differ diff --git a/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/first_frame_dir/image_landmarks.txt b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/first_frame_dir/image_landmarks.txt new file mode 100644 index 0000000000000000000000000000000000000000..b9f28b39bbdabe1a8eaca94c099ba51b2c959b90 --- /dev/null +++ b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/first_frame_dir/image_landmarks.txt @@ -0,0 +1,136 @@ +8.084375000000000000e+01 +1.148320312500000000e+02 +7.978125000000000000e+01 +1.314257812500000000e+02 +7.978125000000000000e+01 +1.494023437500000000e+02 +8.403125000000000000e+01 +1.659960937500000000e+02 +8.934375000000000000e+01 +1.798242187500000000e+02 +9.571875000000000000e+01 +1.922695312500000000e+02 +1.031562500000000000e+02 +2.033320312500000000e+02 +1.116562500000000000e+02 +2.143945312500000000e+02 +1.233437500000000000e+02 +2.185429687500000000e+02 +1.403437500000000000e+02 +2.185429687500000000e+02 +1.573437500000000000e+02 +2.171601562500000000e+02 +1.732812500000000000e+02 +2.088632812500000000e+02 +1.870937500000000000e+02 +2.019492187500000000e+02 +1.998437500000000000e+02 +1.908867187500000000e+02 +2.062187500000000000e+02 +1.729101562500000000e+02 +2.072812500000000000e+02 +1.521679687500000000e+02 +2.072812500000000000e+02 +1.134492187500000000e+02 +8.084375000000000000e+01 +1.037695312500000000e+02 +8.721875000000000000e+01 +9.823828125000000000e+01 +9.465625000000000000e+01 +9.754687500000000000e+01 +1.031562500000000000e+02 +9.892968750000000000e+01 +1.105937500000000000e+02 +1.016953125000000000e+02 +1.360937500000000000e+02 +1.010039062500000000e+02 +1.467187500000000000e+02 +9.892968750000000000e+01 +1.594687500000000000e+02 +9.892968750000000000e+01 +1.716875000000000000e+02 +1.003125000000000000e+02 +1.828437500000000000e+02 +1.093007812500000000e+02 +1.212187500000000000e+02 +1.203632812500000000e+02 +1.190937500000000000e+02 +1.328085937500000000e+02 +1.169687500000000000e+02 +1.438710937500000000e+02 +1.159062500000000000e+02 +1.563164062500000000e+02 +1.063437500000000000e+02 +1.632304687500000000e+02 +1.116562500000000000e+02 +1.646132812500000000e+02 +1.180312500000000000e+02 +1.659960937500000000e+02 +1.254687500000000000e+02 +1.659960937500000000e+02 +1.329062500000000000e+02 +1.659960937500000000e+02 +8.615625000000000000e+01 +1.189804687500000000e+02 +9.040625000000000000e+01 +1.148320312500000000e+02 +1.042187500000000000e+02 +1.162148437500000000e+02 +1.084687500000000000e+02 +1.231289062500000000e+02 +1.020937500000000000e+02 +1.245117187500000000e+02 +9.040625000000000000e+01 +1.231289062500000000e+02 +1.445937500000000000e+02 +1.231289062500000000e+02 +1.509687500000000000e+02 +1.175976562500000000e+02 +1.669062500000000000e+02 +1.162148437500000000e+02 +1.732812500000000000e+02 +1.217460937500000000e+02 +1.669062500000000000e+02 +1.258945312500000000e+02 +1.520312500000000000e+02 +1.258945312500000000e+02 +1.052812500000000000e+02 +1.881210937500000000e+02 +1.084687500000000000e+02 +1.812070312500000000e+02 +1.137812500000000000e+02 +1.770585937500000000e+02 +1.180312500000000000e+02 +1.784414062500000000e+02 +1.222812500000000000e+02 +1.784414062500000000e+02 +1.329062500000000000e+02 +1.825898437500000000e+02 +1.445937500000000000e+02 +1.895039062500000000e+02 +1.360937500000000000e+02 +1.950351562500000000e+02 +1.286562500000000000e+02 +1.978007812500000000e+02 +1.201562500000000000e+02 +1.978007812500000000e+02 +1.137812500000000000e+02 +1.964179687500000000e+02 +1.095312500000000000e+02 +1.936523437500000000e+02 +1.063437500000000000e+02 +1.881210937500000000e+02 +1.116562500000000000e+02 +1.853554687500000000e+02 +1.180312500000000000e+02 +1.853554687500000000e+02 +1.307812500000000000e+02 +1.881210937500000000e+02 +1.424687500000000000e+02 +1.908867187500000000e+02 +1.307812500000000000e+02 +1.881210937500000000e+02 +1.190937500000000000e+02 +1.881210937500000000e+02 +1.127187500000000000e+02 +1.881210937500000000e+02 diff --git a/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.mat b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.mat new file mode 100644 index 0000000000000000000000000000000000000000..18cee43a2888848f4ba5e5ea8aca3c417500a8d0 Binary files /dev/null and b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.mat differ diff --git a/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.mp4 b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c7ca5bffdd95c45e5c4fb00f318147addef00b0a Binary files /dev/null and b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.mp4 differ diff --git a/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.txt b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.txt new file mode 100644 index 0000000000000000000000000000000000000000..bad31dbcc849a484e9144f0a98081572e329fb62 --- /dev/null +++ b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100.txt @@ -0,0 +1,92 @@ +-0.3291 -0.1598 0.43955 0.32648 0.59491 -0.1686 0.37293 -0.2642 0.17441 -0.2315 0.07730 0.27166 -0.0297 -0.9746 -1.1084 0.43305 0.40627 1.23409 0.24205 0.51549 -0.2520 -0.7977 0.20743 0.34780 -0.0125 -0.0074 -0.3704 -0.2005 -0.3804 -0.3122 0.12269 0.16314 0.18076 -0.1496 0.14755 0.30011 0.15958 0.01618 0.02480 0.44274 0.48291 0.14302 0.22731 0.14393 -0.0554 -0.0356 0.13260 0.01592 -0.0171 0.14028 0.07320 0.04554 -0.0417 0.06590 0.02072 -0.0807 -0.0032 -0.0300 -0.0552 0.06494 -0.0028 -0.0843 0.10287 0.01839 0.12522 -0.4127 -0.0331 0.01919 0.03835 0.56520 +-0.5060 -0.1858 0.34753 0.10764 0.47592 -0.2500 0.32450 -0.3326 0.12229 -0.1853 0.09011 0.22128 -0.0680 -1.0262 -0.9945 0.24959 0.44215 1.22582 0.24665 0.42421 -0.2043 -0.7796 0.31156 0.30881 -0.0033 0.01400 -0.3480 -0.2811 -0.4279 -0.3078 0.14509 0.13663 0.17695 -0.1448 0.10338 0.26329 0.18986 0.01613 0.03860 0.42717 0.46460 0.14033 0.20256 0.16676 -0.0354 -0.0289 0.12506 -0.0071 -0.0317 0.15232 0.06486 0.04925 -0.0387 0.07553 0.01758 -0.0806 -0.0110 -0.0256 -0.0593 0.06349 -0.0015 -0.0802 0.09693 0.01136 0.12330 -0.4122 -0.0312 0.01796 0.03903 0.56794 +-0.4492 -0.1729 0.39897 0.18588 0.50908 -0.2249 0.35513 -0.3130 0.13362 -0.2930 0.08527 0.21494 -0.1101 -1.1427 -1.0630 0.33032 0.43612 1.23015 0.25433 0.40769 -0.2000 -0.7817 0.37374 0.30391 -0.1055 0.02258 -0.3208 -0.2528 -0.4342 -0.3119 0.13521 0.15868 0.16615 -0.1470 0.08706 0.25445 0.19734 0.03221 -0.0133 0.44209 0.50355 0.13918 0.21918 0.17956 -0.0433 -0.0373 0.12469 -0.0045 -0.0329 0.14706 0.06055 0.04662 -0.0435 0.07367 0.01891 -0.0725 -0.0110 -0.0225 -0.0642 0.06170 0.00099 -0.0806 0.10233 0.00744 0.12182 -0.4120 -0.0297 0.01684 0.03945 0.57047 +-0.1688 -0.1583 0.51882 0.37342 0.59763 -0.0384 0.34922 -0.2743 0.15872 -0.2953 -0.0719 0.31657 -0.0727 -1.1524 -1.0804 0.51227 0.43593 1.24591 0.24696 0.45369 -0.3103 -0.8072 0.31315 0.28661 -0.0889 0.02695 -0.3617 -0.1485 -0.3668 -0.2901 0.11542 0.16334 0.15669 -0.1264 0.07585 0.27670 0.15788 0.01724 0.02219 0.46349 0.49532 0.12378 0.23356 0.19189 -0.0489 -0.0360 0.12694 0.01186 -0.0156 0.15142 0.06987 0.04574 -0.0464 0.06795 0.02527 -0.0820 -0.0070 -0.0183 -0.0646 0.04996 -0.0085 -0.0792 0.11460 0.01452 0.12079 -0.4121 -0.0287 0.01583 0.03961 0.57278 +-0.0537 -0.0819 0.62838 0.53188 0.71643 -0.0228 0.36203 -0.2883 0.16101 -0.1965 -0.0593 0.37122 -0.0406 -1.1315 -1.0897 0.63002 0.38212 1.27933 0.23534 0.51326 -0.3273 -0.7897 0.20216 0.26714 -0.0375 0.00024 -0.4045 -0.1255 -0.3470 -0.2941 0.10061 0.16403 0.15482 -0.1390 0.09347 0.31274 0.13796 0.02351 0.04568 0.47524 0.47012 0.13777 0.25440 0.18910 -0.0466 -0.0269 0.12843 0.02778 -0.0138 0.14079 0.08446 0.04397 -0.0406 0.06030 0.02645 -0.0812 -0.0037 -0.0212 -0.0590 0.06044 -0.0102 -0.0929 0.10690 0.01686 0.12021 -0.4124 -0.0283 0.01492 0.03951 0.57487 +-0.0422 -0.1145 0.59413 0.60416 0.79064 0.01885 0.38435 -0.3089 0.13352 -0.2288 -0.0814 0.34198 -0.0518 -1.1451 -1.0460 0.66060 0.37559 1.26045 0.24583 0.49631 -0.3711 -0.7672 0.13975 0.30255 0.04953 -0.0324 -0.2861 -0.1037 -0.3281 -0.2446 0.06849 0.20494 0.14322 -0.1297 0.14149 0.36415 0.13801 0.03673 0.04677 0.47159 0.44364 0.15309 0.23550 0.18189 -0.0427 -0.0358 0.11313 0.02598 0.00074 0.12856 0.08739 0.04576 -0.0358 0.07225 0.02877 -0.0750 -0.0059 -0.0222 -0.0573 0.06211 -0.0080 -0.0918 0.10787 0.01371 0.12007 -0.4129 -0.0283 0.01412 0.03916 0.57674 +-0.0865 -0.1198 0.52258 0.55730 0.72483 0.04289 0.36146 -0.3001 0.12733 -0.3012 -0.1114 0.30743 -0.0754 -1.1382 -1.0320 0.58685 0.41994 1.23154 0.26005 0.49363 -0.3083 -0.7832 0.16011 0.31658 0.05323 -0.0313 -0.2788 -0.1120 -0.3816 -0.2247 0.07493 0.22716 0.14139 -0.1368 0.13458 0.34279 0.16659 0.03926 0.03570 0.45461 0.42261 0.13278 0.23076 0.18880 -0.0493 -0.0299 0.10516 0.01341 -0.0102 0.13440 0.08443 0.04661 -0.0387 0.07064 0.02736 -0.0733 -0.0046 -0.0183 -0.0570 0.05972 -0.0079 -0.0937 0.11007 0.01023 0.12038 -0.4137 -0.0288 0.01342 0.03854 0.57840 +-0.1939 -0.1237 0.45043 0.58570 0.75750 -0.0302 0.34038 -0.2635 0.13860 -0.3372 -0.1450 0.38045 -0.0535 -1.0004 -0.8935 0.44737 0.41739 1.22884 0.25607 0.50730 -0.2386 -0.7547 0.23397 0.33564 0.00777 -0.0193 -0.2838 -0.1348 -0.3635 -0.2566 0.08618 0.16133 0.16260 -0.1252 0.13475 0.27853 0.16400 0.01767 0.05317 0.44101 0.42541 0.13493 0.19814 0.18219 -0.0502 -0.0356 0.11308 0.01365 -0.0128 0.13392 0.08760 0.04435 -0.0420 0.06423 0.02223 -0.0787 -0.0021 -0.0158 -0.0563 0.05763 -0.0074 -0.0865 0.10599 0.02119 0.12120 -0.4158 -0.0302 0.01356 0.03666 0.57541 +-0.0284 -0.0750 0.51348 0.64387 0.81589 0.02631 0.36540 -0.1275 0.19420 -0.3757 -0.0919 0.41140 0.05483 -0.8095 -0.7545 0.48894 0.40494 1.19337 0.21713 0.48088 -0.1016 -0.7355 0.22688 0.36382 0.01853 -0.0310 -0.2172 -0.2233 -0.4263 -0.2531 0.10219 0.13054 0.21388 -0.1291 0.21822 0.15323 0.20093 0.02835 0.02686 0.45130 0.39300 0.15760 0.19852 0.17217 -0.0533 -0.0403 0.11787 0.00950 -0.0225 0.13356 0.06432 0.03942 -0.0491 0.07189 0.00842 -0.0746 -0.0098 -0.0253 -0.0662 0.05074 -0.0020 -0.0769 0.11056 0.01669 0.12154 -0.4175 -0.0321 0.01311 0.03567 0.57299 +-0.3277 -0.1046 0.38421 0.43662 0.74202 -0.1512 0.33919 -0.2117 0.16507 -0.3088 -0.0482 0.36250 -0.0070 -0.9659 -0.7873 0.37684 0.41831 1.21735 0.23784 0.46333 -0.1641 -0.8023 0.25779 0.32550 -0.0444 -0.0141 -0.2840 -0.2085 -0.3881 -0.2900 0.09705 0.14119 0.19090 -0.1280 0.17723 0.19301 0.18859 0.01541 0.02792 0.46903 0.41020 0.15704 0.17784 0.16941 -0.0419 -0.0322 0.11922 -0.0029 -0.0175 0.13668 0.06887 0.04691 -0.0528 0.07406 0.02214 -0.0696 -0.0091 -0.0245 -0.0572 0.06361 -0.0023 -0.0822 0.10661 0.01259 0.12341 -0.4190 -0.0344 0.01239 0.03485 0.57668 +-0.3564 -0.1130 0.43069 0.22742 0.53917 -0.1464 0.32330 -0.2559 0.13539 -0.3288 0.03330 0.24735 -0.0714 -1.0899 -0.9298 0.32779 0.42973 1.20353 0.26928 0.44315 -0.1069 -0.7646 0.37779 0.35265 -0.0872 0.00155 -0.2767 -0.3039 -0.4528 -0.3135 0.12346 0.15477 0.17943 -0.1452 0.11741 0.20294 0.23346 0.03612 -0.0050 0.46267 0.46643 0.13723 0.21607 0.19098 -0.0386 -0.0267 0.12810 -0.0164 -0.0300 0.14669 0.06285 0.04284 -0.0475 0.07404 0.01610 -0.0769 -0.0115 -0.0216 -0.0537 0.06222 -0.0015 -0.0808 0.09619 0.01029 0.12539 -0.4197 -0.0364 0.01168 0.03337 0.59117 +-0.1073 -0.0837 0.56692 0.37357 0.60880 0.01860 0.29439 -0.2187 0.16444 -0.3413 -0.0654 0.28481 -0.0484 -1.0689 -0.9626 0.47773 0.41900 1.23702 0.27927 0.45386 -0.1903 -0.7918 0.34588 0.29394 -0.1085 0.01316 -0.3510 -0.2214 -0.4052 -0.3029 0.10508 0.16794 0.19188 -0.1336 0.11248 0.23281 0.23510 0.04233 0.01167 0.43274 0.49053 0.12073 0.24644 0.19611 -0.0461 -0.0374 0.12296 0.01389 -0.0262 0.14187 0.06097 0.04098 -0.0483 0.06532 0.01960 -0.0741 -0.0082 -0.0229 -0.0565 0.05289 -0.0078 -0.0798 0.10303 0.01299 0.12810 -0.4210 -0.0379 0.01101 0.03191 0.60222 +-0.0753 -0.0878 0.65741 0.30735 0.53638 -0.0302 0.30868 -0.2278 0.18822 -0.2316 0.03590 0.35652 -0.0531 -1.0656 -1.0880 0.57952 0.38759 1.25479 0.25802 0.50636 -0.2503 -0.8234 0.27378 0.24825 -0.1233 0.00616 -0.4553 -0.1723 -0.4097 -0.3234 0.12859 0.17325 0.19684 -0.1391 0.13040 0.24560 0.19037 0.04645 0.01130 0.43851 0.51828 0.12501 0.27691 0.18496 -0.0678 -0.0359 0.12263 0.02821 -0.0147 0.12691 0.08139 0.04383 -0.0401 0.06766 0.02515 -0.0801 -0.0042 -0.0221 -0.0647 0.05585 -0.0019 -0.0778 0.11209 0.01743 0.13048 -0.4207 -0.0391 0.01045 0.03070 0.61403 +-0.2706 -0.1430 0.48934 0.32097 0.57457 -0.1097 0.32150 -0.2876 0.14404 -0.1816 0.02791 0.34039 -0.0692 -1.0400 -1.1391 0.49073 0.39445 1.25214 0.26688 0.50083 -0.2945 -0.8047 0.26651 0.27585 -0.0532 -0.0124 -0.3956 -0.1568 -0.3479 -0.3035 0.12243 0.17851 0.17287 -0.1477 0.11045 0.29056 0.15712 0.02479 0.03269 0.45628 0.51877 0.13115 0.25150 0.18472 -0.0489 -0.0316 0.12432 0.01427 -0.0258 0.12716 0.07011 0.04896 -0.0413 0.07391 0.02790 -0.0815 -0.0045 -0.0211 -0.0598 0.05924 0.00508 -0.0883 0.10956 0.01574 0.13209 -0.4207 -0.0397 0.00985 0.03011 0.61593 +-0.4637 -0.1612 0.42023 0.13713 0.47062 -0.2200 0.32130 -0.3430 0.16526 -0.1870 0.04317 0.27814 -0.0758 -1.0842 -1.0566 0.37840 0.41035 1.24881 0.25798 0.47071 -0.2035 -0.7967 0.29993 0.29572 -0.0264 -0.0116 -0.3593 -0.2618 -0.4406 -0.3142 0.14185 0.16344 0.18050 -0.1414 0.07224 0.27187 0.20226 0.03862 0.00665 0.43864 0.48070 0.12457 0.21099 0.17081 -0.0492 -0.0314 0.12212 0.00775 -0.0390 0.16682 0.06413 0.04345 -0.0386 0.07472 0.01879 -0.0786 -0.0100 -0.0212 -0.0650 0.06311 -0.0015 -0.0736 0.10965 0.01914 0.13234 -0.4197 -0.0393 0.00934 0.02940 0.61815 +-0.5084 -0.1706 0.39968 0.05170 0.43699 -0.1947 0.31529 -0.3369 0.11816 -0.1704 0.06604 0.20388 -0.0974 -1.1362 -1.0973 0.30798 0.42398 1.23555 0.25610 0.43201 -0.2140 -0.8141 0.34474 0.24413 -0.0728 0.02618 -0.3864 -0.2504 -0.4137 -0.3265 0.16657 0.13772 0.17327 -0.1390 0.04595 0.27967 0.18427 0.02176 0.03353 0.42679 0.49592 0.12626 0.23255 0.17713 -0.0415 -0.0331 0.12392 -0.0052 -0.0384 0.15510 0.05824 0.05005 -0.0423 0.07062 0.01734 -0.0820 -0.0089 -0.0191 -0.0644 0.06156 0.00035 -0.0808 0.11054 0.01564 0.13150 -0.4194 -0.0384 0.00892 0.02858 0.62464 +-0.4378 -0.1451 0.43951 0.09404 0.47721 -0.1755 0.34420 -0.3019 0.14779 -0.2472 0.00873 0.19384 -0.0782 -1.1676 -1.0522 0.36053 0.42951 1.22333 0.27175 0.40877 -0.2379 -0.7846 0.41521 0.27244 -0.1066 0.02870 -0.3554 -0.2498 -0.3832 -0.3119 0.14879 0.14649 0.16831 -0.1406 0.03804 0.24651 0.18754 0.03029 0.02136 0.45150 0.52578 0.13055 0.24492 0.20335 -0.0508 -0.0347 0.12466 0.00587 -0.0281 0.15119 0.06080 0.04946 -0.0454 0.07182 0.01616 -0.0775 -0.0148 -0.0222 -0.0592 0.05583 0.00217 -0.0784 0.10581 0.01060 0.12842 -0.4187 -0.0374 0.00877 0.02891 0.61688 +-0.3437 -0.1275 0.50223 0.21265 0.51709 -0.1063 0.32054 -0.2659 0.16506 -0.3094 -0.0304 0.24385 -0.0965 -1.1361 -1.0083 0.40846 0.43789 1.20698 0.25901 0.40541 -0.2184 -0.7614 0.41027 0.30028 -0.1231 0.02237 -0.3510 -0.2422 -0.4074 -0.3147 0.14913 0.15912 0.17577 -0.1442 0.04325 0.21959 0.20988 0.04063 0.01509 0.43133 0.53823 0.11688 0.23253 0.20308 -0.0555 -0.0345 0.12500 0.00646 -0.0309 0.14528 0.05861 0.04269 -0.0430 0.06635 0.01725 -0.0838 -0.0107 -0.0190 -0.0628 0.05923 0.00156 -0.0750 0.11262 0.02127 0.12536 -0.4195 -0.0367 0.00868 0.02990 0.60766 +-0.1526 -0.1169 0.53084 0.38729 0.60753 0.01764 0.31686 -0.2508 0.15405 -0.3115 -0.0700 0.34562 -0.0412 -1.0772 -0.9764 0.46155 0.42092 1.24358 0.24482 0.45046 -0.2468 -0.7808 0.35590 0.29543 -0.1187 0.02201 -0.3741 -0.2006 -0.3789 -0.3191 0.13847 0.16112 0.19035 -0.1439 0.09541 0.24414 0.18775 0.02324 0.02079 0.45478 0.49946 0.12162 0.25351 0.19446 -0.0553 -0.0382 0.12372 0.01334 -0.0185 0.12761 0.05802 0.04518 -0.0423 0.06549 0.02268 -0.0813 -0.0096 -0.0217 -0.0601 0.05162 -0.0006 -0.0836 0.10603 0.01936 0.12195 -0.4206 -0.0369 0.00846 0.03031 0.60549 +-0.0827 -0.0691 0.62263 0.42716 0.65380 0.02540 0.33982 -0.2430 0.18477 -0.1898 0.00078 0.37428 -0.0592 -1.0757 -1.0499 0.5998 0.37489 1.24102 0.24090 0.51129 -0.2664 -0.7855 0.25599 0.27732 -0.1235 0.00652 -0.3950 -0.1762 -0.3644 -0.3247 0.12182 0.17141 0.18626 -0.1330 0.12814 0.28803 0.17351 0.01909 0.03010 0.46738 0.48336 0.12162 0.24829 0.19570 -0.0529 -0.0358 0.12519 0.02469 -0.0085 0.13214 0.07546 0.04190 -0.0402 0.05874 0.02655 -0.0801 -0.0086 -0.0233 -0.0610 0.05415 -0.0050 -0.0852 0.11279 0.02253 0.12003 -0.4224 -0.0372 0.00847 0.03048 0.59230 +-0.1187 -0.1123 0.59884 0.40237 0.60117 -0.0338 0.32900 -0.2925 0.14750 -0.1880 0.01925 0.35128 -0.0625 -1.1123 -1.1340 0.59343 0.36890 1.29065 0.23838 0.52189 -0.3245 -0.7895 0.19678 0.26141 -0.0863 -0.0119 -0.4120 -0.1469 -0.3423 -0.3062 0.11349 0.18633 0.18200 -0.1306 0.13271 0.32324 0.14160 0.01404 0.04180 0.46948 0.49006 0.12507 0.25052 0.17788 -0.0467 -0.0373 0.12356 0.01667 -0.0027 0.13858 0.08070 0.04027 -0.0405 0.06216 0.02795 -0.0768 -0.0063 -0.0205 -0.0628 0.05900 -0.0066 -0.0849 0.11456 0.01707 0.11823 -0.4234 -0.0374 0.00862 0.03101 0.58282 +-0.3252 -0.1625 0.44496 0.31234 0.55149 -0.1116 0.33639 -0.3356 0.13289 -0.2872 0.01958 0.28606 -0.0935 -1.1790 -1.0957 0.46971 0.43301 1.22846 0.24329 0.48498 -0.2825 -0.7891 0.27240 0.30180 -0.0611 0.00801 -0.3376 -0.1418 -0.3768 -0.2811 0.11495 0.17412 0.14748 -0.1291 0.12027 0.32180 0.18390 0.00826 0.01782 0.44034 0.47466 0.13541 0.22729 0.17835 -0.0440 -0.0347 0.12243 0.00318 -0.0181 0.14638 0.07607 0.04516 -0.0443 0.06557 0.02340 -0.0739 -0.0084 -0.0192 -0.0584 0.05764 -0.0056 -0.0856 0.10969 0.01436 0.11746 -0.4251 -0.0381 0.00848 0.03157 0.58414 +-0.3069 -0.1420 0.46908 0.30438 0.57017 -0.1420 0.29855 -0.2891 0.15582 -0.2846 -0.0045 0.29676 -0.0918 -1.0653 -1.0014 0.39999 0.43522 1.22557 0.25690 0.40600 -0.2032 -0.7915 0.37065 0.29498 -0.1239 0.00285 -0.3659 -0.2092 -0.4043 -0.2951 0.12364 0.16288 0.15304 -0.1348 0.09856 0.22984 0.17645 0.02444 0.01234 0.41908 0.49239 0.13726 0.22904 0.18318 -0.0417 -0.0340 0.12523 0.00714 -0.0231 0.14663 0.07191 0.04576 -0.0385 0.07145 0.02102 -0.0788 -0.0050 -0.0159 -0.0552 0.05524 0.00077 -0.0842 0.10453 0.01505 0.11765 -0.4263 -0.0390 0.00854 0.03091 0.59169 +-0.1566 -0.0839 0.54540 0.39704 0.64721 0.01596 0.32813 -0.2028 0.19132 -0.3442 -0.1267 0.37559 -0.0294 -0.9551 -0.8806 0.40077 0.44012 1.22174 0.24377 0.42939 -0.1948 -0.7629 0.37143 0.29258 -0.1292 0.02473 -0.4066 -0.1754 -0.3660 -0.3168 0.13723 0.12666 0.19444 -0.1278 0.08917 0.18975 0.15229 0.02224 0.04462 0.45071 0.50715 0.11415 0.25038 0.19224 -0.0626 -0.0384 0.12425 0.01094 -0.0207 0.14124 0.07224 0.04626 -0.0530 0.06174 0.01705 -0.0792 -0.0088 -0.0238 -0.0602 0.05536 -0.0024 -0.0792 0.10990 0.01600 0.11930 -0.4271 -0.0397 0.00817 0.03030 0.60124 +-0.1751 -0.0860 0.55546 0.37941 0.67185 0.02135 0.33108 -0.2234 0.17814 -0.2945 -0.1044 0.37705 -0.0229 -0.9731 -0.9304 0.44104 0.42574 1.23856 0.23825 0.47907 -0.2376 -0.7399 0.35216 0.26842 -0.1546 0.03891 -0.4834 -0.1173 -0.3475 -0.3430 0.15020 0.12149 0.19418 -0.1330 0.03312 0.20255 0.13212 0.01068 0.05957 0.45581 0.53800 0.11070 0.26584 0.19434 -0.0707 -0.0418 0.12709 0.01451 -0.0200 0.14487 0.06811 0.04639 -0.0540 0.06205 0.01855 -0.0811 -0.0086 -0.0246 -0.0620 0.05058 0.00116 -0.0771 0.10425 0.01576 0.12089 -0.4256 -0.0401 0.00803 0.03037 0.59795 +-0.2636 -0.1018 0.51301 0.40685 0.68512 -0.1035 0.31856 -0.2781 0.14916 -0.2615 -0.0121 0.36768 -0.0765 -1.0355 -0.9614 0.42033 0.41849 1.26190 0.23763 0.50536 -0.2010 -0.7679 0.30110 0.29927 -0.1413 0.01240 -0.4217 -0.1471 -0.3624 -0.3115 0.14213 0.14008 0.18161 -0.1318 0.09917 0.23824 0.14871 0.00536 0.04201 0.45568 0.48924 0.12904 0.23986 0.19149 -0.0535 -0.0383 0.12996 0.01552 -0.0197 0.14790 0.07617 0.04236 -0.0405 0.07068 0.02038 -0.0795 -0.0069 -0.0190 -0.0602 0.06214 -0.0080 -0.0786 0.10517 0.01554 0.12181 -0.4239 -0.0393 0.00795 0.03121 0.58184 +-0.3654 -0.1399 0.42405 0.54815 0.76531 -0.1680 0.28296 -0.1725 0.13359 -0.3393 -0.0138 0.45020 0.03008 -0.9978 -0.7955 0.30133 0.48570 1.07083 0.30088 0.24499 -0.1740 -1.0139 -0.0024 0.28129 -0.1844 0.11202 -0.3821 -0.1847 -0.2423 -0.3050 0.25276 0.08927 0.08993 -0.2871 0.06670 0.20383 -0.0102 -0.0172 -0.0674 0.37472 0.50756 0.26865 0.08690 0.25457 -0.0885 -0.0092 0.10358 0.02437 -0.0360 0.13088 0.08298 0.05039 -0.0465 0.08360 0.02724 -0.0841 -0.0053 -0.0313 -0.0583 0.06613 0.00310 -0.0809 0.09008 0.01727 0.12080 -0.4215 -0.0383 0.00822 0.03143 0.58174 +-0.1325 -0.0376 0.59691 0.76252 0.84941 -0.0286 0.28179 -0.0466 0.16847 -0.4257 -0.1907 0.56641 0.12820 -0.8421 -0.5308 0.38300 0.60031 0.96539 0.36490 0.06239 -0.1110 -1.2038 -0.2007 0.27360 -0.2905 0.20541 -0.4801 -0.2009 -0.1743 -0.3449 0.33969 0.02375 0.01830 -0.4242 0.00293 0.14248 -0.1024 -0.0049 -0.1148 0.31992 0.59441 0.35919 0.02603 0.32422 -0.1549 0.01527 0.08086 0.06902 -0.0512 0.11552 0.08400 0.05073 -0.0463 0.10004 0.02814 -0.0764 -0.0194 -0.0539 -0.0525 0.07922 0.00440 -0.0802 0.06588 0.01984 0.11799 -0.4199 -0.0370 0.00858 0.03087 0.59149 +-0.0128 0.04283 0.63869 0.87546 0.88172 0.10933 0.30120 -0.0115 0.15851 -0.4799 -0.3240 0.51141 0.18844 -0.8205 -0.3705 0.33421 0.63092 0.88402 0.40820 -0.0617 -0.0938 -1.1710 -0.2693 0.26762 -0.2187 0.21148 -0.3709 -0.3364 -0.1546 -0.2674 0.35664 0.01742 0.00613 -0.4578 -0.0058 0.11986 -0.0783 0.01821 -0.1604 0.28707 0.60231 0.39255 -0.0075 0.33849 -0.1717 0.00059 0.07081 0.06791 -0.0382 0.11308 0.07614 0.05331 -0.0452 0.09326 0.02504 -0.0814 -0.0056 -0.0439 -0.0670 0.06803 0.00694 -0.0854 0.07753 0.01974 0.11434 -0.4182 -0.0356 0.00944 0.03168 0.59012 +0.02957 -0.0118 0.65466 0.81710 0.77997 0.21815 0.25836 -0.0366 0.17893 -0.4614 -0.3003 0.42807 0.12158 -0.9691 -0.4814 0.40690 0.63791 0.86059 0.42706 -0.0784 -0.0266 -1.1389 -0.1327 0.33269 -0.1690 0.17082 -0.3096 -0.4208 -0.2685 -0.3021 0.31724 0.09306 0.01114 -0.4220 0.04883 0.11509 0.05040 0.05087 -0.1883 0.29008 0.58438 0.36488 0.02296 0.33631 -0.1658 0.00106 0.07265 0.06629 -0.0366 0.12034 0.05946 0.05592 -0.0483 0.09615 0.03023 -0.0728 -0.0065 -0.0380 -0.0504 0.07177 0.00661 -0.0817 0.07724 0.00816 0.11230 -0.4177 -0.0344 0.01042 0.03322 0.58610 +0.06610 -0.0161 0.64912 0.61636 0.75577 0.11788 0.30217 -0.1785 0.16868 -0.3633 -0.1443 0.36104 0.03280 -1.0777 -0.7841 0.50617 0.47204 1.05492 0.33848 0.21803 -0.1390 -1.0001 -0.0277 0.32322 -0.1432 0.07956 -0.3321 -0.3237 -0.3504 -0.2930 0.21340 0.14364 0.10653 -0.2869 0.11224 0.20469 0.11462 0.04006 -0.0933 0.37621 0.53571 0.26390 0.12385 0.25817 -0.1104 -0.0077 0.08797 0.03585 -0.0218 0.11765 0.08264 0.05167 -0.0432 0.07785 0.02221 -0.0679 -0.0073 -0.0333 -0.0555 0.06698 -0.0017 -0.0962 0.08881 0.01090 0.11121 -0.4166 -0.0335 0.01135 0.03442 0.59821 +-0.0105 -0.0848 0.60971 0.45961 0.67291 0.04068 0.35931 -0.2594 0.14946 -0.2491 -0.0809 0.31432 -0.0478 -1.1094 -1.0410 0.63829 0.36843 1.26834 0.25741 0.48740 -0.2386 -0.7643 0.21778 0.27233 -0.0381 -0.0182 -0.3287 -0.1710 -0.3837 -0.2874 0.10530 0.18303 0.17280 -0.1361 0.15132 0.30178 0.19319 0.01826 0.04194 0.45936 0.45223 0.14473 0.22440 0.19206 -0.0442 -0.0243 0.11826 0.02779 -0.0103 0.14024 0.07950 0.04716 -0.0347 0.06067 0.02518 -0.0722 -0.0060 -0.0225 -0.0575 0.06081 -0.0030 -0.0974 0.11093 0.01515 0.11251 -0.4160 -0.0323 0.01259 0.03561 0.59009 +-0.2678 -0.1765 0.44034 0.45305 0.65445 -0.1396 0.35881 -0.3052 0.14188 -0.3026 -0.0444 0.29752 -0.0546 -1.1103 -1.0119 0.47806 0.40582 1.23003 0.26493 0.49455 -0.2742 -0.7735 0.19952 0.34117 -0.0232 -0.0071 -0.3370 -0.1201 -0.3518 -0.2731 0.09450 0.17753 0.14695 -0.1195 0.12249 0.32709 0.18619 0.01357 0.03126 0.47065 0.46151 0.13174 0.21599 0.18009 -0.0495 -0.0420 0.11763 0.01116 -0.0155 0.13972 0.07403 0.04873 -0.0502 0.06284 0.02283 -0.0771 -0.0045 -0.0234 -0.0578 0.06089 -0.0023 -0.0894 0.10682 0.01505 0.11457 -0.4150 -0.0308 0.01386 0.03707 0.58882 +-0.1767 -0.1234 0.42169 0.74639 0.89709 -0.0206 0.33467 -0.2103 0.19128 -0.3647 -0.1488 0.47228 0.04571 -0.8486 -0.6520 0.40416 0.40427 1.20271 0.20835 0.53075 -0.2635 -0.7677 0.19953 0.29778 -0.0766 0.01144 -0.4067 -0.0510 -0.3340 -0.2720 0.10596 0.13446 0.19435 -0.1073 0.19345 0.19417 0.10820 -0.0083 0.05710 0.44648 0.40516 0.14620 0.19994 0.15645 -0.0301 -0.0317 0.12666 0.00426 -0.0202 0.13715 0.07150 0.04435 -0.0453 0.06865 0.02111 -0.0745 -0.0046 -0.0249 -0.0503 0.06497 -0.0011 -0.0805 0.10194 0.01718 0.11721 -0.4145 -0.0302 0.01468 0.03811 0.59951 +0.02603 -0.0196 0.52616 0.80752 0.87356 0.01991 0.37757 -0.1247 0.20386 -0.3732 -0.2465 0.44931 0.09911 -0.6906 -0.5708 0.46149 0.38529 1.20308 0.20337 0.49613 -0.2099 -0.7060 0.16770 0.29538 -0.1119 -0.0060 -0.3653 -0.1750 -0.3533 -0.2425 0.09466 0.10353 0.21012 -0.1179 0.19925 0.19102 0.10665 -0.0037 0.05768 0.47005 0.41530 0.13381 0.22449 0.17695 -0.0497 -0.0449 0.13006 0.01183 0.00191 0.13740 0.06906 0.03682 -0.0530 0.06679 0.01608 -0.0785 -0.0041 -0.0186 -0.0684 0.05225 0.00219 -0.0884 0.09941 0.01870 0.12035 -0.4132 -0.0298 0.01532 0.03821 0.60274 +-0.1371 -0.0688 0.45743 0.53843 0.76166 -0.1133 0.39834 -0.1761 0.21566 -0.3061 -0.0550 0.37660 0.02162 -0.8029 -0.8604 0.43208 0.38106 1.25860 0.22945 0.53235 -0.2096 -0.7795 0.17965 0.33962 -0.0539 0.00481 -0.3831 -0.1804 -0.4107 -0.3207 0.11240 0.11949 0.22826 -0.1418 0.18585 0.20718 0.15976 0.01851 0.04621 0.46758 0.46335 0.14921 0.22332 0.17076 -0.0624 -0.0260 0.12464 0.02768 -0.0184 0.16270 0.06897 0.04059 -0.0533 0.07096 0.01016 -0.0693 -0.0169 -0.0241 -0.0586 0.05921 -0.0003 -0.0742 0.09274 0.00703 0.12412 -0.4122 -0.0296 0.01547 0.03790 0.60970 +-0.4379 -0.1713 0.34744 0.36608 0.70588 -0.1813 0.38632 -0.2320 0.17123 -0.2964 0.03707 0.35096 -0.0230 -0.9784 -0.9028 0.36585 0.40064 1.23405 0.22528 0.51805 -0.1947 -0.7849 0.25824 0.34659 -0.1417 0.01277 -0.3619 -0.1795 -0.3754 -0.3115 0.12476 0.12863 0.20338 -0.1125 0.16397 0.21302 0.12667 0.00196 0.03280 0.46224 0.43317 0.16172 0.18321 0.14868 -0.0448 -0.0354 0.12125 0.00117 -0.0291 0.14579 0.07146 0.04410 -0.0456 0.06789 0.01473 -0.0713 -0.0085 -0.0225 -0.0588 0.06098 0.00159 -0.0812 0.09708 0.01012 0.12651 -0.4105 -0.0292 0.01552 0.03822 0.59613 +-0.3976 -0.1454 0.39109 0.10942 0.53104 -0.2547 0.34478 -0.2342 0.14253 -0.2558 0.04539 0.24243 -0.0316 -1.0087 -0.9929 0.24141 0.44106 1.20995 0.25239 0.40202 -0.1697 -0.7766 0.38385 0.35757 -0.0982 0.00939 -0.2769 -0.2830 -0.4563 -0.3146 0.13728 0.12073 0.18328 -0.1483 0.10027 0.21488 0.18937 0.01740 0.02672 0.45132 0.46048 0.15865 0.20726 0.16852 -0.0476 -0.0341 0.12205 0.00351 -0.0305 0.14695 0.05976 0.04570 -0.0549 0.07474 0.01408 -0.0847 -0.0092 -0.0225 -0.0588 0.05782 -0.0012 -0.0754 0.10211 0.01162 0.12871 -0.4105 -0.0295 0.01533 0.03826 0.58351 +-0.1140 -0.0538 0.60264 0.34738 0.61338 0.00620 0.33341 -0.1687 0.20699 -0.3004 -0.1332 0.30917 0.02669 -0.8707 -0.8436 0.41544 0.44255 1.20902 0.24410 0.42075 -0.1249 -0.7931 0.36559 0.37466 -0.1548 0.01980 -0.3376 -0.3279 -0.4869 -0.3295 0.17137 0.14601 0.22109 -0.1438 0.14231 0.13406 0.21072 0.02046 0.00384 0.44533 0.48516 0.12250 0.23564 0.18131 -0.0648 -0.0227 0.12605 0.00945 -0.0296 0.15881 0.07223 0.04321 -0.0518 0.05888 0.00808 -0.0739 -0.0152 -0.0248 -0.0646 0.05889 0.00374 -0.0626 0.10239 0.01433 0.12974 -0.4118 -0.0313 0.01511 0.03693 0.58687 +0.35026 0.05805 0.86065 0.68381 0.72294 0.31318 0.36534 -0.0497 0.24536 -0.3312 -0.4634 0.26678 0.21304 -0.5217 -0.3852 0.36411 0.43907 1.09436 0.24644 0.24961 -0.0947 -0.6641 0.44371 0.35729 -0.0439 -0.0133 -0.1476 -0.4989 -0.5091 -0.2430 0.15232 0.17467 0.20752 -0.1504 0.21350 0.08937 0.28407 0.06128 0.02091 0.42103 0.42687 0.13071 0.27406 0.20622 -0.0641 -0.0346 0.10789 0.06197 0.00192 0.15289 0.05485 0.04696 -0.0467 0.06073 0.01827 -0.0619 -0.0340 -0.0251 -0.0670 0.06413 0.00534 -0.0724 0.10542 0.01366 0.13105 -0.4145 -0.0335 0.01482 0.03459 0.58328 +0.07907 -0.0049 0.61806 0.63748 0.76942 0.07550 0.39699 -0.1534 0.16493 -0.3446 -0.2942 0.18150 0.11959 -0.7041 -0.5486 0.29703 0.44281 1.09283 0.24182 0.31207 -0.0934 -0.7037 0.36153 0.40748 0.03031 -0.0258 -0.1237 -0.4675 -0.5189 -0.2643 0.12865 0.17229 0.19342 -0.1384 0.26771 0.12617 0.27484 0.04921 -0.0166 0.48333 0.40468 0.15887 0.23332 0.18420 -0.0644 -0.0282 0.11448 0.02361 -0.0186 0.14832 0.06497 0.04752 -0.0533 0.06534 0.01703 -0.0713 -0.0125 -0.0272 -0.0661 0.05644 0.00851 -0.0718 0.10380 0.00915 0.13207 -0.4171 -0.0360 0.01465 0.03329 0.57648 +-0.2448 -0.1254 0.49038 0.31439 0.61852 -0.0709 0.34011 -0.2173 0.19022 -0.3267 -0.0080 0.23426 -0.0373 -1.0547 -0.8389 0.41709 0.42415 1.17938 0.25287 0.41321 -0.0719 -0.7708 0.32914 0.36479 -0.1138 0.00131 -0.2692 -0.3350 -0.4915 -0.3119 0.10975 0.16954 0.18231 -0.1451 0.19863 0.16481 0.28324 0.03798 -0.0391 0.46328 0.42889 0.14607 0.20580 0.16407 -0.0472 -0.0355 0.12692 -0.0039 -0.0234 0.13349 0.06421 0.04811 -0.0516 0.07379 0.01678 -0.0749 -0.0137 -0.0301 -0.0537 0.06368 -0.0027 -0.0804 0.10415 0.00995 0.13365 -0.4199 -0.0385 0.01415 0.03214 0.57721 +-0.3173 -0.1222 0.42882 0.36299 0.60929 -0.0893 0.33434 -0.2496 0.15199 -0.3226 -0.0905 0.31370 0.00413 -1.0289 -0.8735 0.35878 0.43675 1.24600 0.23942 0.44567 -0.2657 -0.7501 0.37433 0.26950 -0.0983 0.01206 -0.4065 -0.1818 -0.3446 -0.2771 0.14833 0.11594 0.17037 -0.1385 0.05571 0.21060 0.15097 0.02078 0.04545 0.44765 0.50194 0.12580 0.25042 0.18818 -0.0541 -0.0294 0.12076 0.01131 -0.0174 0.14098 0.06132 0.04655 -0.0479 0.06917 0.01229 -0.0782 -0.0135 -0.0282 -0.0574 0.05777 0.00110 -0.0781 0.10226 0.00956 0.13483 -0.4214 -0.0407 0.01361 0.03041 0.59215 +-0.1770 -0.0794 0.49197 0.52020 0.68240 -0.0182 0.30769 -0.2420 0.18642 -0.3877 -0.2410 0.41295 -0.0009 -0.9710 -0.7399 0.36574 0.44119 1.24163 0.23971 0.43116 -0.2698 -0.7563 0.35301 0.23437 -0.0987 0.01030 -0.4248 -0.1310 -0.3129 -0.2614 0.13680 0.12457 0.17080 -0.1001 0.08207 0.20916 0.11406 0.03176 0.06050 0.44662 0.50137 0.10133 0.24092 0.19293 -0.0592 -0.0343 0.12934 0.01673 -0.0205 0.14607 0.05852 0.04545 -0.0369 0.06539 0.02079 -0.0830 -0.0117 -0.0187 -0.0602 0.05195 0.00052 -0.0834 0.11600 0.01490 0.13669 -0.4231 -0.0423 0.01309 0.02868 0.60103 +-0.1279 -0.0923 0.56140 0.55630 0.69879 0.05681 0.28737 -0.2494 0.17952 -0.3605 -0.2132 0.37311 -0.0220 -1.0301 -0.7618 0.41611 0.44665 1.22976 0.27488 0.46967 -0.2424 -0.7948 0.31706 0.29069 -0.0625 0.00839 -0.3782 -0.1828 -0.3909 -0.2739 0.11470 0.15663 0.19738 -0.1106 0.13593 0.20334 0.16829 0.04317 0.03305 0.44970 0.47058 0.11809 0.23488 0.19233 -0.0572 -0.0372 0.11916 0.01202 -0.0148 0.13735 0.07462 0.04340 -0.0473 0.06567 0.01785 -0.0741 -0.0045 -0.0179 -0.0598 0.05686 -0.0043 -0.0791 0.10959 0.01337 0.13841 -0.4231 -0.0434 0.01255 0.02718 0.61884 +-0.0842 -0.1079 0.52034 0.53169 0.68645 0.03508 0.31590 -0.2737 0.15179 -0.3069 -0.1238 0.35308 -0.0262 -1.0819 -0.9258 0.51658 0.39816 1.24727 0.24482 0.49559 -0.2852 -0.7954 0.26029 0.29154 -0.0672 0.00135 -0.3774 -0.1564 -0.3678 -0.2846 0.10136 0.16771 0.18469 -0.1223 0.13637 0.27690 0.15294 0.03370 0.02901 0.46317 0.46310 0.12795 0.24047 0.18212 -0.0588 -0.0325 0.12097 0.01104 -0.0162 0.13999 0.07071 0.04381 -0.0451 0.06712 0.02156 -0.0783 -0.0077 -0.0173 -0.0566 0.05237 -0.0123 -0.0909 0.11314 0.01364 0.13941 -0.4221 -0.0438 0.01188 0.02624 0.63268 +-0.0309 -0.1142 0.57361 0.55649 0.71897 0.01342 0.35893 -0.2830 0.15511 -0.2533 -0.0958 0.34371 -0.0351 -1.1072 -1.0010 0.56204 0.39412 1.26432 0.23850 0.49476 -0.3064 -0.7968 0.23523 0.27885 -0.0384 -0.0144 -0.3669 -0.1314 -0.3572 -0.2760 0.10752 0.16819 0.15497 -0.1386 0.12397 0.30622 0.16220 0.01921 0.02689 0.46650 0.46104 0.12582 0.24141 0.18827 -0.0471 -0.0323 0.12499 0.01942 -0.0114 0.13842 0.07888 0.04331 -0.0373 0.05979 0.02366 -0.0779 -0.0044 -0.0198 -0.0574 0.06167 -0.0082 -0.0921 0.11370 0.01392 0.14037 -0.4198 -0.0434 0.01140 0.02510 0.63934 +-0.0400 -0.0997 0.61157 0.50763 0.72154 0.03306 0.39416 -0.3187 0.14588 -0.2330 -0.0380 0.30870 -0.0654 -1.1449 -1.0845 0.64860 0.37282 1.26642 0.23838 0.45750 -0.3366 -0.7813 0.19372 0.28254 -0.0195 -0.0277 -0.3240 -0.1301 -0.3347 -0.2667 0.08495 0.19685 0.15038 -0.1280 0.13948 0.35394 0.15534 0.02737 0.04196 0.47160 0.46471 0.14331 0.23714 0.18590 -0.0403 -0.0343 0.11043 0.02019 0.00224 0.13496 0.07837 0.04670 -0.0367 0.06814 0.02963 -0.0769 -0.0089 -0.0217 -0.0597 0.05942 -0.0071 -0.0881 0.11352 0.01605 0.14118 -0.4179 -0.0428 0.01106 0.02387 0.64870 +-0.1690 -0.1024 0.54339 0.38394 0.61716 -0.0059 0.36100 -0.2910 0.15525 -0.2476 -0.0397 0.24852 -0.0822 -1.1248 -1.0650 0.52225 0.40424 1.25798 0.24287 0.47953 -0.2542 -0.7664 0.22810 0.28569 -0.0355 -0.0205 -0.3262 -0.1405 -0.3492 -0.2872 0.09964 0.19361 0.15293 -0.1366 0.12059 0.31628 0.18130 0.01322 0.03379 0.46223 0.46698 0.12706 0.22414 0.19057 -0.0432 -0.0383 0.11273 0.00721 -0.0068 0.14761 0.08056 0.04185 -0.0345 0.07002 0.02826 -0.0736 -0.0080 -0.0206 -0.0610 0.06376 -0.0027 -0.0870 0.11268 0.01990 0.13972 -0.4159 -0.0418 0.01073 0.02402 0.64346 +-0.4486 -0.1895 0.40131 0.27383 0.58010 -0.2073 0.38658 -0.3228 0.11604 -0.2449 0.03501 0.28497 -0.1008 -1.1070 -1.0455 0.41130 0.40847 1.26497 0.25132 0.47625 -0.2487 -0.7812 0.28666 0.33396 -0.0534 -0.0042 -0.3383 -0.1880 -0.3666 -0.3002 0.13289 0.15110 0.14941 -0.1298 0.10122 0.29207 0.16269 0.00457 0.01710 0.44295 0.47098 0.14310 0.20876 0.17768 -0.0410 -0.0383 0.11377 -0.0021 -0.0270 0.14735 0.07378 0.04541 -0.0400 0.06627 0.02918 -0.0748 -0.0083 -0.0200 -0.0509 0.06014 -0.0022 -0.0724 0.10591 0.02012 0.13784 -0.4154 -0.0411 0.01034 0.02488 0.63785 +-0.4717 -0.1855 0.38573 0.32609 0.62094 -0.1633 0.30781 -0.2355 0.13816 -0.2810 0.05686 0.34954 -0.0601 -0.9767 -0.9112 0.31871 0.41394 1.22121 0.23943 0.45242 -0.1920 -0.8208 0.31329 0.33778 -0.1266 0.00713 -0.3097 -0.2132 -0.4078 -0.2994 0.14806 0.14773 0.17498 -0.1289 0.14525 0.20730 0.16072 -0.0046 0.03372 0.43079 0.42407 0.14852 0.19357 0.16888 -0.0272 -0.0327 0.12011 -0.0043 -0.0344 0.15922 0.06942 0.04454 -0.0448 0.07044 0.02003 -0.0812 -0.0082 -0.0214 -0.0576 0.06136 0.00076 -0.0789 0.11131 0.01998 0.13532 -0.4163 -0.0410 0.01012 0.02510 0.63417 +-0.1668 -0.0849 0.52717 0.51369 0.69530 -0.0484 0.33548 -0.1768 0.19128 -0.3993 -0.0859 0.45499 0.02072 -0.8077 -0.7256 0.36864 0.42910 1.22011 0.22989 0.48058 -0.1612 -0.7979 0.33408 0.32357 -0.1629 0.00108 -0.3808 -0.1807 -0.3743 -0.2877 0.13682 0.12976 0.19580 -0.1326 0.12733 0.17358 0.15331 0.00805 0.03551 0.46127 0.45370 0.12701 0.21634 0.17345 -0.0389 -0.0279 0.12109 0.00986 -0.0263 0.14660 0.07595 0.04528 -0.0536 0.06348 0.00580 -0.0677 -0.0127 -0.0278 -0.0605 0.06063 0.00454 -0.0774 0.09266 0.00844 0.13345 -0.4186 -0.0412 0.01034 0.02517 0.61817 +0.11010 -0.0048 0.63464 0.72923 0.75876 0.19566 0.35450 -0.1053 0.24563 -0.4026 -0.3421 0.40312 0.14361 -0.6627 -0.4816 0.45162 0.43033 1.15915 0.23313 0.41303 -0.1557 -0.7001 0.31126 0.34453 -0.1363 -0.0033 -0.3117 -0.3029 -0.4649 -0.3011 0.16752 0.15226 0.23283 -0.1338 0.18409 0.11412 0.19391 0.01987 0.04162 0.46745 0.45576 0.11519 0.24769 0.19354 -0.0616 -0.0342 0.12505 0.03844 -0.0052 0.14571 0.06386 0.04706 -0.0522 0.06110 0.01532 -0.0678 -0.0200 -0.0278 -0.0658 0.05582 0.00355 -0.0660 0.09816 0.01812 0.13076 -0.4207 -0.0408 0.01063 0.02561 0.60463 +0.22986 0.00597 0.70579 0.88779 0.84442 0.27198 0.38751 -0.1113 0.26128 -0.4206 -0.4063 0.36374 0.14266 -0.6925 -0.4548 0.48435 0.43023 1.20886 0.22570 0.43732 -0.1639 -0.7204 0.22916 0.31734 -0.0265 -0.0178 -0.3393 -0.2527 -0.3398 -0.2536 0.11710 0.16092 0.22496 -0.1219 0.16418 0.14008 0.16304 0.03233 0.07252 0.48525 0.44359 0.13142 0.24658 0.18189 -0.0624 -0.0280 0.11818 0.03529 2.31477 0.13731 0.06901 0.04471 -0.0462 0.06026 0.01876 -0.0643 -0.0181 -0.0234 -0.0628 0.05843 -0.0044 -0.0703 0.10080 0.01614 0.12860 -0.4234 -0.0410 0.01079 0.02629 0.60136 +0.26558 0.00594 0.65299 1.00828 0.91146 0.22837 0.36497 -0.1356 0.21528 -0.4274 -0.4183 0.34839 0.12880 -0.7053 -0.4406 0.45829 0.40432 1.17265 0.22033 0.41937 -0.1748 -0.7190 0.21515 0.33313 0.00618 -0.0446 -0.2684 -0.2078 -0.3181 -0.2144 0.07922 0.16142 0.21262 -0.1243 0.21713 0.15814 0.15036 0.00421 0.05996 0.49026 0.39541 0.14729 0.21865 0.17590 -0.0536 -0.0323 0.11457 0.03829 0.00412 0.13620 0.07780 0.04463 -0.0486 0.06376 0.01844 -0.0648 -0.0140 -0.0214 -0.0564 0.05885 -0.0095 -0.0722 0.10736 0.00787 0.12768 -0.4254 -0.0417 0.01113 0.02594 0.60076 +0.21233 0.00329 0.60084 1.08666 0.98767 0.24078 0.37375 -0.1417 0.19078 -0.4055 -0.4495 0.37129 0.12210 -0.6132 -0.3449 0.41532 0.40587 1.13716 0.22640 0.41710 -0.2027 -0.6584 0.18643 0.33209 0.05575 -0.0432 -0.2218 -0.2090 -0.3045 -0.1789 0.05019 0.16598 0.18340 -0.1242 0.23559 0.17898 0.14272 0.00082 0.06184 0.46495 0.37705 0.16313 0.21498 0.17142 -0.0435 -0.0296 0.11643 0.03003 0.00969 0.12393 0.07432 0.04912 -0.0479 0.06264 0.01733 -0.0736 -0.0174 -0.0192 -0.0571 0.05526 -0.0083 -0.0815 0.10355 0.01005 0.12820 -0.4267 -0.0421 0.01099 0.02571 0.60209 +0.19445 -0.0060 0.48041 1.23119 1.09209 0.21114 0.38213 -0.1738 0.19179 -0.4205 -0.4846 0.39984 0.14786 -0.5487 -0.2917 0.39436 0.39637 1.09415 0.20846 0.38653 -0.2776 -0.6441 0.15211 0.33231 0.08402 -0.0543 -0.2121 -0.1789 -0.2651 -0.1441 0.04061 0.16072 0.17438 -0.1148 0.28595 0.19738 0.09502 -0.0152 0.06214 0.48244 0.35508 0.17260 0.20823 0.17179 -0.0447 -0.0331 0.12050 0.02089 0.00477 0.12091 0.07475 0.04792 -0.0474 0.05952 0.01286 -0.0713 -0.0120 -0.0185 -0.0536 0.05620 -0.0063 -0.0880 0.09599 0.01136 0.12891 -0.4259 -0.0419 0.01095 0.02586 0.59904 +0.12407 -0.0486 0.41993 1.09332 1.06709 0.11327 0.40007 -0.1607 0.19204 -0.4331 -0.3631 0.46592 0.19179 -0.5676 -0.4197 0.44930 0.36760 1.14590 0.20036 0.46027 -0.2340 -0.6736 0.11248 0.32931 0.02903 -0.0273 -0.2651 -0.1293 -0.2927 -0.1696 0.08060 0.14650 0.21355 -0.1159 0.27282 0.16133 0.11645 -0.0236 0.05623 0.47097 0.36206 0.17283 0.20433 0.16864 -0.0438 -0.0244 0.12035 0.02631 -0.0059 0.13586 0.06776 0.04395 -0.0510 0.05840 0.01558 -0.0633 -0.0159 -0.0252 -0.0510 0.05975 0.00144 -0.0835 0.08057 0.00213 0.12926 -0.4242 -0.0407 0.01089 0.02679 0.58125 +-0.0050 -0.0607 0.41668 0.94744 1.01073 0.04175 0.39824 -0.2031 0.18830 -0.3730 -0.3118 0.51194 0.11358 -0.7029 -0.5601 0.48047 0.36861 1.20673 0.23171 0.44624 -0.2587 -0.7272 0.16500 0.35555 -0.0212 -0.0162 -0.3094 -0.1318 -0.3315 -0.2124 0.08380 0.14359 0.20956 -0.1107 0.23273 0.17893 0.11488 -0.0122 0.05448 0.50763 0.36613 0.17753 0.18106 0.18972 -0.0545 -0.0347 0.12194 0.02429 -0.0078 0.13740 0.06897 0.04183 -0.0516 0.06763 0.00938 -0.0672 -0.0079 -0.0203 -0.0580 0.05232 -0.0028 -0.0837 0.08464 0.00797 0.12855 -0.4220 -0.0398 0.01135 0.02711 0.57562 +-0.2509 -0.1158 0.36977 0.63685 0.87614 -0.0467 0.39198 -0.1891 0.19182 -0.3015 -0.1139 0.47470 0.00856 -0.9129 -0.7064 0.42990 0.37131 1.24746 0.24106 0.54879 -0.2034 -0.7796 0.21707 0.32023 -0.0981 -0.0106 -0.3865 -0.1008 -0.3563 -0.3084 0.10662 0.13429 0.19952 -0.1227 0.16511 0.18360 0.10277 -0.0006 0.04738 0.46924 0.42318 0.16187 0.20567 0.14992 -0.0387 -0.0262 0.12622 0.01232 -0.0165 0.14258 0.07067 0.04736 -0.0553 0.06273 0.01842 -0.0786 -0.0014 -0.0216 -0.0591 0.05608 -0.0018 -0.0852 0.09414 0.00992 0.12523 -0.4217 -0.0388 0.01210 0.02664 0.59137 +-0.2677 -0.1501 0.39714 0.52300 0.75808 -0.0933 0.37310 -0.2029 0.18033 -0.3096 -0.0550 0.40745 -0.0212 -0.9648 -0.8274 0.40425 0.38103 1.26152 0.21161 0.52759 -0.2058 -0.7960 0.25044 0.27523 -0.1298 0.00576 -0.4174 -0.1333 -0.3509 -0.2975 0.13295 0.13447 0.19655 -0.1244 0.14855 0.19803 0.11478 0.00484 0.05331 0.45604 0.45170 0.14934 0.21975 0.16084 -0.0365 -0.0367 0.12590 -0.0012 -0.0195 0.14395 0.06547 0.04550 -0.0514 0.06460 0.02128 -0.0783 -0.0070 -0.0194 -0.0576 0.05834 -0.0007 -0.0777 0.10285 0.01710 0.12190 -0.4198 -0.0372 0.01280 0.02761 0.59109 +-0.2616 -0.1293 0.43559 0.46988 0.69959 -0.0921 0.35443 -0.2212 0.17859 -0.3142 -0.0776 0.38024 -0.0234 -1.0147 -0.8316 0.40196 0.40536 1.26795 0.25158 0.49269 -0.2488 -0.8129 0.31615 0.23886 -0.1320 0.00665 -0.4447 -0.1097 -0.3371 -0.2888 0.15169 0.12822 0.20326 -0.1158 0.09122 0.20037 0.10752 0.01225 0.05714 0.45669 0.49223 0.12110 0.24404 0.18456 -0.0432 -0.0306 0.12166 0.00823 -0.0187 0.14085 0.06943 0.04623 -0.0469 0.06442 0.01661 -0.0813 -0.0081 -0.0180 -0.0669 0.05607 0.00057 -0.0766 0.10535 0.01472 0.11995 -0.4173 -0.0349 0.01331 0.02937 0.58765 +-0.1634 -0.1035 0.53780 0.40174 0.63497 -0.0324 0.33330 -0.2126 0.19661 -0.3138 -0.0779 0.32201 -0.0378 -1.0198 -0.9192 0.42775 0.41196 1.23601 0.24694 0.48684 -0.2112 -0.8002 0.35342 0.25863 -0.1403 0.01436 -0.4155 -0.1464 -0.3780 -0.3159 0.14746 0.14374 0.18549 -0.1222 0.08117 0.21044 0.15311 0.02309 0.04210 0.44042 0.50777 0.10866 0.25513 0.19529 -0.0497 -0.0342 0.12751 0.01540 -0.0207 0.14457 0.06261 0.04547 -0.0457 0.06900 0.01933 -0.0864 -0.0076 -0.0202 -0.0623 0.05581 -0.0001 -0.0792 0.11067 0.01576 0.11934 -0.4154 -0.0328 0.01390 0.03045 0.59196 +-0.0313 -0.0591 0.60694 0.43852 0.65236 0.02615 0.28650 -0.1561 0.15394 -0.3357 -0.0620 0.35340 0.01306 -1.1067 -0.8972 0.47907 0.48673 1.05799 0.32979 0.19713 -0.1428 -1.0392 0.04828 0.27173 -0.2040 0.10046 -0.3659 -0.3051 -0.3586 -0.3377 0.24030 0.11912 0.10166 -0.2998 0.07606 0.20259 0.11650 0.04916 -0.0982 0.37024 0.56568 0.26139 0.13876 0.25870 -0.1227 -0.0117 0.09645 0.03299 -0.0360 0.13407 0.07161 0.05492 -0.0491 0.08843 0.02354 -0.0784 -0.0124 -0.0352 -0.0589 0.06182 -0.0029 -0.0805 0.09248 0.01425 0.12032 -0.4130 -0.0309 0.01470 0.03212 0.57267 +-0.0151 -0.0544 0.67914 0.57691 0.70187 -0.0052 0.28820 -0.1374 0.13355 -0.3164 -0.0393 0.34932 0.07666 -1.1438 -0.8918 0.46310 0.56425 0.93378 0.38648 -0.0102 -0.1134 -1.1990 -0.2419 0.30240 -0.1765 0.15684 -0.3326 -0.3479 -0.2447 -0.3087 0.30582 0.09344 -0.0008 -0.4374 0.03895 0.23453 0.03803 0.02088 -0.1903 0.30032 0.59210 0.37781 0.01808 0.32416 -0.1553 0.00629 0.07441 0.05431 -0.0337 0.11627 0.08903 0.05457 -0.0369 0.09377 0.03244 -0.0752 -0.0112 -0.0438 -0.0558 0.07534 0.00175 -0.0966 0.08089 0.01504 0.12266 -0.4111 -0.0294 0.01518 0.03418 0.55337 +-0.0749 -0.0741 0.64888 0.54961 0.67542 -0.0040 0.28952 -0.1509 0.11654 -0.3790 -0.0237 0.32609 0.04288 -1.1482 -0.8103 0.40801 0.59142 0.88157 0.39728 -0.0693 -0.0441 -1.2283 -0.2696 0.34019 -0.1770 0.17488 -0.2871 -0.3990 -0.2551 -0.3202 0.33595 0.09254 -0.0346 -0.4666 0.06576 0.23365 0.05897 0.02631 -0.2405 0.26275 0.58841 0.40857 -0.0138 0.33780 -0.1627 0.00659 0.07142 0.04715 -0.0419 0.11753 0.09537 0.05507 -0.0361 0.10129 0.03807 -0.0721 -0.0082 -0.0413 -0.0561 0.07956 0.00239 -0.0982 0.07950 0.01543 0.12583 -0.4094 -0.0289 0.01512 0.03591 0.54383 +-0.0686 -0.0764 0.63920 0.64712 0.71315 0.05609 0.27174 -0.1556 0.09763 -0.4284 -0.1074 0.31111 0.03434 -1.1011 -0.7172 0.38624 0.59702 0.87790 0.39405 -0.0499 -0.0282 -1.1671 -0.1745 0.37745 -0.1751 0.16014 -0.2552 -0.4536 -0.3034 -0.3033 0.30060 0.11042 -0.0178 -0.4250 0.09052 0.21478 0.10334 0.05446 -0.2297 0.28835 0.55267 0.37152 -0.0005 0.32878 -0.1531 0.00365 0.07437 0.04251 -0.0442 0.11685 0.08966 0.05437 -0.0362 0.09647 0.03607 -0.0746 -0.0118 -0.0397 -0.0562 0.07638 0.00207 -0.1046 0.07751 0.00895 0.12998 -0.4089 -0.0289 0.01469 0.03667 0.53008 +-0.0787 -0.0753 0.65890 0.44849 0.61548 0.03650 0.32174 -0.1787 0.16779 -0.3254 0.01341 0.33177 0.00388 -1.0615 -0.9412 0.49197 0.50639 1.06470 0.33544 0.19709 -0.1191 -1.0140 0.02996 0.31137 -0.1479 0.09236 -0.3386 -0.3290 -0.3588 -0.3394 0.23770 0.11951 0.08928 -0.3009 0.08793 0.23010 0.15007 0.04291 -0.1257 0.35479 0.54400 0.26673 0.11274 0.26623 -0.1079 -0.0113 0.10122 0.03539 -0.0274 0.12721 0.07973 0.05221 -0.0428 0.08845 0.03067 -0.0746 -0.0160 -0.0332 -0.0617 0.07361 0.00135 -0.0891 0.09042 0.01743 0.13361 -0.4084 -0.0289 0.01379 0.03754 0.52229 +-0.1847 -0.1230 0.55672 0.34576 0.55485 0.00594 0.34737 -0.2906 0.15061 -0.2742 0.00150 0.27610 -0.0786 -1.0630 -1.0488 0.51563 0.41274 1.22563 0.25580 0.47647 -0.2381 -0.7982 0.26114 0.31782 -0.0570 -0.0039 -0.3516 -0.1985 -0.3906 -0.3117 0.12602 0.17148 0.15400 -0.1496 0.12592 0.28807 0.20893 0.01534 0.00416 0.44993 0.48259 0.14765 0.21720 0.17748 -0.0549 -0.0279 0.11548 0.01024 -0.0175 0.14599 0.07700 0.04492 -0.0425 0.06725 0.02411 -0.0744 -0.0072 -0.0224 -0.0619 0.05745 -0.0042 -0.0833 0.10906 0.01882 0.13649 -0.4092 -0.0289 0.01281 0.03859 0.50099 +-0.2010 -0.1446 0.55728 0.39623 0.59981 0.00937 0.31644 -0.2667 0.14632 -0.3265 -0.0464 0.27368 -0.0725 -1.0999 -0.9362 0.49525 0.42345 1.20579 0.27811 0.45202 -0.2068 -0.8088 0.27992 0.36547 -0.0590 -0.0072 -0.2864 -0.2357 -0.4387 -0.3086 0.11973 0.19991 0.16736 -0.1419 0.16588 0.27265 0.23118 0.04188 -0.0233 0.44681 0.46613 0.13419 0.22130 0.18460 -0.0525 -0.0335 0.11942 0.00348 -0.0246 0.13967 0.06892 0.04689 -0.0361 0.06752 0.02202 -0.0801 -0.0039 -0.0232 -0.0545 0.05781 -0.0012 -0.0892 0.10555 0.01667 0.13810 -0.4117 -0.0293 0.01176 0.03903 0.48675 +-0.2594 -0.1744 0.43151 0.53302 0.76725 -0.0984 0.36338 -0.2805 0.13532 -0.3025 -0.0526 0.34181 -0.0075 -0.9956 -0.8477 0.39663 0.41089 1.24452 0.20401 0.50210 -0.2545 -0.7649 0.25648 0.31420 -0.0674 0.00535 -0.3733 -0.1275 -0.3584 -0.2875 0.12268 0.14730 0.18347 -0.1249 0.11510 0.23135 0.11402 -0.0008 0.04770 0.46956 0.42244 0.14184 0.20066 0.16081 -0.0402 -0.0316 0.11895 0.00927 -0.0256 0.13704 0.08269 0.04549 -0.0447 0.06475 0.01559 -0.0809 -0.0035 -0.0225 -0.0525 0.06135 -0.0028 -0.0867 0.10537 0.02017 0.13746 -0.4162 -0.0307 0.01100 0.03853 0.49475 +-0.3238 -0.1403 0.32982 0.57496 0.81940 -0.1145 0.34092 -0.2037 0.15746 -0.3575 -0.0138 0.38868 0.04653 -0.8736 -0.8055 0.32766 0.41097 1.20622 0.19763 0.55713 -0.2129 -0.8062 0.19860 0.34013 -0.1027 0.01802 -0.3893 -0.0951 -0.3097 -0.2910 0.12298 0.14173 0.20147 -0.1183 0.16718 0.21608 0.10050 -0.0039 0.05517 0.46654 0.43854 0.13860 0.18554 0.14582 -0.0357 -0.0410 0.13367 0.00410 -0.0175 0.13840 0.06831 0.04352 -0.0552 0.06457 0.02004 -0.0857 -0.0047 -0.0247 -0.0584 0.05767 -0.0025 -0.0705 0.10265 0.01726 0.13568 -0.4209 -0.0323 0.01055 0.03754 0.50250 +-0.3797 -0.1169 0.34095 0.54137 0.76586 -0.1720 0.33838 -0.1693 0.17596 -0.3061 -0.0392 0.44885 0.03106 -0.8872 -0.7905 0.38072 0.38318 1.25024 0.25663 0.54522 -0.2385 -0.7822 0.23817 0.27376 -0.1597 0.01699 -0.4113 -0.0821 -0.3237 -0.3026 0.13630 0.12167 0.22735 -0.1137 0.13582 0.19252 0.09551 0.00464 0.07392 0.46965 0.45287 0.14550 0.19517 0.15554 -0.0400 -0.0343 0.12946 0.01603 -0.0300 0.13931 0.05945 0.04893 -0.0460 0.07118 0.01513 -0.0860 -0.0082 -0.0212 -0.0533 0.06452 0.00249 -0.0826 0.10187 0.01185 0.13536 -0.4238 -0.0339 0.01005 0.03785 0.48905 +-0.3576 -0.1300 0.43168 0.22217 0.54417 -0.1832 0.34957 -0.2073 0.15501 -0.2085 -0.0336 0.29471 -0.0118 -0.9832 -0.9630 0.32353 0.42444 1.25460 0.24738 0.45542 -0.2586 -0.7804 0.32640 0.23610 -0.1017 1.28443 -0.4036 -0.1864 -0.3415 -0.3072 0.15720 0.13549 0.18946 -0.1275 0.04417 0.21244 0.12644 0.03026 0.07413 0.45854 0.50851 0.11694 0.24751 0.19141 -0.0470 -0.0303 0.12085 0.01431 -0.0230 0.14892 0.06200 0.04283 -0.0473 0.06518 0.01956 -0.0884 -0.0057 -0.0150 -0.0649 0.05805 0.00578 -0.0785 0.11019 0.01722 0.13495 -0.4251 -0.0350 0.00970 0.03777 0.47986 +-0.3608 -0.1586 0.47422 0.17791 0.50423 -0.0936 0.35315 -0.2831 0.17840 -0.2425 -0.0520 0.23451 -0.0665 -1.1697 -1.0773 0.37596 0.42243 1.25684 0.26199 0.47348 -0.3084 -0.7685 0.37592 0.22570 -0.1111 0.04204 -0.4194 -0.1281 -0.3150 -0.3287 0.16246 0.13512 0.17256 -0.1234 -0.0351 0.24633 0.11421 0.02828 0.06908 0.45780 0.53769 0.12239 0.27515 0.20185 -0.0543 -0.0354 0.12243 0.02902 -0.0191 0.14586 0.05823 0.04549 -0.0471 0.07079 0.02324 -0.0887 -0.0093 -0.0217 -0.0633 0.05942 0.00223 -0.0781 0.11023 0.02702 0.13368 -0.4263 -0.0361 0.00941 0.03733 0.48044 +-0.2569 -0.1043 0.51888 0.24349 0.52662 -0.0545 0.32115 -0.3050 0.16673 -0.2990 -0.0153 0.23239 -0.0892 -1.1881 -1.1039 0.44089 0.43945 1.22784 0.27129 0.45294 -0.2198 -0.8073 0.37827 0.26047 -0.1027 0.00950 -0.3743 -0.1977 -0.3749 -0.3081 0.13339 0.15436 0.15319 -0.1384 0.04447 0.25270 0.18842 0.03400 0.02839 0.44813 0.52482 0.12814 0.25520 0.20033 -0.0584 -0.0375 0.12271 0.01785 -0.0225 0.14911 0.07175 0.04919 -0.0424 0.06852 0.02357 -0.0776 -0.0107 -0.0191 -0.0590 0.05612 -0.0026 -0.0774 0.10714 0.01792 0.13090 -0.4254 -0.0371 0.00946 0.03763 0.46939 +-0.4849 -0.1303 0.44357 0.07513 0.46019 -0.2241 0.36318 -0.3562 0.13181 -0.1851 0.01893 0.22699 -0.0784 -1.2316 -1.1876 0.40931 0.40827 1.29864 0.25918 0.47395 -0.3306 -0.7997 0.37840 0.21775 -0.1603 0.02284 -0.4570 -0.1268 -0.3350 -0.3418 0.17141 0.13239 0.16502 -0.1143 -0.0082 0.27805 0.11275 0.00605 0.04528 0.42392 0.55083 0.12696 0.25631 0.20585 -0.0517 -0.0399 0.12591 0.01391 -0.0126 0.14642 0.07082 0.04632 -0.0509 0.06761 0.02314 -0.0793 -0.0103 -0.0184 -0.0554 0.05287 0.00338 -0.0674 0.10893 0.02035 0.12844 -0.4253 -0.0376 0.00954 0.03820 0.45364 +-0.6544 -0.1959 0.36685 0.00532 0.48510 -0.4034 0.35666 -0.3804 0.09594 -0.1188 0.13121 0.23913 -0.0736 -1.1927 -1.1648 0.34910 0.36276 1.26704 0.22720 0.45641 -0.3331 -0.7419 0.36328 0.26092 -0.1603 0.03951 -0.4462 -0.1979 -0.3630 -0.3359 0.19062 0.12176 0.18098 -0.1197 0.02121 0.27470 0.10671 0.01668 0.03774 0.44036 0.54810 0.13713 0.23234 0.18461 -0.0480 -0.0402 0.12623 0.00016 -0.0263 0.15333 0.05477 0.04630 -0.0425 0.06347 0.01570 -0.0920 -0.0186 -0.0225 -0.0599 0.06204 0.00449 -0.0700 0.11173 0.01349 0.12517 -0.4244 -0.0376 0.00919 0.03837 0.45745 +-0.5872 -0.1670 0.42481 0.04069 0.48063 -0.3878 0.35527 -0.3115 0.13316 -0.1460 0.13261 0.20408 -0.0749 -1.1179 -1.0832 0.30397 0.40339 1.21134 0.21573 0.44361 -0.2250 -0.7340 0.35337 0.31615 -0.1701 0.03488 -0.3846 -0.2851 -0.4327 -0.3411 0.16792 0.13274 0.17697 -0.1186 0.05618 0.23692 0.17115 0.02450 0.03481 0.42080 0.52795 0.13560 0.22005 0.16314 -0.0535 -0.0352 0.12360 0.00740 -0.0243 0.15076 0.05453 0.04766 -0.0423 0.06969 0.01704 -0.0894 -0.0161 -0.0231 -0.0584 0.05815 0.00205 -0.0717 0.10904 0.01680 0.12189 -0.4253 -0.0368 0.00886 0.03850 0.45929 +-0.4312 -0.1353 0.50485 0.10196 0.48066 -0.2468 0.31533 -0.2589 0.15297 -0.2496 0.05226 0.23804 -0.0716 -1.0647 -1.0922 0.37053 0.43406 1.20272 0.27218 0.39923 -0.1942 -0.7666 0.44072 0.31602 -0.1461 0.03725 -0.3487 -0.3011 -0.4296 -0.3222 0.16255 0.13583 0.17889 -0.1419 0.06245 0.20846 0.21757 0.02319 0.01234 0.42371 0.53471 0.12305 0.23207 0.19543 -0.0421 -0.0369 0.12297 0.00875 -0.0273 0.14814 0.05699 0.04451 -0.0420 0.06807 0.01089 -0.0831 -0.0123 -0.0209 -0.0609 0.05628 0.00506 -0.0733 0.11013 0.01470 0.11901 -0.4250 -0.0357 0.00880 0.03895 0.45703 +-0.2131 -0.1196 0.54936 0.32788 0.56769 -0.0275 0.29977 -0.2349 0.17696 -0.3055 -0.0404 0.32368 -0.0646 -1.1069 -1.0371 0.45270 0.43784 1.22839 0.25378 0.44430 -0.2151 -0.8077 0.36571 0.29369 -0.1511 0.02646 -0.3783 -0.2083 -0.3753 -0.3253 0.13002 0.16018 0.18689 -0.1376 0.08451 0.22309 0.18968 0.02283 0.01689 0.44808 0.50824 0.11679 0.24211 0.19069 -0.0524 -0.0346 0.12825 0.00801 -0.0233 0.14454 0.06375 0.04302 -0.0487 0.06336 0.01795 -0.0814 -0.0069 -0.0207 -0.0597 0.04942 -0.0037 -0.0732 0.11128 0.01453 0.11818 -0.4261 -0.0345 0.00842 0.03917 0.45650 +-0.0586 -0.0851 0.60559 0.52299 0.69915 -0.0132 0.31181 -0.2078 0.13820 -0.2706 -0.0601 0.39759 0.01874 -1.1463 -0.9621 0.50754 0.48639 1.09948 0.30294 0.25117 -0.2188 -1.0334 -0.0226 0.27382 -0.1612 0.09069 -0.3932 -0.2162 -0.2615 -0.3201 0.23006 0.11930 0.06879 -0.2985 0.04275 0.27611 0.06373 0.01489 -0.0821 0.38801 0.55290 0.24071 0.12644 0.26896 -0.1097 -0.0158 0.10096 0.03590 -0.0236 0.12563 0.08271 0.04658 -0.0365 0.07707 0.02909 -0.0810 -0.0066 -0.0326 -0.0580 0.06598 -0.0056 -0.0898 0.09583 0.01675 0.11929 -0.4261 -0.0334 0.00812 0.03821 0.46670 +-0.0059 -0.0684 0.68334 0.64766 0.76340 -0.0465 0.30672 -0.1474 0.12791 -0.2567 -0.0088 0.38312 0.04167 -1.1815 -0.8962 0.50963 0.53075 0.96424 0.37609 0.05775 -0.1847 -1.1807 -0.2960 0.29311 -0.1091 0.15301 -0.3362 -0.2965 -0.2168 -0.3099 0.28091 0.10540 -0.0094 -0.4339 0.04483 0.27884 0.01275 0.04079 -0.1819 0.30347 0.58823 0.37910 0.03099 0.32681 -0.1581 0.00208 0.07536 0.06441 -0.0345 0.11086 0.09702 0.05420 -0.0351 0.09612 0.04045 -0.0754 -0.0082 -0.0410 -0.0585 0.07691 0.00107 -0.0984 0.07401 0.01431 0.12184 -0.4259 -0.0325 0.00754 0.03726 0.47770 +0.01449 -0.0694 0.67075 0.68011 0.79949 -0.0142 0.30115 -0.1571 0.10423 -0.2718 -0.0419 0.37159 0.04407 -1.1949 -0.8980 0.50192 0.56086 0.92103 0.38662 -0.0277 -0.1976 -1.2181 -0.3557 0.30870 -0.0942 0.16492 -0.3109 -0.3077 -0.1843 -0.2950 0.29552 0.11050 -0.0376 -0.4610 0.03665 0.29586 -0.0193 0.03876 -0.1889 0.30235 0.59771 0.40839 -0.0045 0.35132 -0.1706 0.00971 0.06389 0.07202 -0.0382 0.10757 0.09920 0.05727 -0.0291 0.10033 0.04367 -0.0740 -0.0114 -0.0440 -0.0572 0.08118 0.00414 -0.0990 0.06655 0.01255 0.12489 -0.4232 -0.0321 0.00754 0.03673 0.46707 +0.00230 -0.0849 0.64259 0.67449 0.79230 0.05088 0.32394 -0.1915 0.10642 -0.3284 -0.0752 0.33166 0.04793 -1.2002 -0.8585 0.47126 0.56663 0.90662 0.37201 -0.0167 -0.2003 -1.1736 -0.2638 0.32153 -0.0522 0.14250 -0.2981 -0.2900 -0.1964 -0.2893 0.28809 0.12183 -0.0385 -0.4190 0.02393 0.28893 -0.0062 0.04168 -0.1488 0.31635 0.59085 0.37871 0.01275 0.33964 -0.1602 0.00591 0.06735 0.06506 -0.0350 0.11158 0.09496 0.05790 -0.0289 0.09944 0.04305 -0.0800 -0.0103 -0.0410 -0.0555 0.07994 -8.0067 -0.0955 0.07872 0.01313 0.12844 -0.4211 -0.0320 0.00757 0.03658 0.44588 +-0.0725 -0.1100 0.55028 0.53268 0.69548 0.07093 0.34532 -0.2468 0.12800 -0.3542 -0.0665 0.28904 0.01486 -1.1534 -0.8777 0.46115 0.50659 1.06298 0.31124 0.19110 -0.2169 -0.9848 -0.0284 0.27840 -0.0386 0.08008 -0.3389 -0.1982 -0.2663 -0.2815 0.21834 0.13987 0.04164 -0.2912 0.05932 0.29406 0.06959 0.04237 -0.0771 0.35913 0.53572 0.27386 0.11315 0.25657 -0.1139 -0.0125 0.09095 0.03239 -0.0325 0.13009 0.08956 0.05478 -0.0311 0.08640 0.03788 -0.0798 -0.0080 -0.0308 -0.0538 0.07215 -0.0026 -0.0926 0.09051 0.01358 0.13135 -0.4177 -0.0323 0.00782 0.03611 0.44343 +-0.1293 -0.1177 0.50118 0.42127 0.64641 0.03634 0.37717 -0.3244 0.13691 -0.3210 -0.0809 0.27558 -0.0464 -1.1122 -1.0222 0.51862 0.41273 1.25263 0.22855 0.49100 -0.3175 -0.7831 0.23842 0.25481 -0.0250 -0.0061 -0.4123 -0.0751 -0.3370 -0.2734 0.10247 0.16837 0.15664 -0.1219 0.09534 0.31851 0.13471 0.02603 0.03764 0.45366 0.48475 0.13978 0.23955 0.17862 -0.0531 -0.0276 0.11161 0.00808 -0.0242 0.14353 0.08131 0.04846 -0.0437 0.06828 0.02902 -0.0774 -0.0036 -0.0227 -0.0561 0.05426 -0.0077 -0.0867 0.10605 0.01563 0.13107 -0.4163 -0.0322 0.00813 0.03617 0.44889 +-0.3067 -0.1459 0.48777 0.25833 0.52201 -0.1015 0.35521 -0.3268 0.13926 -0.3151 -0.0499 0.20701 -0.0836 -1.1794 -1.0406 0.42383 0.43813 1.25455 0.23863 0.47051 -0.2949 -0.7901 0.33716 0.26250 -0.0836 0.02030 -0.3870 -0.1349 -0.3440 -0.3314 0.13690 0.15356 0.15247 -0.1203 0.05819 0.27092 0.16271 0.02392 0.02762 0.44061 0.52211 0.13451 0.24589 0.20002 -0.0512 -0.0362 0.12278 -0.0040 -0.0263 0.15421 0.07339 0.04742 -0.0404 0.06304 0.02246 -0.0761 -0.0154 -0.0199 -0.0533 0.05920 -0.0039 -0.0748 0.09989 0.01279 0.12964 -0.4150 -0.0321 0.00857 0.03643 0.45842 +-0.3708 -0.1444 0.46308 0.25051 0.54378 -0.1285 0.33149 -0.2846 0.14788 -0.2764 -0.0478 0.23251 -0.1089 -1.1440 -0.9955 0.41907 0.43046 1.23517 0.25490 0.40356 -0.2829 -0.7802 0.37769 0.27436 -0.1231 0.02972 -0.3645 -0.2174 -0.3554 -0.3082 0.14307 0.15705 0.15980 -0.1271 0.03813 0.23814 0.16725 0.04168 0.02759 0.44574 0.53107 0.13381 0.24424 0.19962 -0.0533 -0.0354 0.12479 -0.0053 -0.0200 0.14192 0.06510 0.04763 -0.0453 0.06929 0.01730 -0.0848 -0.0123 -0.0217 -0.0588 0.05651 0.00329 -0.0738 0.10483 0.01794 0.12708 -0.4140 -0.0320 0.00913 0.03691 0.47202 +-0.3575 -0.1289 0.49407 0.24898 0.54191 -0.1132 0.32470 -0.2521 0.16061 -0.2936 -0.0728 0.25691 -0.0658 -1.0833 -0.9615 0.39423 0.45445 1.21151 0.25428 0.38487 -0.2696 -0.7446 0.41991 0.28023 -0.1449 0.04078 -0.3842 -0.2198 -0.3731 -0.3256 0.14185 0.14350 0.17359 -0.1346 0.02926 0.20267 0.16504 0.03475 0.03505 0.44230 0.54075 0.12693 0.25949 0.19962 -0.0605 -0.0397 0.13063 0.01229 -0.0156 0.14088 0.06599 0.04789 -0.0521 0.06961 0.01840 -0.0845 -0.0129 -0.0242 -0.0598 0.05673 0.00010 -0.0722 0.10685 0.01801 0.12337 -0.4132 -0.0319 0.00981 0.03760 0.48970 +-0.1929 -0.1023 0.54644 0.38110 0.60654 -0.0228 0.31027 -0.1930 0.20612 -0.3294 -0.1057 0.30568 -0.0133 -1.0061 -0.9217 0.42205 0.44421 1.21939 0.25582 0.45357 -0.2332 -0.7640 0.37612 0.26485 -0.1807 0.03521 -0.4287 -0.1961 -0.3748 -0.3393 0.15100 0.14239 0.21101 -0.1261 0.05799 0.18915 0.18686 0.04195 0.02449 0.43962 0.52633 0.10834 0.26247 0.19373 -0.0583 -0.0395 0.12688 0.00924 -0.0217 0.14616 0.06217 0.04449 -0.0500 0.06320 0.01771 -0.0803 -0.0164 -0.0256 -0.0642 0.05695 -0.0003 -0.0726 0.10854 0.01464 0.11853 -0.4125 -0.0318 0.01061 0.03851 0.51144 +-0.1316 -0.0759 0.53856 0.40167 0.61879 1.89989 0.31986 -0.1657 0.22459 -0.3390 -0.1005 0.30320 0.00270 -0.9693 -0.8702 0.42618 0.42094 1.21737 0.25094 0.46531 -0.2082 -0.7630 0.35526 0.22226 -0.2043 0.04108 -0.4642 -0.1826 -0.3820 -0.3512 0.15712 0.12537 0.23009 -0.1214 0.08140 0.17615 0.18996 0.04374 0.01480 0.43095 0.54380 0.10915 0.26850 0.18877 -0.0599 -0.0342 0.12984 0.00714 -0.0238 0.14891 0.06140 0.04458 -0.0454 0.06566 0.01933 -0.0774 -0.0172 -0.0257 -0.0692 0.05713 -0.0013 -0.0739 0.10608 0.01389 0.11255 -0.4121 -0.0318 0.01154 0.03963 0.53726 diff --git a/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100_enhanced.mp4 b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100_enhanced.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..852841f903d3c5cbe09e3b62aa320b4b3f5ea250 Binary files /dev/null and b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/image##nana_speech_cut_4sec 1-0-100_enhanced.mp4 differ diff --git a/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/input/image.png b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/input/image.png new file mode 100644 index 0000000000000000000000000000000000000000..c48e8ff4c66f1dcc5466442046e5509217af1e2d Binary files /dev/null and b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/input/image.png differ diff --git a/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/input/nana_speech_cut_4sec 1-0-100.wav b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/input/nana_speech_cut_4sec 1-0-100.wav new file mode 100644 index 0000000000000000000000000000000000000000..b84d0ec2cae941fa9b4d828df76340818bda4d13 Binary files /dev/null and b/results/c6e207ec-5e2b-4164-8cdd-8d13be7a51f3/input/nana_speech_cut_4sec 1-0-100.wav differ diff --git a/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/first_frame_dir/image.mat b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/first_frame_dir/image.mat new file mode 100644 index 0000000000000000000000000000000000000000..de079d967a1208954bd4d185cf3720d05eec2471 Binary files /dev/null and b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/first_frame_dir/image.mat differ diff --git a/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/first_frame_dir/image.png b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/first_frame_dir/image.png new file mode 100644 index 0000000000000000000000000000000000000000..befdaf7452fd92d72d6d899bd55a608f83e06cc4 Binary files /dev/null and b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/first_frame_dir/image.png differ diff --git a/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/first_frame_dir/image_landmarks.txt b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/first_frame_dir/image_landmarks.txt new file mode 100644 index 0000000000000000000000000000000000000000..8e4088d870e65e9b195c5f5efbd6e33bf90a57d0 --- /dev/null +++ b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/first_frame_dir/image_landmarks.txt @@ -0,0 +1,136 @@ +5.869531250000000000e+01 +1.211171875000000000e+02 +5.869531250000000000e+01 +1.500703125000000000e+02 +5.869531250000000000e+01 +1.688046875000000000e+02 +6.249218750000000000e+01 +1.926484375000000000e+02 +7.261718750000000000e+01 +2.113828125000000000e+02 +8.527343750000000000e+01 +2.267109375000000000e+02 +1.004609375000000000e+02 +2.386328125000000000e+02 +1.194453125000000000e+02 +2.437421875000000000e+02 +1.384296875000000000e+02 +2.437421875000000000e+02 +1.561484375000000000e+02 +2.437421875000000000e+02 +1.713359375000000000e+02 +2.386328125000000000e+02 +1.839921875000000000e+02 +2.267109375000000000e+02 +1.953828125000000000e+02 +2.113828125000000000e+02 +2.029765625000000000e+02 +1.943515625000000000e+02 +2.093046875000000000e+02 +1.739140625000000000e+02 +2.105703125000000000e+02 +1.534765625000000000e+02 +2.105703125000000000e+02 +1.296328125000000000e+02 +7.514843750000000000e+01 +1.074921875000000000e+02 +8.717187500000000000e+01 +1.015312500000000000e+02 +1.004609375000000000e+02 +1.015312500000000000e+02 +1.143828125000000000e+02 +1.032343750000000000e+02 +1.264062500000000000e+02 +1.074921875000000000e+02 +1.599453125000000000e+02 +1.083437500000000000e+02 +1.719687500000000000e+02 +1.049375000000000000e+02 +1.827265625000000000e+02 +1.040859375000000000e+02 +1.934843750000000000e+02 +1.049375000000000000e+02 +2.042421875000000000e+02 +1.126015625000000000e+02 +1.434921875000000000e+02 +1.279296875000000000e+02 +1.434921875000000000e+02 +1.415546875000000000e+02 +1.434921875000000000e+02 +1.568828125000000000e+02 +1.422265625000000000e+02 +1.705078125000000000e+02 +1.245078125000000000e+02 +1.790234375000000000e+02 +1.321015625000000000e+02 +1.807265625000000000e+02 +1.409609375000000000e+02 +1.824296875000000000e+02 +1.498203125000000000e+02 +1.824296875000000000e+02 +1.574140625000000000e+02 +1.807265625000000000e+02 +8.780468750000000000e+01 +1.245234375000000000e+02 +9.539843750000000000e+01 +1.194140625000000000e+02 +1.131171875000000000e+02 +1.228203125000000000e+02 +1.194453125000000000e+02 +1.296328125000000000e+02 +1.118515625000000000e+02 +1.330390625000000000e+02 +9.413281250000000000e+01 +1.296328125000000000e+02 +1.624765625000000000e+02 +1.330390625000000000e+02 +1.700703125000000000e+02 +1.245234375000000000e+02 +1.865234375000000000e+02 +1.245234375000000000e+02 +1.928515625000000000e+02 +1.296328125000000000e+02 +1.865234375000000000e+02 +1.347421875000000000e+02 +1.700703125000000000e+02 +1.347421875000000000e+02 +1.080546875000000000e+02 +2.011640625000000000e+02 +1.219765625000000000e+02 +1.994609375000000000e+02 +1.358984375000000000e+02 +1.977578125000000000e+02 +1.409609375000000000e+02 +1.977578125000000000e+02 +1.460234375000000000e+02 +1.977578125000000000e+02 +1.574140625000000000e+02 +2.011640625000000000e+02 +1.688046875000000000e+02 +2.045703125000000000e+02 +1.599453125000000000e+02 +2.113828125000000000e+02 +1.510859375000000000e+02 +2.181953125000000000e+02 +1.396953125000000000e+02 +2.181953125000000000e+02 +1.283046875000000000e+02 +2.147890625000000000e+02 +1.181796875000000000e+02 +2.096796875000000000e+02 +1.105859375000000000e+02 +2.011640625000000000e+02 +1.257734375000000000e+02 +2.045703125000000000e+02 +1.409609375000000000e+02 +2.062734375000000000e+02 +1.536171875000000000e+02 +2.062734375000000000e+02 +1.662734375000000000e+02 +2.045703125000000000e+02 +1.536171875000000000e+02 +2.045703125000000000e+02 +1.409609375000000000e+02 +2.062734375000000000e+02 +1.245078125000000000e+02 +2.045703125000000000e+02 diff --git a/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100.mat b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100.mat new file mode 100644 index 0000000000000000000000000000000000000000..9cb9ec25fc14a27814c46615e85c35fb717d3b1a Binary files /dev/null and b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100.mat differ diff --git a/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100.mp4 b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6331cc091fde4ed0e847a2a7f09c961f96fe6410 Binary files /dev/null and b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100.mp4 differ diff --git a/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100.txt b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100.txt new file mode 100644 index 0000000000000000000000000000000000000000..a206bc010c665090d4dfd45820f1071a6c8556d1 --- /dev/null +++ b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100.txt @@ -0,0 +1,92 @@ +-0.4278 0.11022 0.14870 -0.2325 0.01787 -0.4796 -0.0643 -0.2972 0.25088 0.13831 0.30814 0.36796 -0.4101 -0.9264 -0.2901 0.48847 0.39547 0.29986 0.48494 0.48839 -0.2810 -0.2537 0.15106 0.33192 -0.0136 0.32432 -0.2574 -0.3313 -0.2323 -0.5627 -0.2871 0.00675 -0.1613 -0.0024 0.15430 0.04477 0.12900 -0.0454 -0.1306 0.26167 0.46351 -0.0125 0.01944 0.18737 0.10583 0.00578 0.06965 -0.0416 -0.0402 0.17248 0.04880 0.03180 -0.0671 0.00404 0.00068 -0.0339 0.03742 -0.0102 -0.0330 0.01098 -0.0253 -0.0932 0.08337 -0.0208 0.09515 0.08558 -0.0627 -0.0217 0.01687 0.44922 +-0.6047 0.08414 0.05667 -0.4514 -0.1011 -0.5610 -0.1128 -0.3657 0.19877 0.18448 0.32096 0.31758 -0.4484 -0.9779 -0.1762 0.30501 0.43134 0.29158 0.48953 0.39711 -0.2334 -0.2356 0.25520 0.29294 -0.0044 0.34578 -0.2350 -0.4119 -0.2798 -0.5582 -0.2647 -0.0197 -0.1652 0.00229 0.11013 0.00794 0.15928 -0.0454 -0.1168 0.24610 0.44520 -0.0152 -0.0053 0.21020 0.12585 0.01247 0.06212 -0.0647 -0.0547 0.18453 0.04046 0.03551 -0.0641 0.01368 -0.0024 -0.0338 0.02966 -0.0058 -0.0372 0.00953 -0.0240 -0.0891 0.07743 -0.0278 0.09422 0.08665 -0.0608 -0.0228 0.01754 0.45172 +-0.5478 0.09706 0.10812 -0.3731 -0.0679 -0.5359 -0.0821 -0.3461 0.21009 0.07680 0.31611 0.31124 -0.4905 -1.0944 -0.2447 0.38574 0.42531 0.29591 0.49722 0.38059 -0.2290 -0.2376 0.31737 0.28803 -0.1066 0.35436 -0.2078 -0.3836 -0.2861 -0.5624 -0.2746 0.00229 -0.1760 8.16397 0.09381 -0.0008 0.16676 -0.0294 -0.1687 0.26102 0.48414 -0.0164 0.01131 0.22299 0.11796 0.00411 0.06174 -0.0621 -0.0559 0.17926 0.03615 0.03289 -0.0689 0.01182 -0.0011 -0.0257 0.02959 -0.0027 -0.0421 0.00774 -0.0214 -0.0895 0.08284 -0.0317 0.09345 0.08735 -0.0594 -0.0239 0.01800 0.45430 +-0.2675 0.11166 0.22797 -0.1856 0.02059 -0.3494 -0.0880 -0.3074 0.23520 0.07450 0.15885 0.41287 -0.4531 -1.1041 -0.2621 0.56769 0.42512 0.31168 0.48984 0.42658 -0.3393 -0.2632 0.25678 0.27073 -0.0900 0.35873 -0.2487 -0.2793 -0.2187 -0.5406 -0.2944 0.00695 -0.1854 0.02065 0.08260 0.02136 0.12730 -0.0443 -0.1332 0.28243 0.47592 -0.0318 0.02569 0.23533 0.11238 0.00536 0.06400 -0.0457 -0.0387 0.18362 0.04547 0.03201 -0.0718 0.00610 0.00523 -0.0352 0.03362 0.00150 -0.0425 -0.0039 -0.0310 -0.0881 0.09511 -0.0246 0.09284 0.08765 -0.0583 -0.0250 0.01825 0.45695 +-0.1524 0.18810 0.33753 -0.0271 0.13939 -0.3338 -0.0752 -0.3214 0.23749 0.17323 0.17151 0.46752 -0.4210 -1.0832 -0.2713 0.68544 0.37131 0.34510 0.47823 0.48616 -0.3563 -0.2457 0.14579 0.25126 -0.0386 0.33201 -0.2915 -0.2563 -0.1989 -0.5445 -0.3092 0.00764 -0.1873 0.00805 0.10022 0.05740 0.10738 -0.0381 -0.1097 0.29417 0.45072 -0.0178 0.04653 0.23254 0.11465 0.01452 0.06548 -0.0297 -0.0369 0.17299 0.06006 0.03024 -0.0660 -0.0015 0.00642 -0.0344 0.03692 -0.0014 -0.0369 0.00648 -0.0326 -0.1018 0.08740 -0.0223 0.09240 0.08757 -0.0576 -0.0259 0.01830 0.45967 +-0.1408 0.15545 0.30327 0.04509 0.21360 -0.2921 -0.0529 -0.3419 0.20999 0.14100 0.14939 0.43828 -0.4322 -1.0968 -0.2276 0.71601 0.36478 0.32621 0.48872 0.46921 -0.4002 -0.2232 0.08338 0.28667 0.04843 0.29937 -0.1731 -0.2345 -0.1800 -0.4951 -0.3413 0.04855 -0.1989 0.01741 0.14823 0.10881 0.10743 -0.0248 -0.1086 0.29052 0.42423 -0.0025 0.02763 0.22532 0.11852 0.00562 0.05018 -0.0315 -0.0223 0.16076 0.06299 0.03202 -0.0611 0.01039 0.00874 -0.0282 0.03472 -0.0024 -0.0351 0.00815 -0.0304 -0.1007 0.08838 -0.0255 0.09211 0.08711 -0.0572 -0.0268 0.01815 0.46246 +-0.1851 0.15016 0.23173 -0.0017 0.14779 -0.2681 -0.0758 -0.3332 0.20381 0.06855 0.11942 0.40373 -0.4558 -1.0899 -0.2136 0.64226 0.40913 0.29731 0.50293 0.46653 -0.3374 -0.2392 0.10374 0.30071 0.05212 0.30046 -0.1659 -0.2428 -0.2335 -0.4752 -0.3349 0.07078 -0.2007 0.01031 0.14133 0.08745 0.13601 -0.0223 -0.1197 0.27354 0.40321 -0.0228 0.02289 0.23224 0.11198 0.01144 0.04222 -0.0441 -0.0332 0.16660 0.06003 0.03288 -0.0641 0.00879 0.00732 -0.0264 0.03606 0.00148 -0.0348 0.00575 -0.0304 -0.1026 0.09057 -0.0289 0.09199 0.08626 -0.0572 -0.0276 0.01779 0.46533 +-0.2925 0.14622 0.15957 0.02663 0.18046 -0.3412 -0.0969 -0.2966 0.21508 0.03252 0.08580 0.47675 -0.4338 -0.9522 -0.0751 0.50279 0.40658 0.29461 0.49896 0.48020 -0.2676 -0.2107 0.17760 0.31977 0.00666 0.31242 -0.1709 -0.2655 -0.2154 -0.5071 -0.3236 0.00494 -0.1795 0.02192 0.14150 0.02319 0.13342 -0.0439 -0.1022 0.25994 0.40601 -0.0206 -0.0097 0.22563 0.11108 0.00582 0.05014 -0.0439 -0.0359 0.16612 0.06320 0.03061 -0.0674 0.00238 0.00219 -0.0319 0.03854 0.00399 -0.0341 0.00367 -0.0299 -0.0954 0.08650 -0.0180 0.09170 0.08390 -0.0578 -0.0277 0.01628 0.46776 +-0.1271 0.19498 0.22263 0.08480 0.23885 -0.2846 -0.0719 -0.1606 0.27068 -0.0059 0.13887 0.50770 -0.3255 -0.7612 0.06380 0.54435 0.39413 0.25914 0.46002 0.45377 -0.1306 -0.1915 0.17051 0.34794 0.01742 0.30071 -0.1042 -0.3540 -0.2782 -0.5036 -0.3076 -0.0258 -0.1282 0.01798 0.22497 -0.1021 0.17035 -0.0332 -0.1285 0.27023 0.37360 0.00200 -0.0093 0.21561 0.10799 0.00110 0.05492 -0.0480 -0.0456 0.16576 0.03992 0.02569 -0.0744 0.01004 -0.0116 -0.0278 0.03087 -0.0055 -0.0440 -0.0032 -0.0245 -0.0858 0.09107 -0.0225 0.09063 0.08197 -0.0588 -0.0284 0.01574 0.46915 +-0.4264 0.16537 0.09336 -0.1224 0.16498 -0.4622 -0.0981 -0.2447 0.24155 0.06092 0.18259 0.45880 -0.3874 -0.9176 0.03100 0.43226 0.40750 0.28312 0.48072 0.43622 -0.1931 -0.2583 0.20142 0.30962 -0.0455 0.31758 -0.1711 -0.3393 -0.2400 -0.5405 -0.3128 -0.0151 -0.1512 0.01912 0.18398 -0.0623 0.15801 -0.0462 -0.1274 0.28796 0.39080 0.00143 -0.0300 0.21285 0.11937 0.00918 0.05628 -0.0605 -0.0405 0.16888 0.04447 0.03317 -0.0782 0.01220 0.00210 -0.0228 0.03151 -0.0047 -0.0350 0.00965 -0.0248 -0.0911 0.08712 -0.0266 0.09133 0.07999 -0.0602 -0.0293 0.01538 0.47289 +-0.4551 0.15696 0.13984 -0.3316 -0.0378 -0.4574 -0.1140 -0.2889 0.21186 0.04093 0.26414 0.34365 -0.4518 -1.0417 -0.1114 0.38321 0.41892 0.26930 0.51217 0.41605 -0.1359 -0.2205 0.32142 0.33678 -0.0883 0.33333 -0.1637 -0.4346 -0.3047 -0.5639 -0.2864 -0.0016 -0.1627 0.00193 0.12415 -0.0524 0.20288 -0.0255 -0.1604 0.28161 0.44703 -0.0183 0.00820 0.23442 0.12266 0.01473 0.06515 -0.0740 -0.0530 0.17889 0.03844 0.02911 -0.0729 0.01219 -0.0039 -0.0301 0.02915 -0.0018 -0.0315 0.00826 -0.0240 -0.0897 0.07670 -0.0289 0.09248 0.07844 -0.0615 -0.0302 0.01433 0.48558 +-0.2059 0.18628 0.27607 -0.1854 0.03176 -0.2923 -0.1429 -0.2518 0.24091 0.02850 0.16540 0.38111 -0.4288 -1.0206 -0.1443 0.53315 0.40819 0.30278 0.52216 0.42676 -0.2194 -0.2478 0.28951 0.27807 -0.1096 0.34494 -0.2380 -0.3521 -0.2571 -0.5534 -0.3047 0.01155 -0.1502 0.01345 0.11922 -0.0225 0.20452 -0.0192 -0.1437 0.25167 0.47113 -0.0348 0.03857 0.23954 0.11511 0.00398 0.06001 -0.0436 -0.0492 0.17407 0.03657 0.02724 -0.0737 0.00347 -0.0004 -0.0273 0.03246 -0.0030 -0.0344 -0.0010 -0.0302 -0.0888 0.08353 -0.0262 0.09452 0.07599 -0.0626 -0.0310 0.01326 0.49476 +-0.1739 0.18221 0.36656 -0.2517 -0.0406 -0.3412 -0.1286 -0.2609 0.26470 0.13812 0.26674 0.45282 -0.4335 -1.0174 -0.2697 0.63494 0.37678 0.32056 0.50090 0.47926 -0.2793 -0.2794 0.21741 0.23237 -0.1244 0.33794 -0.3423 -0.3031 -0.2615 -0.5739 -0.2812 0.01686 -0.1453 0.00800 0.13714 -0.0097 0.15979 -0.0151 -0.1441 0.25745 0.49888 -0.0305 0.06904 0.22840 0.09342 0.00544 0.05968 -0.0293 -0.0378 0.15911 0.05699 0.03009 -0.0655 0.00581 0.00511 -0.0333 0.03641 -0.0023 -0.0426 0.00189 -0.0244 -0.0867 0.09260 -0.0217 0.09654 0.07523 -0.0639 -0.0316 0.01243 0.50075 +-0.3693 0.12700 0.19849 -0.2380 -0.0024 -0.4207 -0.1158 -0.3207 0.22051 0.18818 0.25876 0.43669 -0.4496 -0.9918 -0.3207 0.54615 0.38364 0.31791 0.50976 0.47373 -0.3235 -0.2607 0.21014 0.25997 -0.0543 0.31930 -0.2827 -0.2876 -0.1998 -0.5540 -0.2874 0.02212 -0.1692 -0.0005 0.11720 0.03522 0.12654 -0.0368 -0.1227 0.27521 0.49937 -0.0244 0.04363 0.22815 0.11238 0.00976 0.06138 -0.0432 -0.0488 0.15936 0.04571 0.03522 -0.0667 0.01205 0.00786 -0.0347 0.03610 -0.0013 -0.0376 0.00528 -0.0174 -0.0972 0.09006 -0.0234 0.09820 0.07410 -0.0649 -0.0321 0.01215 0.49564 +-0.5624 0.10881 0.12938 -0.4219 -0.1064 -0.5310 -0.1160 -0.3761 0.24174 0.18273 0.27401 0.37444 -0.4561 -1.0359 -0.2382 0.43382 0.39954 0.31458 0.50087 0.44361 -0.2326 -0.2527 0.24356 0.27985 -0.0275 0.32014 -0.2464 -0.3926 -0.2925 -0.5647 -0.2680 0.00705 -0.1616 0.00567 0.07899 0.01652 0.17168 -0.0229 -0.1487 0.25758 0.46129 -0.0310 0.00312 0.21425 0.11209 0.00993 0.05917 -0.0498 -0.0621 0.19902 0.03973 0.02972 -0.0640 0.01287 -0.0012 -0.0318 0.03059 -0.0014 -0.0428 0.00915 -0.0240 -0.0825 0.09015 -0.0200 0.09862 0.07418 -0.0651 -0.0326 0.01158 0.49263 +-0.6071 0.09940 0.10883 -0.5073 -0.1400 -0.5057 -0.1220 -0.3700 0.19464 0.19938 0.29688 0.30018 -0.4778 -1.0880 -0.2789 0.36340 0.41318 0.30132 0.49899 0.40491 -0.2430 -0.2701 0.28837 0.22825 -0.0739 0.35796 -0.2735 -0.3812 -0.2656 -0.5770 -0.2432 -0.0186 -0.1688 0.00809 0.05270 0.02433 0.15369 -0.0398 -0.1218 0.24572 0.47652 -0.0293 0.02468 0.22057 0.11973 0.00829 0.06097 -0.0628 -0.0614 0.18730 0.03384 0.03631 -0.0677 0.00877 -0.0026 -0.0352 0.03176 0.00069 -0.0422 0.00760 -0.0221 -0.0897 0.09105 -0.0235 0.09782 0.07370 -0.0651 -0.0329 0.01082 0.49648 +-0.5365 0.12490 0.14866 -0.4650 -0.0998 -0.4865 -0.0931 -0.3349 0.22426 0.12252 0.23958 0.29015 -0.4586 -1.1193 -0.2339 0.41595 0.41871 0.28909 0.51463 0.38167 -0.2670 -0.2406 0.35884 0.25657 -0.1077 0.36047 -0.2424 -0.3805 -0.2350 -0.5624 -0.2610 -0.0098 -0.1738 0.00645 0.04479 -0.0088 0.15696 -0.0313 -0.1340 0.27044 0.50637 -0.0250 0.03705 0.24679 0.11049 0.00670 0.06172 -0.0517 -0.0511 0.18339 0.03640 0.03573 -0.0707 0.00996 -0.0038 -0.0307 0.02585 -0.0024 -0.0370 0.00187 -0.0203 -0.0873 0.08631 -0.0286 0.09496 0.07379 -0.0651 -0.0330 0.01109 0.48815 +-0.4424 0.14245 0.21138 -0.3464 -0.0599 -0.4173 -0.1167 -0.2990 0.24154 0.06038 0.20042 0.34015 -0.4769 -1.0879 -0.1899 0.46388 0.42708 0.27275 0.50190 0.37831 -0.2475 -0.2174 0.35390 0.28440 -0.1242 0.35415 -0.2381 -0.3730 -0.2593 -0.5652 -0.2607 0.00273 -0.1663 0.00288 0.05000 -0.0357 0.17930 -0.0209 -0.1403 0.25026 0.51882 -0.0387 0.02466 0.24651 0.10577 0.00691 0.06206 -0.0511 -0.0540 0.17748 0.03421 0.02895 -0.0683 0.00450 -0.0027 -0.0370 0.02990 0.00077 -0.0406 0.00527 -0.0209 -0.0839 0.09313 -0.0179 0.09214 0.07240 -0.0650 -0.0330 0.01192 0.47588 +-0.2512 0.15307 0.23998 -0.1717 0.03049 -0.2933 -0.1204 -0.2839 0.23053 0.05831 0.16079 0.44192 -0.4216 -1.0289 -0.1581 0.51696 0.41011 0.30935 0.48770 0.42335 -0.2758 -0.2368 0.29953 0.27955 -0.1198 0.35379 -0.2612 -0.3313 -0.2308 -0.5695 -0.2714 0.00473 -0.1518 0.00318 0.10216 -0.0111 0.15717 -0.0383 -0.1346 0.27371 0.48005 -0.0339 0.04564 0.23789 0.10591 0.00318 0.06078 -0.0442 -0.0415 0.15981 0.03362 0.03145 -0.0677 0.00364 0.00264 -0.0345 0.03101 -0.0018 -0.0379 -0.0023 -0.0231 -0.0925 0.08654 -0.0198 0.08853 0.07111 -0.0654 -0.0332 0.01226 0.47524 +-0.1813 0.20084 0.33178 -0.1319 0.07677 -0.2856 -0.0974 -0.2760 0.26124 0.17995 0.23163 0.47058 -0.4395 -1.0274 -0.2315 0.65521 0.36408 0.30679 0.48379 0.48419 -0.2954 -0.2415 0.19962 0.26145 -0.1246 0.33830 -0.2821 -0.3070 -0.2163 -0.5752 -0.2880 0.01502 -0.1558 0.01405 0.13489 0.03269 0.14293 -0.0425 -0.1253 0.28631 0.46395 -0.0339 0.04042 0.23914 0.10832 0.00562 0.06225 -0.0328 -0.0315 0.16434 0.05106 0.02817 -0.0656 -0.0031 0.00651 -0.0333 0.03201 -0.0035 -0.0388 0.00019 -0.0275 -0.0942 0.09330 -0.0166 0.08597 0.06940 -0.0658 -0.0332 0.01231 0.46617 +-0.2174 0.15765 0.30799 -0.1567 0.02414 -0.3448 -0.1083 -0.3256 0.22397 0.18176 0.25010 0.44758 -0.4429 -1.0640 -0.3157 0.64885 0.35809 0.35642 0.48127 0.49478 -0.3536 -0.2455 0.14041 0.24553 -0.0874 0.31985 -0.2990 -0.2777 -0.1942 -0.5566 -0.2963 0.02995 -0.1601 0.01650 0.13946 0.06790 0.11102 -0.0475 -0.1136 0.28841 0.47066 -0.0305 0.04265 0.22132 0.11458 0.00411 0.06062 -0.0409 -0.0258 0.17078 0.05630 0.02653 -0.0658 0.00031 0.00792 -0.0300 0.03430 -0.0006 -0.0407 0.00504 -0.0291 -0.0938 0.09507 -0.0221 0.08404 0.06894 -0.0660 -0.0331 0.01275 0.45899 +-0.4239 0.10750 0.15411 -0.2467 -0.0255 -0.4226 -0.1009 -0.3687 0.20937 0.08254 0.25043 0.38236 -0.4739 -1.1307 -0.2774 0.52513 0.42220 0.29422 0.48618 0.45788 -0.3116 -0.2451 0.21603 0.28592 -0.0622 0.33978 -0.2247 -0.2726 -0.2286 -0.5316 -0.2949 0.01773 -0.1946 0.01802 0.12702 0.06645 0.15332 -0.0533 -0.1375 0.25927 0.45526 -0.0201 0.01942 0.22179 0.11722 0.00667 0.05948 -0.0543 -0.0412 0.17858 0.05167 0.03143 -0.0696 0.00372 0.00337 -0.0271 0.03224 0.00058 -0.0362 0.00368 -0.0281 -0.0945 0.09019 -0.0248 0.08391 0.06786 -0.0664 -0.0333 0.01310 0.46088 +-0.4056 0.12798 0.17823 -0.2546 -0.0068 -0.4530 -0.1387 -0.3222 0.23229 0.08520 0.22629 0.39306 -0.4721 -1.0170 -0.1830 0.45541 0.42442 0.29133 0.49979 0.37890 -0.2323 -0.2475 0.31429 0.27910 -0.1250 0.33462 -0.2529 -0.3400 -0.2562 -0.5455 -0.2862 0.00649 -0.1891 0.01232 0.10530 -0.0254 0.14587 -0.0371 -0.1430 0.23801 0.47298 -0.0183 0.02117 0.22661 0.11955 0.00735 0.06228 -0.0504 -0.0462 0.17883 0.04751 0.03203 -0.0639 0.00960 0.00098 -0.0320 0.03566 0.00389 -0.0331 0.00127 -0.0217 -0.0932 0.08503 -0.0241 0.08522 0.06745 -0.0670 -0.0333 0.01215 0.47094 +-0.2553 0.18608 0.25455 -0.1620 0.07017 -0.2950 -0.1091 -0.2359 0.26779 0.02561 0.10412 0.47189 -0.4098 -0.9068 -0.0623 0.45619 0.42932 0.28751 0.48666 0.40229 -0.2239 -0.2189 0.31506 0.27670 -0.1303 0.35651 -0.2937 -0.3062 -0.2179 -0.5673 -0.2726 -0.0297 -0.1477 0.01933 0.09591 -0.0655 0.12171 -0.0393 -0.1107 0.26964 0.48774 -0.0414 0.04251 0.23567 0.09863 0.00301 0.06130 -0.0466 -0.0438 0.17344 0.04784 0.03252 -0.0783 -0.0001 -0.0029 -0.0324 0.03180 -0.0040 -0.0381 0.00140 -0.0249 -0.0881 0.09040 -0.0232 0.08806 0.06752 -0.0673 -0.0336 0.01122 0.48343 +-0.2737 0.18401 0.26461 -0.1796 0.09481 -0.2896 -0.1062 -0.2565 0.25462 0.07530 0.12636 0.47335 -0.4033 -0.9249 -0.1121 0.49646 0.41493 0.30432 0.48114 0.45196 -0.2667 -0.1959 0.29579 0.25255 -0.1557 0.37068 -0.3704 -0.2480 -0.1993 -0.5935 -0.2596 -0.0348 -0.1479 0.01410 0.03987 -0.0527 0.10154 -0.0509 -0.0958 0.27474 0.51860 -0.0449 0.05797 0.23778 0.09051 -0.0004 0.06414 -0.0430 -0.0431 0.17707 0.04371 0.03266 -0.0794 0.00020 -0.0014 -0.0343 0.03199 -0.0048 -0.0398 -0.0033 -0.0213 -0.0860 0.08476 -0.0234 0.09091 0.07007 -0.0675 -0.0337 0.01100 0.47917 +-0.3623 0.16821 0.22216 -0.1522 0.10808 -0.4145 -0.1187 -0.3112 0.22564 0.10828 0.21872 0.46398 -0.4569 -0.9872 -0.1431 0.47575 0.40768 0.32767 0.48052 0.47825 -0.2300 -0.2239 0.24473 0.28339 -0.1424 0.34417 -0.3087 -0.2779 -0.2143 -0.5620 -0.2677 -0.0163 -0.1605 0.01533 0.10592 -0.0171 0.11813 -0.0562 -0.1134 0.27462 0.46984 -0.0265 0.03199 0.23493 0.10770 0.00303 0.06701 -0.0420 -0.0428 0.18010 0.05177 0.02863 -0.0659 0.00883 0.00034 -0.0327 0.03371 0.00074 -0.0381 0.00818 -0.0305 -0.0875 0.08568 -0.0236 0.09321 0.07247 -0.0668 -0.0338 0.01142 0.46293 +-0.4669 0.11417 0.09248 -0.0324 0.18022 -0.4511 -0.1197 -0.2658 0.23098 0.05701 0.20067 0.52876 -0.3993 -0.8972 -0.0377 0.41769 0.38132 0.30436 0.47103 0.48379 -0.2755 -0.2406 0.19609 0.26281 -0.1164 0.34191 -0.2658 -0.2214 -0.1751 -0.5350 -0.2750 -0.0178 -0.1569 0.02524 0.13879 -0.0229 0.04708 -0.0769 -0.1057 0.27832 0.41236 -0.0193 -0.0027 0.21852 0.13220 0.01169 0.06471 -0.0521 -0.0434 0.17511 0.05235 0.03175 -0.0732 0.00490 0.00093 -0.0359 0.03784 -0.0001 -0.0368 0.00336 -0.0240 -0.0866 0.08695 -0.0224 0.09347 0.07567 -0.0661 -0.0335 0.01137 0.46251 +-0.2363 0.20379 0.23276 0.16472 0.25789 -0.2894 -0.0931 -0.1882 0.28261 -0.0081 0.01075 0.63079 -0.3404 -0.6996 0.17862 0.54810 0.42108 0.33312 0.47687 0.51390 -0.2705 -0.2470 0.20166 0.25305 -0.1672 0.35379 -0.3612 -0.1622 -0.1719 -0.5586 -0.2824 -0.0438 -0.1524 0.02024 0.12729 -0.0614 0.02531 -0.0632 -0.0593 0.29124 0.43858 -0.0347 0.03099 0.22458 0.11336 0.01973 0.06125 -0.0227 -0.0460 0.16938 0.04839 0.02817 -0.0740 0.00787 -0.0031 -0.0271 0.02579 -0.0135 -0.0314 0.00941 -0.0265 -0.0833 0.07584 -0.0203 0.09072 0.07781 -0.0654 -0.0333 0.01085 0.47123 +-0.1172 0.28111 0.26640 0.27336 0.28858 -0.1458 -0.0668 -0.1651 0.27683 -0.0570 -0.1257 0.57224 -0.2900 -0.6676 0.32683 0.51150 0.43298 0.28531 0.50563 0.44293 -0.2678 -0.1684 0.18398 0.24654 -0.0816 0.33949 -0.2514 -0.2789 -0.1683 -0.4769 -0.2890 -0.0403 -0.1455 0.01972 0.13157 -0.0783 0.06693 -0.0396 -0.0815 0.27533 0.43132 -0.0279 0.02104 0.22295 0.10848 0.00094 0.05602 -0.0276 -0.0299 0.16934 0.03928 0.02978 -0.0732 -0.0022 -0.0075 -0.0318 0.04011 -0.0013 -0.0461 -0.0035 -0.0249 -0.0879 0.09076 -0.0205 0.08688 0.07982 -0.0645 -0.0325 0.01163 0.47114 +-0.0713 0.24552 0.33123 0.24082 0.19649 -0.0705 -0.1512 -0.1179 0.27214 -0.0703 -0.0824 0.51019 -0.2980 -0.8789 0.28844 0.51106 0.55225 0.06057 0.61176 0.10712 -0.1136 -0.4115 0.01474 0.31474 -0.1148 0.42109 -0.1940 -0.4763 -0.1851 -0.5362 -0.1869 -0.0238 -0.2548 -0.1427 0.10785 -0.1173 0.09017 -0.0092 -0.2500 0.17674 0.50434 0.10333 -0.0903 0.31615 0.04303 0.02606 0.02897 -0.0064 -0.0471 0.16216 0.03008 0.03827 -0.0747 0.02083 0.00517 -0.0248 0.03617 -0.0090 -0.0287 0.01076 -0.0196 -0.0880 0.07083 -0.0314 0.08426 0.08025 -0.0636 -0.0314 0.01314 0.46600 +-0.0302 0.26653 0.39084 0.07450 0.18517 -0.2154 -0.1628 -0.1633 0.22842 -0.0147 0.09948 0.47153 -0.3083 -1.0712 0.08261 0.51284 0.53609 -0.0135 0.63955 -0.0217 -0.1101 -0.6396 -0.2879 0.30942 -0.1996 0.49285 -0.2218 -0.5298 -0.1375 -0.5598 -0.1021 -0.0521 -0.3118 -0.2720 0.06672 -0.0735 0.01368 -0.0230 -0.3424 0.12741 0.57695 0.21424 -0.1786 0.36520 0.00326 0.05010 0.00577 -0.0065 -0.0574 0.14022 0.06322 0.04185 -0.0676 0.02947 0.00719 -0.0223 0.03126 -0.0226 -0.0328 0.02006 -0.0204 -0.1077 0.05623 -0.0278 0.08231 0.08106 -0.0630 -0.0304 0.01437 0.48183 +-0.1035 0.21695 0.40030 -0.0564 0.11196 -0.3260 -0.1472 -0.1718 0.18410 0.06767 0.18247 0.44609 -0.3301 -1.1658 -0.1016 0.57183 0.54476 -0.0014 0.64576 -0.0714 -0.1228 -0.6789 -0.3483 0.26165 -0.1774 0.51728 -0.2224 -0.4900 -0.0738 -0.5787 -0.0687 -0.0719 -0.3598 -0.3194 0.02739 -0.0107 -0.0132 -0.0471 -0.3478 0.10897 0.58442 0.25399 -0.2199 0.39448 -0.0018 0.05819 0.00716 0.00818 -0.0647 0.14837 0.06756 0.04322 -0.0574 0.03250 0.01769 -0.0282 0.02954 -0.0255 -0.0340 0.02446 -0.0161 -0.1127 0.05871 -0.0230 0.08273 0.08128 -0.0619 -0.0291 0.01557 0.47920 +-0.3613 0.12207 0.22279 -0.0672 0.09190 -0.5008 -0.1408 -0.2297 0.18070 0.01943 0.21566 0.42575 -0.3467 -1.1562 -0.0847 0.42379 0.56343 -0.0061 0.63873 -0.0111 -0.1729 -0.6422 -0.3156 0.32997 -0.1487 0.50804 -0.2300 -0.4203 -0.0581 -0.5604 -0.1031 -0.0675 -0.3666 -0.2698 0.01163 0.02031 -0.0026 -0.0514 -0.3350 0.13719 0.57854 0.21452 -0.2047 0.36661 0.00474 0.03630 0.01135 -0.0122 -0.0668 0.15026 0.06084 0.04381 -0.0732 0.03130 0.01409 -0.0329 0.03148 -0.0241 -0.0345 0.02278 -0.0163 -0.1041 0.05788 -0.0232 0.08451 0.08233 -0.0604 -0.0277 0.01699 0.48291 +-0.2725 0.16239 0.17155 0.20883 0.32810 -0.3595 -0.1372 -0.1831 0.24683 -0.0214 0.09823 0.58632 -0.2856 -0.8527 0.22677 0.39864 0.48703 0.10071 0.52397 0.23775 -0.2201 -0.4530 -0.1116 0.28451 -0.1468 0.44510 -0.2970 -0.2759 -0.1050 -0.5429 -0.1859 -0.0712 -0.2430 -0.1253 0.13486 -0.0897 -0.0103 -0.0718 -0.2154 0.18074 0.46155 0.12302 -0.1261 0.27938 0.07165 0.03023 0.03964 -0.0343 -0.0590 0.15731 0.05332 0.03551 -0.0694 0.02364 0.00734 -0.0291 0.03342 -0.0165 -0.0275 0.01981 -0.0189 -0.0926 0.06609 -0.0215 0.08751 0.08325 -0.0597 -0.0267 0.01771 0.50027 +-0.0726 0.25032 0.23531 0.24845 0.29652 -0.2910 -0.0597 -0.1577 0.28034 -0.0034 -0.0156 0.54561 -0.2812 -0.6423 0.24744 0.51690 0.37448 0.26885 0.44626 0.46903 -0.2389 -0.1620 0.11133 0.27950 -0.1130 0.32575 -0.2523 -0.3057 -0.2052 -0.4929 -0.3152 -0.0528 -0.1320 0.02918 0.20599 -0.0643 0.07607 -0.0653 -0.0977 0.28898 0.39590 -0.0217 0.01662 0.22038 0.11150 -0.0035 0.06711 -0.0457 -0.0211 0.16960 0.04466 0.02308 -0.0784 0.00494 -0.0039 -0.0317 0.03650 0.00118 -0.0462 -0.0017 -0.0202 -0.0973 0.07992 -0.0205 0.09167 0.08522 -0.0591 -0.0258 0.01733 0.51008 +-0.2357 0.20115 0.16657 -0.0206 0.18462 -0.4243 -0.0389 -0.2092 0.29213 0.06364 0.17581 0.47290 -0.3587 -0.7546 -0.0420 0.48750 0.37025 0.32437 0.47234 0.50525 -0.2386 -0.2355 0.12329 0.32374 -0.0550 0.33659 -0.2701 -0.3112 -0.2626 -0.5712 -0.2974 -0.0368 -0.1139 0.00527 0.19260 -0.0481 0.12918 -0.0431 -0.1092 0.28651 0.44395 -0.0063 0.01545 0.21420 0.09882 0.01541 0.06169 -0.0298 -0.0414 0.19490 0.04457 0.02685 -0.0787 0.00911 -0.0098 -0.0225 0.02374 -0.0043 -0.0364 0.00525 -0.0228 -0.0831 0.07325 -0.0321 0.09690 0.08694 -0.0586 -0.0253 0.01640 0.52257 +-0.5365 0.09865 0.05659 -0.1929 0.12884 -0.4923 -0.0509 -0.2651 0.24771 0.07336 0.26792 0.44726 -0.4033 -0.9302 -0.0845 0.42126 0.38984 0.29981 0.46817 0.49095 -0.2237 -0.2409 0.20188 0.33072 -0.1428 0.34455 -0.2490 -0.3103 -0.2273 -0.5620 -0.2851 -0.0277 -0.1387 0.03462 0.17072 -0.0423 0.09609 -0.0596 -0.1226 0.28117 0.41377 0.00612 -0.0246 0.19211 0.11648 0.00599 0.05831 -0.0563 -0.0522 0.17799 0.04706 0.03036 -0.0710 0.00604 -0.0053 -0.0245 0.03214 -0.0026 -0.0367 0.00702 -0.0208 -0.0901 0.07759 -0.0290 0.10113 0.08929 -0.0580 -0.0249 0.01615 0.51247 +-0.4962 0.12460 0.10023 -0.4496 -0.0459 -0.5657 -0.0925 -0.2672 0.21901 0.11394 0.27623 0.33873 -0.4120 -0.9605 -0.1745 0.29683 0.43025 0.27571 0.49528 0.37492 -0.1988 -0.2326 0.32748 0.34169 -0.0993 0.34116 -0.1639 -0.4137 -0.3081 -0.5650 -0.2725 -0.0356 -0.1588 -0.0011 0.10702 -0.0404 0.15879 -0.0442 -0.1286 0.27025 0.44108 0.00305 -0.0006 0.21196 0.11360 0.00725 0.05910 -0.0540 -0.0535 0.17915 0.03535 0.03197 -0.0803 0.01289 -0.0059 -0.0379 0.03138 -0.0026 -0.0366 0.00386 -0.0237 -0.0843 0.08262 -0.0276 0.10522 0.08962 -0.0580 -0.0250 0.01572 0.50212 +-0.2126 0.21615 0.31179 -0.2116 0.03634 -0.3047 -0.1039 -0.2018 0.28347 0.06940 0.09762 0.40547 -0.3536 -0.8224 -0.0253 0.47086 0.43174 0.27478 0.48699 0.39365 -0.1540 -0.2491 0.30922 0.35878 -0.1559 0.35158 -0.2246 -0.4587 -0.3388 -0.5800 -0.2385 -0.0103 -0.1210 0.00325 0.14906 -0.1212 0.18014 -0.0411 -0.1515 0.26426 0.46575 -0.0330 0.02777 0.22474 0.09641 0.01873 0.06310 -0.0481 -0.0526 0.19101 0.04783 0.02947 -0.0772 -0.0029 -0.0119 -0.0271 0.02545 -0.0050 -0.0424 0.00493 -0.0187 -0.0715 0.08290 -0.0248 0.10789 0.08848 -0.0599 -0.0251 0.01418 0.50475 +0.25160 0.32807 0.56980 0.12474 0.14590 0.00218 -0.0719 -0.0827 0.32184 0.03859 -0.2325 0.36308 -0.1673 -0.4735 0.43309 0.41953 0.42826 0.16013 0.48933 0.22250 -0.1237 -0.1201 0.38734 0.34142 -0.0450 0.31843 -0.0347 -0.6297 -0.3610 -0.4935 -0.2575 0.01828 -0.1346 -0.0032 0.22025 -0.1659 0.25349 -0.0003 -0.1345 0.23997 0.40747 -0.0248 0.06619 0.24966 0.09713 0.00680 0.04495 0.00440 -0.0211 0.18509 0.03045 0.03323 -0.0721 -0.0011 -0.0017 -0.0151 0.00658 -0.0053 -0.0448 0.01017 -0.0171 -0.0813 0.08593 -0.0255 0.10973 0.08546 -0.0624 -0.0253 0.01191 0.49882 +-0.0195 0.26503 0.32720 0.07841 0.19238 -0.2355 -0.0403 -0.1865 0.24141 0.02520 -0.0634 0.27780 -0.2607 -0.6558 0.26969 0.35245 0.43200 0.15860 0.48471 0.28497 -0.1224 -0.1597 0.30516 0.39161 0.02921 0.30596 -0.0108 -0.5982 -0.3708 -0.5148 -0.2812 0.01590 -0.1487 0.00873 0.27446 -0.1291 0.24426 -0.0124 -0.1720 0.30226 0.38528 0.00327 0.02545 0.22764 0.09680 0.01319 0.05153 -0.0339 -0.0416 0.18052 0.04057 0.03379 -0.0787 0.00349 -0.0030 -0.0245 0.02816 -0.0074 -0.0439 0.00248 -0.0139 -0.0807 0.08431 -0.0300 0.11119 0.08228 -0.0649 -0.0254 0.01069 0.49008 +-0.3434 0.14461 0.19953 -0.2446 0.04148 -0.3819 -0.0971 -0.2504 0.26670 0.04309 0.22278 0.33056 -0.4176 -1.0065 -0.0206 0.47251 0.41334 0.24515 0.49576 0.38611 -0.1010 -0.2268 0.27277 0.34892 -0.1149 0.33309 -0.1563 -0.4657 -0.3434 -0.5624 -0.3001 0.01316 -0.1598 0.00196 0.20538 -0.0905 0.25266 -0.0236 -0.1945 0.28221 0.40948 -0.0095 -0.0020 0.20750 0.11400 0.00589 0.06398 -0.0615 -0.0465 0.16569 0.03981 0.03437 -0.0770 0.01193 -0.0032 -0.0281 0.02697 -0.0103 -0.0316 0.00972 -0.0252 -0.0893 0.08465 -0.0292 0.11270 0.07865 -0.0676 -0.0258 0.00971 0.48719 +-0.4159 0.14780 0.13797 -0.1960 0.03225 -0.4003 -0.1029 -0.2826 0.22846 0.04713 0.14033 0.41000 -0.3762 -0.9807 -0.0552 0.41419 0.42594 0.31176 0.48231 0.41857 -0.2947 -0.2061 0.31796 0.25362 -0.0994 0.34384 -0.2935 -0.3126 -0.1965 -0.5276 -0.2615 -0.0404 -0.1717 0.00863 0.06246 -0.0447 0.12039 -0.0408 -0.1099 0.26658 0.48253 -0.0297 0.04255 0.23162 0.10719 0.01199 0.05782 -0.0462 -0.0405 0.17318 0.03692 0.03282 -0.0733 0.00732 -0.0077 -0.0314 0.02713 -0.0083 -0.0353 0.00381 -0.0213 -0.0870 0.08276 -0.0296 0.11372 0.07608 -0.0700 -0.0263 0.00819 0.49865 +-0.2757 0.19059 0.20111 -0.0388 0.10536 -0.3292 -0.1296 -0.2751 0.26290 -0.0178 -0.0101 0.50925 -0.3812 -0.9228 0.07841 0.42115 0.43038 0.30740 0.48260 0.40406 -0.2989 -0.2122 0.29665 0.21849 -0.0998 0.34208 -0.3118 -0.2617 -0.1647 -0.5118 -0.2730 -0.0318 -0.1713 0.04693 0.08882 -0.0461 0.08348 -0.0298 -0.0949 0.26555 0.48197 -0.0542 0.03305 0.23637 0.10200 0.00710 0.06639 -0.0408 -0.0435 0.17827 0.03412 0.03172 -0.0623 0.00354 0.00075 -0.0362 0.02887 0.00107 -0.0380 -0.0020 -0.0219 -0.0923 0.09650 -0.0243 0.11505 0.07348 -0.0719 -0.0268 0.00667 0.50550 +-0.2266 0.17762 0.27054 -0.0027 0.12175 -0.2541 -0.1499 -0.2825 0.25600 0.00926 0.01762 0.46941 -0.4024 -0.9818 0.05652 0.47153 0.43584 0.29553 0.51777 0.44257 -0.2715 -0.2508 0.26069 0.27481 -0.0636 0.34016 -0.2652 -0.3136 -0.2428 -0.5244 -0.2951 0.00024 -0.1447 0.03651 0.14268 -0.0520 0.13771 -0.0184 -0.1223 0.26863 0.45118 -0.0375 0.02701 0.23577 0.10405 0.00415 0.05621 -0.0455 -0.0379 0.16955 0.05022 0.02966 -0.0727 0.00382 -0.0021 -0.0273 0.03612 0.00188 -0.0376 0.00290 -0.0268 -0.0881 0.09009 -0.0258 0.11594 0.07292 -0.0733 -0.0273 0.00534 0.52167 +-0.1829 0.16202 0.22948 -0.0273 0.10941 -0.2759 -0.1214 -0.3067 0.22827 0.06287 0.10696 0.44938 -0.4066 -1.0336 -0.1074 0.57199 0.38735 0.31303 0.48771 0.46848 -0.3142 -0.2514 0.20392 0.27566 -0.0683 0.33313 -0.2645 -0.2872 -0.2197 -0.5350 -0.3085 0.01133 -0.1574 0.02476 0.14312 0.02156 0.12236 -0.0279 -0.1264 0.28211 0.44369 -0.0276 0.03260 0.22556 0.10242 0.00887 0.05802 -0.0465 -0.0393 0.17219 0.04631 0.03008 -0.0705 0.00526 0.00153 -0.0315 0.03291 0.00246 -0.0345 -0.0015 -0.0348 -0.0998 0.09365 -0.0255 0.11619 0.07406 -0.0743 -0.0280 0.00443 0.53404 +-0.1296 0.15572 0.28276 -0.0025 0.14193 -0.2975 -0.0783 -0.3161 0.23159 0.11647 0.13501 0.44001 -0.4155 -1.0589 -0.1827 0.61746 0.38332 0.33009 0.48138 0.46766 -0.3355 -0.2527 0.17887 0.26297 -0.0395 0.31735 -0.2539 -0.2622 -0.2091 -0.5265 -0.3023 0.01180 -0.1871 0.00853 0.13072 0.05088 0.13162 -0.0424 -0.1285 0.28543 0.44163 -0.0297 0.03353 0.23171 0.11418 0.00904 0.06205 -0.0381 -0.0344 0.17062 0.05448 0.02957 -0.0626 -0.0020 0.00362 -0.0311 0.03627 8.07957 -0.0352 0.00771 -0.0307 -0.1010 0.09420 -0.0252 0.11655 0.07693 -0.0745 -0.0284 0.00320 0.54007 +-0.1387 0.17023 0.32072 -0.0514 0.14450 -0.2779 -0.0431 -0.3517 0.22236 0.13675 0.19283 0.40500 -0.4458 -1.0966 -0.2662 0.70402 0.36201 0.33219 0.48126 0.43040 -0.3657 -0.2373 0.13735 0.26666 -0.0206 0.30407 -0.2111 -0.2608 -0.1866 -0.5172 -0.3249 0.04046 -0.1917 0.01906 0.14623 0.09860 0.12476 -0.0342 -0.1134 0.29054 0.44531 -0.0122 0.02927 0.22934 0.12095 0.00710 0.04748 -0.0373 -0.0208 0.16716 0.05397 0.03297 -0.0620 0.00629 0.00959 -0.0301 0.03173 -0.0019 -0.0375 0.00546 -0.0296 -0.0970 0.09403 -0.0231 0.11693 0.07948 -0.0742 -0.0287 0.00174 0.54996 +-0.2677 0.16760 0.25253 -0.1751 0.04013 -0.3169 -0.0763 -0.3241 0.23172 0.12215 0.19109 0.34482 -0.4626 -1.0765 -0.2467 0.57767 0.39343 0.32375 0.48575 0.45242 -0.2832 -0.2224 0.17173 0.26981 -0.0366 0.31125 -0.2132 -0.2712 -0.2011 -0.5377 -0.3102 0.03722 -0.1892 0.01046 0.12733 0.06093 0.15072 -0.0484 -0.1216 0.28116 0.44757 -0.0285 0.01627 0.23401 0.11803 0.00310 0.04979 -0.0503 -0.0299 0.17981 0.05616 0.02812 -0.0599 0.00817 0.00822 -0.0268 0.03258 -0.0007 -0.0388 0.00980 -0.0252 -0.0959 0.09318 -0.0193 0.11502 0.08214 -0.0734 -0.0290 0.00165 0.54742 +-0.5472 0.08043 0.11046 -0.2852 0.00306 -0.5183 -0.0507 -0.3559 0.19251 0.12489 0.26585 0.38127 -0.4812 -1.0588 -0.2272 0.46671 0.39767 0.33074 0.49420 0.44915 -0.2778 -0.2372 0.23029 0.31809 -0.0545 0.32757 -0.2253 -0.3187 -0.2184 -0.5506 -0.2769 -0.0052 -0.1927 0.01726 0.10797 0.03672 0.13211 -0.0570 -0.1383 0.26189 0.45158 -0.0125 0.00089 0.22112 0.12028 0.00311 0.05083 -0.0597 -0.0501 0.17955 0.04938 0.03167 -0.0654 0.00442 0.00915 -0.0280 0.03233 -0.0002 -0.0287 0.00618 -0.0247 -0.0813 0.08642 -0.0191 0.11280 0.08304 -0.0728 -0.0295 0.00222 0.54445 +-0.5704 0.08443 0.09487 -0.2329 0.04390 -0.4743 -0.1295 -0.2686 0.21463 0.08873 0.28771 0.44584 -0.4405 -0.9285 -0.0929 0.37413 0.40314 0.28697 0.48231 0.42532 -0.2210 -0.2768 0.25692 0.32190 -0.1277 0.33891 -0.1967 -0.3440 -0.2596 -0.5498 -0.2618 -0.0086 -0.1671 0.01819 0.15200 -0.0480 0.13014 -0.0662 -0.1216 0.24972 0.40467 -0.0070 -0.0142 0.21232 0.13406 0.00872 0.05717 -0.0619 -0.0575 0.19142 0.04502 0.03081 -0.0702 0.00859 -3.3807 -0.0344 0.03246 -0.0016 -0.0354 0.00740 -0.0217 -0.0878 0.09182 -0.0192 0.11035 0.08242 -0.0727 -0.0297 0.00220 0.54137 +-0.2654 0.18509 0.23632 -0.0453 0.11826 -0.3594 -0.1018 -0.2099 0.26776 -0.0294 0.14492 0.55129 -0.3596 -0.7594 0.09265 0.42406 0.41829 0.28587 0.47277 0.45348 -0.1903 -0.2539 0.27772 0.30769 -0.1640 0.33286 -0.2679 -0.3115 -0.2262 -0.5382 -0.2730 -0.0266 -0.1463 0.01453 0.13408 -0.0817 0.12273 -0.0535 -0.1199 0.28020 0.43430 -0.0285 0.00847 0.21688 0.12239 0.01345 0.05815 -0.0477 -0.0494 0.17880 0.05155 0.03155 -0.0790 0.00163 -0.0142 -0.0209 0.02787 -0.0080 -0.0383 0.00667 -0.0179 -0.0864 0.07317 -0.0307 0.10872 0.08009 -0.0727 -0.0295 0.00224 0.52759 +0.01144 0.26520 0.34379 0.17016 0.18172 -0.1153 -0.0828 -0.1383 0.32211 -0.0328 -0.1113 0.49942 -0.2367 -0.6144 0.33671 0.50704 0.41952 0.22491 0.47602 0.38593 -0.1847 -0.1561 0.25489 0.32865 -0.1374 0.32838 -0.1987 -0.4337 -0.3168 -0.5516 -0.2423 -0.0041 -0.1093 0.01329 0.19084 -0.1412 0.16333 -0.0417 -0.1137 0.28638 0.43636 -0.0404 0.03982 0.23698 0.09969 0.00720 0.06211 -0.0191 -0.0283 0.17791 0.03946 0.03333 -0.0775 -0.0007 -0.0047 -0.0210 0.02063 -0.0080 -0.0436 0.00186 -0.0189 -0.0750 0.07867 -0.0210 0.10631 0.07805 -0.0722 -0.0294 0.00278 0.51488 +0.13119 0.27599 0.41494 0.32871 0.26738 -0.0390 -0.0498 -0.1444 0.33776 -0.0508 -0.1754 0.46004 -0.2377 -0.6442 0.36349 0.53977 0.41942 0.27463 0.46859 0.41021 -0.1929 -0.1763 0.17279 0.30146 -0.0276 0.31394 -0.2263 -0.3834 -0.1917 -0.5041 -0.2927 0.00454 -0.1171 0.02521 0.17092 -0.1152 0.13246 -0.0292 -0.0828 0.30419 0.42419 -0.0241 0.03871 0.22532 0.09886 0.01339 0.05523 -0.0222 -0.0230 0.16951 0.04461 0.03097 -0.0716 -0.0015 -0.0012 -0.0175 0.02256 -0.0035 -0.0406 0.00447 -0.0269 -0.0792 0.08131 -0.0230 0.10461 0.07549 -0.0722 -0.0293 0.00359 0.51094 +0.16692 0.27596 0.36214 0.44921 0.33443 -0.0826 -0.0723 -0.1687 0.29175 -0.0576 -0.1875 0.44469 -0.2515 -0.6570 0.37768 0.51370 0.39351 0.23842 0.46322 0.39227 -0.2038 -0.1750 0.15878 0.31725 0.00507 0.28712 -0.1554 -0.3385 -0.1700 -0.4648 -0.3306 0.00504 -0.1295 0.02276 0.22387 -0.0971 0.11978 -0.0574 -0.0954 0.30919 0.37601 -0.0083 0.01078 0.21934 0.10768 0.00904 0.05162 -0.0192 -0.0189 0.16840 0.05340 0.03089 -0.0740 0.00191 -0.0015 -0.0180 0.02664 -0.0016 -0.0342 0.00489 -0.0320 -0.0811 0.08786 -0.0313 0.10426 0.07361 -0.0726 -0.0291 0.00345 0.50781 +0.11367 0.27331 0.30999 0.52759 0.41063 -0.0702 -0.0635 -0.1747 0.26726 -0.0357 -0.2186 0.46759 -0.2582 -0.5649 0.47336 0.47074 0.39506 0.20292 0.46928 0.39000 -0.2317 -0.1144 0.13006 0.31621 0.05465 0.28854 -0.1088 -0.3398 -0.1564 -0.4294 -0.3596 0.00959 -0.1587 0.02292 0.24234 -0.0763 0.11214 -0.0607 -0.0935 0.28388 0.35764 0.00753 0.00711 0.21486 0.11776 0.01180 0.05348 -0.0275 -0.0133 0.15613 0.04992 0.03539 -0.0733 0.00079 -0.0027 -0.0268 0.02321 0.00062 -0.0350 0.00130 -0.0308 -0.0904 0.08406 -0.0291 0.10507 0.07229 -0.0729 -0.0292 0.00344 0.50610 +0.09579 0.26393 0.18956 0.67212 0.51505 -0.0998 -0.0551 -0.2069 0.26827 -0.0507 -0.2537 0.49614 -0.2325 -0.5005 0.52661 0.44977 0.38556 0.15991 0.45135 0.35943 -0.3067 -0.1001 0.09574 0.31643 0.08291 0.27745 -0.0991 -0.3097 -0.1170 -0.3945 -0.3692 0.00433 -0.1677 0.03229 0.29270 -0.0579 0.06444 -0.0768 -0.0932 0.30137 0.33568 0.01699 0.00036 0.21523 0.11654 0.00833 0.05756 -0.0366 -0.0182 0.15311 0.05035 0.03418 -0.0728 -0.0023 -0.0071 -0.0245 0.02860 0.00129 -0.0314 0.00224 -0.0288 -0.0969 0.07650 -0.0278 0.10577 0.07307 -0.0726 -0.0293 0.00376 0.50040 +0.02541 0.22141 0.12908 0.53424 0.49005 -0.1977 -0.0372 -0.1937 0.26851 -0.0633 -0.1323 0.56222 -0.1885 -0.5194 0.39862 0.50472 0.35679 0.21167 0.44325 0.43317 -0.2631 -0.1296 0.05611 0.31343 0.02792 0.30442 -0.1521 -0.2600 -0.1446 -0.4200 -0.3292 -0.0098 -0.1286 0.03120 0.27957 -0.0940 0.08587 -0.0853 -0.0991 0.28990 0.34266 0.01722 -0.0035 0.21208 0.11743 0.01700 0.05741 -0.0312 -0.0290 0.16806 0.04336 0.03021 -0.0764 -0.0034 -0.0044 -0.0165 0.02477 -0.0054 -0.0289 0.00578 -0.0210 -0.0924 0.06108 -0.0370 0.10604 0.07478 -0.0715 -0.0294 0.00485 0.48115 +-0.1037 0.20924 0.12582 0.38837 0.43369 -0.2692 -0.0390 -0.2362 0.26478 -0.0032 -0.0809 0.60824 -0.2667 -0.6546 0.25819 0.53589 0.35781 0.27250 0.47459 0.41914 -0.2877 -0.1831 0.10863 0.33967 -0.0223 0.31552 -0.1964 -0.2625 -0.1834 -0.4628 -0.3260 -0.0127 -0.1326 0.03642 0.23948 -0.0764 0.08430 -0.0738 -0.1009 0.32657 0.34673 0.02193 -0.0268 0.23316 0.10675 0.00664 0.05899 -0.0332 -0.0308 0.16960 0.04456 0.02810 -0.0770 0.00578 -0.0106 -0.0204 0.03269 -0.0005 -0.0358 -0.0016 -0.0253 -0.0926 0.06515 -0.0312 0.10510 0.07711 -0.0706 -0.0289 0.00522 0.47450 +-0.3495 0.15414 0.07892 0.07778 0.29910 -0.3577 -0.0453 -0.2222 0.26830 0.06822 0.11692 0.57100 -0.3718 -0.8647 0.11185 0.48532 0.36050 0.31323 0.48395 0.52169 -0.2324 -0.2356 0.16070 0.30435 -0.0992 0.32117 -0.2736 -0.2315 -0.2082 -0.5589 -0.3032 -0.0220 -0.1426 0.02439 0.17186 -0.0717 0.07219 -0.0622 -0.1080 0.28818 0.40378 0.00627 -0.0021 0.19336 0.12252 0.01517 0.06327 -0.0452 -0.0396 0.17478 0.04627 0.03363 -0.0807 0.00088 -0.0016 -0.0318 0.03926 -0.0018 -0.0369 0.00212 -0.0243 -0.0941 0.07465 -0.0292 0.10146 0.07778 -0.0695 -0.0281 0.00476 0.48979 +-0.3664 0.11990 0.10629 -0.0360 0.18104 -0.4043 -0.0642 -0.2360 0.25681 0.06019 0.17579 0.50375 -0.4016 -0.9166 -0.0091 0.45967 0.37022 0.32728 0.45450 0.50049 -0.2348 -0.2520 0.19407 0.25936 -0.1309 0.33753 -0.3044 -0.2641 -0.2027 -0.5480 -0.2769 -0.0219 -0.1456 0.02270 0.15530 -0.0573 0.08420 -0.0567 -0.1021 0.27497 0.43230 -0.0062 0.01188 0.20428 0.12474 0.00467 0.06295 -0.0588 -0.0426 0.17615 0.04107 0.03176 -0.0768 0.00275 0.00124 -0.0315 0.03359 0.00039 -0.0354 0.00438 -0.0232 -0.0866 0.08335 -0.0221 0.09744 0.07957 -0.0678 -0.0275 0.00571 0.48986 +-0.3603 0.14065 0.14474 -0.0891 0.12255 -0.4031 -0.0828 -0.2542 0.25507 0.05558 0.15316 0.47654 -0.4038 -0.9664 -0.0132 0.45738 0.39455 0.33372 0.49447 0.46558 -0.2778 -0.2689 0.25978 0.22298 -0.1331 0.33842 -0.3317 -0.2404 -0.1890 -0.5393 -0.2581 -0.0281 -0.1388 0.03131 0.09797 -0.0549 0.07694 -0.0493 -0.0982 0.27562 0.47282 -0.0345 0.03617 0.22800 0.11803 0.01079 0.05871 -0.0493 -0.0417 0.17305 0.04503 0.03249 -0.0723 0.00257 -0.0034 -0.0345 0.03257 0.00175 -0.0447 0.00211 -0.0219 -0.0855 0.08585 -0.0244 0.09464 0.08197 -0.0654 -0.0271 0.00745 0.48677 +-0.2621 0.16651 0.24694 -0.1573 0.05793 -0.3435 -0.1040 -0.2457 0.27309 0.05592 0.15288 0.41831 -0.4182 -0.9716 -0.1008 0.48317 0.40115 0.30177 0.48982 0.45974 -0.2402 -0.2562 0.29705 0.24276 -0.1414 0.34614 -0.3026 -0.2771 -0.2298 -0.5663 -0.2624 -0.0126 -0.1566 0.02493 0.08792 -0.0449 0.12253 -0.0385 -0.1133 0.25935 0.48836 -0.0469 0.04726 0.23873 0.11152 0.00717 0.06457 -0.0421 -0.0438 0.17677 0.03821 0.03174 -0.0710 0.00715 -0.0006 -0.0396 0.03298 -0.0004 -0.0401 0.00185 -0.0226 -0.0881 0.09117 -0.0234 0.09316 0.08381 -0.0631 -0.0266 0.00852 0.49234 +-0.1328 0.19502 0.27537 -0.1420 0.06727 -0.2569 -0.1161 -0.2495 0.25134 0.06058 0.15253 0.43196 -0.4163 -1.0061 -0.1394 0.59542 0.38236 0.29151 0.49995 0.43592 -0.2443 -0.2658 0.24678 0.25326 -0.1360 0.33035 -0.2496 -0.3417 -0.2914 -0.5677 -0.2874 0.01203 -0.1452 0.01245 0.14815 -0.0241 0.17387 -0.0105 -0.1365 0.27383 0.47048 -0.0266 0.04913 0.22265 0.09803 0.00916 0.05758 -0.0435 -0.0434 0.17830 0.04098 0.03628 -0.0758 0.00974 -0.0027 -0.0302 0.03075 -0.0040 -0.0374 -0.0009 -0.0301 -0.0862 0.08934 -0.0254 0.09310 0.08598 -0.0611 -0.0260 0.01022 0.47728 +-0.1189 0.18704 0.31499 -0.0208 0.11034 -0.2660 -0.0867 -0.2790 0.24769 0.10104 0.16223 0.41370 -0.3919 -1.0014 -0.1824 0.62821 0.38502 0.30152 0.49845 0.44123 -0.2729 -0.2422 0.16042 0.28185 -0.0531 0.30523 -0.2137 -0.3092 -0.2422 -0.5223 -0.3162 0.02579 -0.1715 0.00701 0.16331 0.03063 0.16575 -0.0373 -0.1348 0.27164 0.43626 -0.0161 0.02303 0.22451 0.11296 0.01075 0.05480 -0.0374 -0.0285 0.17012 0.05341 0.03202 -0.0647 0.00161 0.00111 -0.0258 0.03405 -0.0034 -0.0347 0.00553 -0.0292 -0.0997 0.09085 -0.0251 0.09447 0.08782 -0.0594 -0.0257 0.01234 0.46174 +-0.1764 0.17998 0.31730 -0.0309 0.09033 -0.2872 -0.1131 -0.2443 0.21394 0.01726 0.19082 0.40465 -0.3865 -1.0476 -0.0525 0.52437 0.48705 0.11510 0.56744 0.16948 -0.1455 -0.4550 -0.0711 0.32172 -0.1089 0.40477 -0.1708 -0.4356 -0.1879 -0.5502 -0.1918 -0.0145 -0.2815 -0.1543 0.13785 0.00688 0.11634 -0.0334 -0.2788 0.16635 0.49321 0.12053 -0.1034 0.30174 0.05802 0.02748 0.03255 -0.0294 -0.0493 0.16176 0.06474 0.03644 -0.0628 0.02260 0.01177 -0.0239 0.03501 -0.0100 -0.0346 0.01679 -0.0248 -0.1038 0.07637 -0.0243 0.09676 0.08944 -0.0588 -0.0258 0.01410 0.45500 +-0.1673 0.19357 0.34834 0.08804 0.13611 -0.2549 -0.1655 -0.1887 0.17411 -0.0586 0.12341 0.40741 -0.3460 -1.0528 0.10109 0.44166 0.58621 -0.0563 0.63693 -0.0770 -0.0572 -0.6231 -0.2309 0.36158 -0.1762 0.49192 -0.1422 -0.5843 -0.1552 -0.5538 -0.1092 -0.0459 -0.3600 -0.2778 0.09726 -0.0405 0.07276 -0.0071 -0.3851 0.10728 0.53327 0.21592 -0.2084 0.37222 0.00813 0.04508 0.01143 -0.0150 -0.0673 0.14905 0.06526 0.04064 -0.0616 0.03462 0.01603 -0.0278 0.02884 -0.0199 -0.0340 0.02242 -0.0204 -0.1136 0.05802 -0.0302 0.10004 0.08996 -0.0586 -0.0263 0.01491 0.44257 +-0.1745 0.21051 0.40876 -0.0890 0.04649 -0.3023 -0.1501 -0.1514 0.22334 0.01780 0.26053 0.44581 -0.3274 -1.0656 -0.0624 0.48645 0.58915 -0.0372 0.65106 -0.0959 -0.0758 -0.6993 -0.2812 0.29809 -0.2181 0.52602 -0.2290 -0.5538 -0.1298 -0.6103 -0.0542 -0.0861 -0.3481 -0.3190 0.02934 -0.0538 0.03155 -0.0206 -0.3982 0.08906 0.60040 0.24356 -0.2133 0.38916 -0.0060 0.05066 0.01419 -0.0032 -0.0662 0.14738 0.06156 0.04338 -0.0669 0.04343 0.01690 -0.0292 0.02205 -0.0248 -0.0389 0.02846 -0.0164 -0.1013 0.05456 -0.0212 0.10278 0.09013 -0.0587 -0.0272 0.01578 0.43623 +-0.2782 0.17549 0.33916 -0.1745 -0.0077 -0.3552 -0.1522 -0.2151 0.18942 0.04783 0.26165 0.40433 -0.3707 -1.1089 -0.1216 0.46137 0.57035 -0.0105 0.62960 -0.0292 -0.1368 -0.6669 -0.2539 0.30662 -0.1826 0.51125 -0.2446 -0.4986 -0.0969 -0.5990 -0.0715 -0.0736 -0.3595 -0.2999 0.01506 -0.0187 0.02005 -0.0496 -0.3621 0.11647 0.59962 0.23042 -0.2034 0.36400 -0.0007 0.05049 0.00920 -0.0131 -0.0688 0.15653 0.06381 0.04000 -0.0655 0.03571 0.01536 -0.0301 0.02886 -0.0231 -0.0386 0.01934 -0.0182 -0.0980 0.06012 -0.0194 0.10489 0.08905 -0.0588 -0.0282 0.01676 0.41830 +-0.2968 0.14126 0.30715 -0.1413 0.03082 -0.3295 -0.1554 -0.2395 0.20187 0.01672 0.20063 0.38771 -0.4038 -1.1039 -0.0574 0.48973 0.50620 0.10379 0.59373 0.15903 -0.1634 -0.4941 -0.0313 0.35219 -0.1292 0.42638 -0.1767 -0.4606 -0.2097 -0.5795 -0.1722 -0.0057 -0.2700 -0.1600 0.10729 -0.0112 0.11266 -0.0216 -0.2959 0.18108 0.52252 0.11102 -0.1048 0.30753 0.04928 0.02842 0.03240 -0.0351 -0.0634 0.15984 0.05075 0.03805 -0.0602 0.02251 0.00826 -0.0347 0.03414 -0.0148 -0.0317 0.01265 -0.0190 -0.1014 0.06969 -0.0220 0.10607 0.08640 -0.0591 -0.0292 0.01709 0.40762 +-0.3581 0.09559 0.14066 -0.0260 0.19021 -0.4095 -0.0739 -0.3136 0.21179 0.06727 0.17819 0.43811 -0.3878 -0.9473 -0.0293 0.45205 0.40008 0.31029 0.44690 0.47500 -0.2835 -0.2209 0.20011 0.29832 -0.0685 0.33712 -0.2603 -0.2582 -0.2102 -0.5380 -0.2871 -0.0090 -0.1586 0.02215 0.12185 -0.0239 0.08344 -0.0624 -0.1077 0.28849 0.40304 -0.0137 -0.0072 0.20425 0.12103 0.00978 0.05600 -0.0483 -0.0487 0.16924 0.05829 0.03176 -0.0701 0.00289 -0.0044 -0.0341 0.03710 -0.0026 -0.0303 0.00739 -0.0253 -0.0957 0.08587 -0.0190 0.10540 0.08172 -0.0605 -0.0299 0.01642 0.41854 +-0.4225 0.12965 0.03897 0.01589 0.24236 -0.4255 -0.0963 -0.2367 0.23394 0.01231 0.21699 0.48498 -0.3338 -0.8253 0.01274 0.38308 0.40016 0.27198 0.44052 0.53003 -0.2419 -0.2622 0.14223 0.32425 -0.1038 0.34979 -0.2763 -0.2259 -0.1616 -0.5414 -0.2868 -0.0146 -0.1406 0.02883 0.17392 -0.0392 0.06992 -0.0655 -0.1002 0.28547 0.41914 -0.0170 -0.0223 0.18926 0.12551 0.00039 0.07072 -0.0534 -0.0405 0.17060 0.04391 0.02978 -0.0806 0.00272 1.04419 -0.0388 0.03589 -0.0049 -0.0363 0.00371 -0.0249 -0.0794 0.08316 -0.0219 0.10385 0.07715 -0.0620 -0.0304 0.01528 0.42814 +-0.4783 0.15308 0.05010 -0.0176 0.18882 -0.4830 -0.0989 -0.2023 0.25243 0.06362 0.19162 0.54515 -0.3493 -0.8389 0.02782 0.43614 0.37237 0.31601 0.49952 0.51812 -0.2676 -0.2382 0.18180 0.25788 -0.1608 0.34877 -0.2984 -0.2128 -0.1755 -0.5531 -0.2735 -0.0347 -0.1148 0.03342 0.14257 -0.0628 0.06493 -0.0569 -0.0814 0.28858 0.43346 -0.0101 -0.0126 0.19898 0.12125 0.00706 0.06652 -0.0415 -0.0531 0.17151 0.03505 0.03519 -0.0714 0.00933 -0.0048 -0.0392 0.03243 -0.0014 -0.0311 0.01056 -0.0199 -0.0916 0.08238 -0.0273 0.10376 0.07402 -0.0635 -0.0310 0.01547 0.41646 +-0.4563 0.14000 0.14083 -0.3368 -0.0328 -0.4942 -0.0877 -0.2403 0.23149 0.16122 0.19720 0.39101 -0.3921 -0.9350 -0.1446 0.37894 0.41363 0.32037 0.49027 0.42832 -0.2877 -0.2364 0.27003 0.22022 -0.1028 0.33178 -0.2906 -0.3172 -0.1934 -0.5576 -0.2526 -0.0208 -0.1526 0.01955 0.05092 -0.0428 0.09586 -0.0313 -0.0812 0.27748 0.48911 -0.0386 0.03964 0.23485 0.11428 0.01104 0.05791 -0.0432 -0.0461 0.18112 0.03760 0.02910 -0.0726 0.00333 -0.0004 -0.0416 0.03489 0.00478 -0.0427 0.00408 -0.0167 -0.0874 0.09070 -0.0220 0.10361 0.07263 -0.0645 -0.0314 0.01524 0.40917 +-0.4594 0.11133 0.18337 -0.3811 -0.0728 -0.4046 -0.0841 -0.3162 0.25487 0.12730 0.17880 0.33081 -0.4469 -1.1215 -0.2589 0.43138 0.41162 0.32260 0.50488 0.44638 -0.3375 -0.2245 0.31956 0.20983 -0.1122 0.37381 -0.3064 -0.2589 -0.1669 -0.5792 -0.2474 -0.0212 -0.1696 0.02370 -0.0283 -0.0090 0.08363 -0.0333 -0.0863 0.27673 0.51828 -0.0332 0.06728 0.24529 0.10693 0.00598 0.05948 -0.0285 -0.0422 0.17806 0.03383 0.03176 -0.0725 0.00894 0.00321 -0.0419 0.03130 -0.0019 -0.0411 0.00546 -0.0202 -0.0870 0.09073 -0.0121 0.10257 0.07138 -0.0654 -0.0318 0.01467 0.41111 +-0.3555 0.16566 0.22803 -0.3155 -0.0504 -0.3655 -0.1161 -0.3381 0.24321 0.07076 0.21549 0.32869 -0.4696 -1.1398 -0.2856 0.49630 0.42864 0.29360 0.51417 0.42584 -0.2489 -0.2632 0.32190 0.24460 -0.1038 0.34128 -0.2613 -0.3284 -0.2267 -0.5585 -0.2764 -0.0020 -0.1889 0.00871 0.05122 -0.0026 0.15784 -0.0276 -0.1270 0.26706 0.50542 -0.0274 0.04733 0.24377 0.10286 0.00383 0.05977 -0.0397 -0.0456 0.18132 0.04734 0.03545 -0.0678 0.00667 0.00354 -0.0308 0.02995 0.00067 -0.0368 0.00216 -0.0251 -0.0864 0.08764 -0.0212 0.09974 0.07198 -0.0661 -0.0318 0.01494 0.40155 +-0.5836 0.13965 0.15271 -0.4839 -0.1168 -0.5351 -0.0741 -0.3893 0.20829 0.18466 0.24978 0.32329 -0.4588 -1.1833 -0.3693 0.46472 0.39746 0.36441 0.50206 0.44685 -0.3596 -0.2557 0.32203 0.20187 -0.1614 0.35461 -0.3440 -0.2576 -0.1868 -0.5922 -0.2384 -0.0239 -0.1771 0.03282 -0.0014 0.02270 0.08217 -0.0555 -0.1101 0.24285 0.53143 -0.0286 0.04844 0.24929 0.10952 0.00147 0.06296 -0.0436 -0.0356 0.17862 0.04642 0.03259 -0.0762 0.00576 0.00311 -0.0325 0.03030 0.00136 -0.0332 -0.0010 -0.0191 -0.0763 0.08943 -0.0188 0.09705 0.07170 -0.0665 -0.0319 0.01557 0.38662 +-0.7530 0.07403 0.07600 -0.5537 -0.0919 -0.7144 -0.0806 -0.4135 0.17242 0.25093 0.36206 0.33543 -0.4540 -1.1444 -0.3464 0.40452 0.35195 0.33281 0.47009 0.42931 -0.3621 -0.1979 0.30691 0.24505 -0.1614 0.37129 -0.3332 -0.3287 -0.2149 -0.5864 -0.2192 -0.0346 -0.1611 0.02735 0.02796 0.01936 0.07613 -0.0449 -0.1176 0.25929 0.52870 -0.0184 0.02447 0.22805 0.11327 0.00121 0.06329 -0.0574 -0.0493 0.18553 0.03037 0.03257 -0.0678 0.00162 -0.0043 -0.0452 0.02206 -0.0027 -0.0377 0.00808 -0.0179 -0.0789 0.09224 -0.0257 0.09347 0.07203 -0.0665 -0.0322 0.01580 0.39027 +-0.6859 0.10302 0.13395 -0.5183 -0.0964 -0.6988 -0.0820 -0.3446 0.20964 0.22381 0.36345 0.30038 -0.4553 -1.0696 -0.2648 0.35938 0.39258 0.27711 0.45862 0.41651 -0.2540 -0.1900 0.29700 0.30027 -0.1712 0.36666 -0.2716 -0.4158 -0.2846 -0.5916 -0.2419 -0.0236 -0.1651 0.02844 0.06293 -0.0184 0.14057 -0.0371 -0.1206 0.23973 0.50854 -0.0200 0.01218 0.20658 0.10770 0.00614 0.06065 -0.0501 -0.0474 0.18296 0.03013 0.03393 -0.0677 0.00783 -0.0029 -0.0426 0.02455 -0.0033 -0.0362 0.00419 -0.0204 -0.0806 0.08954 -0.0224 0.09009 0.07080 -0.0658 -0.0325 0.01593 0.39041 +-0.5299 0.13469 0.21400 -0.4571 -0.0963 -0.5578 -0.1219 -0.2919 0.22945 0.12017 0.28310 0.33434 -0.4520 -1.0164 -0.2738 0.42595 0.42325 0.26848 0.51507 0.37212 -0.2233 -0.2226 0.38435 0.30014 -0.1472 0.36903 -0.2357 -0.4319 -0.2815 -0.5727 -0.2473 -0.0205 -0.1632 0.00514 0.06919 -0.0468 0.18699 -0.0384 -0.1430 0.24264 0.51531 -0.0325 0.02420 0.23887 0.11913 0.00451 0.06002 -0.0488 -0.0504 0.18034 0.03259 0.03077 -0.0674 0.00622 -0.0091 -0.0363 0.02833 -0.0011 -0.0388 0.00232 -0.0174 -0.0822 0.09064 -0.0245 0.08727 0.07078 -0.0648 -0.0326 0.01650 0.38614 +-0.3089 0.16623 0.29923 -0.2096 -0.0012 -0.3663 -0.1721 -0.2077 0.23252 0.03777 0.20670 0.43772 -0.3959 -1.1110 -0.1583 0.44718 0.52060 0.12639 0.56940 0.15131 -0.1717 -0.4930 0.05447 0.28041 -0.2214 0.46013 -0.2686 -0.4331 -0.1463 -0.5962 -0.1619 -0.0454 -0.2505 -0.1557 0.02592 -0.0608 0.07116 -0.0406 -0.2556 0.18235 0.56463 0.09362 -0.0839 0.31362 0.04933 0.02733 0.04123 -0.0305 -0.0620 0.16470 0.04558 0.03418 -0.0727 0.01835 0.00418 -0.0360 0.03119 -0.0123 -0.0369 0.00427 -0.0215 -0.0853 0.07543 -0.0241 0.08661 0.06962 -0.0636 -0.0330 0.01690 0.38438 +-0.1550 0.19755 0.34732 -0.0188 0.12854 -0.3465 -0.1531 -0.1926 0.19793 0.07795 0.18375 0.50808 -0.3223 -1.1399 -0.0953 0.51421 0.55044 0.03103 0.60401 0.01136 -0.1899 -0.6728 -0.2829 0.26002 -0.2176 0.50397 -0.2829 -0.4222 -0.0487 -0.5869 -0.0854 -0.0765 -0.3495 -0.2835 -0.0027 -0.0020 -0.0372 -0.0482 -0.3312 0.13921 0.59413 0.19105 -0.1760 0.37599 0.00394 0.04196 0.01875 -0.0064 -0.0592 0.14821 0.06329 0.03677 -0.0608 0.02870 0.01408 -0.0353 0.03202 -0.0219 -0.0353 0.01907 -0.0243 -0.1013 0.06324 -0.0220 0.08788 0.06939 -0.0627 -0.0333 0.01608 0.39311 +-0.1040 0.20479 0.40063 0.09289 0.18797 -0.3631 -0.1375 -0.1684 0.20020 0.10775 0.22523 0.48297 -0.3288 -1.1437 -0.0657 0.55286 0.53866 -0.0035 0.63353 -0.0225 -0.1992 -0.6825 -0.4034 0.27775 -0.1240 0.50517 -0.2239 -0.4461 -0.0525 -0.5645 -0.1053 -0.0608 -0.3706 -0.3198 0.03851 0.01778 -0.0354 -0.0212 -0.3607 0.10547 0.58399 0.24998 -0.2005 0.38615 -0.0087 0.04763 0.00760 0.01063 -0.0607 0.14066 0.07387 0.04144 -0.0602 0.03764 0.02167 -0.0288 0.03191 -0.0235 -0.0362 0.02471 -0.0204 -0.1079 0.05124 -0.0247 0.09064 0.06932 -0.0620 -0.0338 0.01529 0.40259 +-0.0847 0.19744 0.37175 0.11673 0.22084 -0.3196 -0.1292 -0.2022 0.18489 0.10327 0.18560 0.46434 -0.3461 -1.1362 -0.0917 0.56952 0.53134 0.02035 0.61496 -0.0017 -0.2412 -0.6283 -0.3611 0.29230 -0.0815 0.47632 -0.1973 -0.4197 -0.0523 -0.5414 -0.1379 -0.0360 -0.3608 -0.2809 0.05646 0.04623 -0.0323 -0.0224 -0.3209 0.13821 0.56315 0.22630 -0.1887 0.37886 0.00252 0.04703 0.00576 0.01064 -0.0581 0.14217 0.07355 0.04255 -0.0548 0.03511 0.02238 -0.0269 0.02976 -0.0219 -0.0352 0.02545 -0.0192 -0.1073 0.05033 -0.0267 0.09390 0.07177 -0.0619 -0.0339 0.01500 0.39155 +-0.0986 0.17240 0.31916 0.09821 0.20882 -0.2378 -0.0856 -0.2728 0.19964 0.06254 0.14257 0.41377 -0.3716 -1.1101 -0.0886 0.57543 0.48097 0.10660 0.55671 0.16890 -0.2873 -0.4461 -0.1163 0.30357 0.00194 0.39277 -0.1825 -0.3455 -0.1130 -0.5234 -0.2161 0.00488 -0.3045 -0.1397 0.08295 0.05645 0.03354 -0.0184 -0.2105 0.20301 0.51081 0.11716 -0.1005 0.31948 0.04859 0.03091 0.02366 -0.0076 -0.0455 0.15341 0.06558 0.04024 -0.0553 0.02412 0.01800 -0.0320 0.03233 -0.0120 -0.0339 0.01894 -0.0263 -0.1018 0.07232 -0.0265 0.09762 0.07370 -0.0618 -0.0339 0.01510 0.37065 +-0.1740 0.14410 0.21871 -0.0479 0.11039 -0.2121 -0.0573 -0.3401 0.22540 0.04205 0.14797 0.36761 -0.4145 -1.0528 -0.1198 0.57750 0.40221 0.29651 0.48140 0.42989 -0.3184 -0.2114 0.17003 0.25993 0.02934 0.30997 -0.2226 -0.2348 -0.1991 -0.5115 -0.3094 0.03278 -0.2052 0.02113 0.13140 0.06729 0.12695 -0.0173 -0.1154 0.26272 0.44053 -0.0141 0.02352 0.22052 0.10685 0.00831 0.05208 -0.0441 -0.0399 0.17432 0.05893 0.03615 -0.0578 0.00771 0.01157 -0.0316 0.03516 0.00041 -0.0323 0.00939 -0.0298 -0.0983 0.08738 -0.0261 0.10041 0.07689 -0.0623 -0.0336 0.01480 0.36796 +-0.2279 0.15222 0.21033 -0.1377 0.06937 -0.2746 -0.0601 -0.3575 0.21338 0.04875 0.14991 0.37188 -0.4267 -1.0639 -0.2038 0.57404 0.40192 0.31840 0.47144 0.46390 -0.3466 -0.2391 0.18205 0.23893 -0.0261 0.32561 -0.2993 -0.2058 -0.1889 -0.5238 -0.3074 0.01198 -0.1855 0.02514 0.10208 0.06317 0.10413 -0.0355 -0.1177 0.27259 0.46534 -0.0158 0.03168 0.22205 0.10819 0.01373 0.04866 -0.0494 -0.0473 0.17573 0.05691 0.03472 -0.0691 0.00642 0.00898 -0.0306 0.03707 -0.0029 -0.0339 0.00030 -0.0301 -0.0956 0.08655 -0.0235 0.10005 0.07828 -0.0622 -0.0332 0.01491 0.37261 +-0.4054 0.12408 0.19691 -0.3007 -0.0550 -0.4125 -0.0820 -0.3599 0.21574 0.05471 0.18093 0.30331 -0.4640 -1.1312 -0.2223 0.47925 0.42732 0.32031 0.48152 0.44341 -0.3239 -0.2461 0.28079 0.24663 -0.0847 0.35208 -0.2740 -0.2656 -0.1959 -0.5819 -0.2729 -0.0028 -0.1896 0.02676 0.06494 0.01558 0.13213 -0.0377 -0.1277 0.25954 0.50271 -0.0210 0.03802 0.24345 0.11007 0.00520 0.05983 -0.0616 -0.0494 0.18641 0.04899 0.03368 -0.0658 0.00119 0.00243 -0.0293 0.02526 -0.0001 -0.0311 0.00524 -0.0264 -0.0837 0.08040 -0.0264 0.09845 0.07954 -0.0621 -0.0328 0.01519 0.38120 +-0.4695 0.12556 0.17223 -0.3085 -0.0332 -0.4395 -0.1058 -0.3176 0.22436 0.09335 0.18300 0.32881 -0.4892 -1.0957 -0.1771 0.47449 0.41965 0.30093 0.49778 0.37645 -0.3120 -0.2362 0.32132 0.25848 -0.1242 0.3615 -0.2515 -0.3481 -0.2073 -0.5586 -0.2667 0.00066 -0.1823 0.01998 0.04488 -0.0171 0.13667 -0.0199 -0.1278 0.26467 0.51167 -0.0217 0.03637 0.24306 0.10794 0.00596 0.06184 -0.0629 -0.0430 0.17412 0.04070 0.03390 -0.0707 0.00744 -0.0027 -0.0380 0.02831 -0.0019 -0.0366 0.00255 -0.0191 -0.0827 0.08534 -0.0212 0.09561 0.08069 -0.0620 -0.0322 0.01566 0.39375 +-0.4561 0.14108 0.20322 -0.3100 -0.0351 -0.4242 -0.1126 -0.2852 0.23709 0.07616 0.15799 0.35321 -0.4462 -1.0350 -0.1432 0.44964 0.44364 0.27727 0.49717 0.35777 -0.2987 -0.2006 0.36354 0.26436 -0.1460 0.37256 -0.2713 -0.3506 -0.2250 -0.5760 -0.2680 -0.0128 -0.1685 0.01246 0.03600 -0.0526 0.13446 -0.0268 -0.1203 0.26123 0.52134 -0.0286 0.05162 0.24305 0.10072 0.00169 0.06769 -0.0452 -0.0387 0.17308 0.04159 0.03415 -0.0775 0.00776 -0.0016 -0.0377 0.02771 -0.0044 -0.0377 0.00277 -0.0223 -0.0811 0.08736 -0.0212 0.09153 0.08172 -0.0619 -0.0314 0.01629 0.41026 +-0.2915 0.16764 0.25559 -0.1779 0.02950 -0.3338 -0.1270 -0.2261 0.28260 0.04037 0.12508 0.40198 -0.3937 -0.9578 -0.1033 0.47747 0.43340 0.28516 0.49871 0.42647 -0.2622 -0.2200 0.31975 0.24898 -0.1818 0.36699 -0.3157 -0.3268 -0.2266 -0.5898 -0.2588 -0.0139 -0.1311 0.02101 0.06474 -0.0661 0.15628 -0.0196 -0.1309 0.25855 0.50692 -0.0472 0.05460 0.23717 0.10298 0.00187 0.06394 -0.0483 -0.0448 0.17836 0.03777 0.03076 -0.0754 0.00135 -0.0023 -0.0335 0.02424 -0.0058 -0.0420 0.00299 -0.0228 -0.0815 0.08905 -0.0245 0.08621 0.08263 -0.0618 -0.0306 0.01710 0.43071 +-0.2303 0.19406 0.24771 -0.1573 0.04175 -0.3110 -0.1174 -0.1987 0.30107 0.03073 0.13025 0.39950 -0.3776 -0.9211 -0.0519 0.48160 0.41014 0.28314 0.49382 0.43821 -0.2373 -0.2190 0.29889 0.20638 -0.2054 0.37286 -0.3512 -0.3133 -0.2339 -0.6017 -0.2527 -0.0310 -0.1120 0.02568 0.08815 -0.0791 0.15938 -0.0178 -0.1406 0.24988 0.52439 -0.0464 0.06063 0.23221 0.10133 0.00722 0.06689 -0.0504 -0.0469 0.18111 0.03700 0.03084 -0.0707 0.00381 -0.0007 -0.0306 0.02345 -0.0059 -0.0470 0.00316 -0.0238 -0.0828 0.08659 -0.0253 0.07965 0.08343 -0.0617 -0.0296 0.01808 0.45512 diff --git a/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100_enhanced.mp4 b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100_enhanced.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f71a5b785d0adde7cb6b7d1b32d1fb725e058c2e Binary files /dev/null and b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/image##nana_speech_cut_4sec 1-0-100_enhanced.mp4 differ diff --git a/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/input/image.png b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/input/image.png new file mode 100644 index 0000000000000000000000000000000000000000..9b3328e3e36cbf33024c9e17902fda8f627f0c0f Binary files /dev/null and b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/input/image.png differ diff --git a/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/input/nana_speech_cut_4sec 1-0-100.wav b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/input/nana_speech_cut_4sec 1-0-100.wav new file mode 100644 index 0000000000000000000000000000000000000000..b84d0ec2cae941fa9b4d828df76340818bda4d13 Binary files /dev/null and b/results/f9f9d11e-0b35-4b20-8e2b-cef839dc2817/input/nana_speech_cut_4sec 1-0-100.wav differ diff --git a/scripts/download_models.sh b/scripts/download_models.sh new file mode 100644 index 0000000000000000000000000000000000000000..6898648b153a2826557693dabb5adaf13bee2645 --- /dev/null +++ b/scripts/download_models.sh @@ -0,0 +1,32 @@ +mkdir ./checkpoints + +# lagency download link +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/auido2exp_00300-model.pth -O ./checkpoints/auido2exp_00300-model.pth +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/auido2pose_00140-model.pth -O ./checkpoints/auido2pose_00140-model.pth +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/epoch_20.pth -O ./checkpoints/epoch_20.pth +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/facevid2vid_00189-model.pth.tar -O ./checkpoints/facevid2vid_00189-model.pth.tar +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/shape_predictor_68_face_landmarks.dat -O ./checkpoints/shape_predictor_68_face_landmarks.dat +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/wav2lip.pth -O ./checkpoints/wav2lip.pth +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/mapping_00229-model.pth.tar -O ./checkpoints/mapping_00229-model.pth.tar +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/mapping_00109-model.pth.tar -O ./checkpoints/mapping_00109-model.pth.tar +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/hub.zip -O ./checkpoints/hub.zip +# unzip -n ./checkpoints/hub.zip -d ./checkpoints/ + + +#### download the new links. +wget -nc https://github.com/OpenTalker/SadTalker/releases/download/v0.0.2-rc/mapping_00109-model.pth.tar -O ./checkpoints/mapping_00109-model.pth.tar +wget -nc https://github.com/OpenTalker/SadTalker/releases/download/v0.0.2-rc/mapping_00229-model.pth.tar -O ./checkpoints/mapping_00229-model.pth.tar +wget -nc https://github.com/OpenTalker/SadTalker/releases/download/v0.0.2-rc/SadTalker_V0.0.2_256.safetensors -O ./checkpoints/SadTalker_V0.0.2_256.safetensors +wget -nc https://github.com/OpenTalker/SadTalker/releases/download/v0.0.2-rc/SadTalker_V0.0.2_512.safetensors -O ./checkpoints/SadTalker_V0.0.2_512.safetensors + + +# wget -nc https://github.com/Winfredy/SadTalker/releases/download/v0.0.2/BFM_Fitting.zip -O ./checkpoints/BFM_Fitting.zip +# unzip -n ./checkpoints/BFM_Fitting.zip -d ./checkpoints/ + +### enhancer +mkdir -p ./gfpgan/weights +wget -nc https://github.com/xinntao/facexlib/releases/download/v0.1.0/alignment_WFLW_4HG.pth -O ./gfpgan/weights/alignment_WFLW_4HG.pth +wget -nc https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_Resnet50_Final.pth -O ./gfpgan/weights/detection_Resnet50_Final.pth +wget -nc https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth -O ./gfpgan/weights/GFPGANv1.4.pth +wget -nc https://github.com/xinntao/facexlib/releases/download/v0.2.2/parsing_parsenet.pth -O ./gfpgan/weights/parsing_parsenet.pth + diff --git a/scripts/extension.py b/scripts/extension.py new file mode 100644 index 0000000000000000000000000000000000000000..c90ec25c2811d87a00a2e2a14e270c75d07d713d --- /dev/null +++ b/scripts/extension.py @@ -0,0 +1,189 @@ +import os, sys +from pathlib import Path +import tempfile +import gradio as gr +from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call +from modules.shared import opts, OptionInfo +from modules import shared, paths, script_callbacks +import launch +import glob +from huggingface_hub import snapshot_download + + + +def check_all_files_safetensor(current_dir): + kv = { + "SadTalker_V0.0.2_256.safetensors": "sadtalker-256", + "SadTalker_V0.0.2_512.safetensors": "sadtalker-512", + "mapping_00109-model.pth.tar" : "mapping-109" , + "mapping_00229-model.pth.tar" : "mapping-229" , + } + + if not os.path.isdir(current_dir): + return False + + dirs = os.listdir(current_dir) + + for f in dirs: + if f in kv.keys(): + del kv[f] + + return len(kv.keys()) == 0 + +def check_all_files(current_dir): + kv = { + "auido2exp_00300-model.pth": "audio2exp", + "auido2pose_00140-model.pth": "audio2pose", + "epoch_20.pth": "face_recon", + "facevid2vid_00189-model.pth.tar": "face-render", + "mapping_00109-model.pth.tar" : "mapping-109" , + "mapping_00229-model.pth.tar" : "mapping-229" , + "wav2lip.pth": "wav2lip", + "shape_predictor_68_face_landmarks.dat": "dlib", + } + + if not os.path.isdir(current_dir): + return False + + dirs = os.listdir(current_dir) + + for f in dirs: + if f in kv.keys(): + del kv[f] + + return len(kv.keys()) == 0 + + + +def download_model(local_dir='./checkpoints'): + REPO_ID = 'vinthony/SadTalker' + snapshot_download(repo_id=REPO_ID, local_dir=local_dir, local_dir_use_symlinks=False) + +def get_source_image(image): + return image + +def get_img_from_txt2img(x): + talker_path = Path(paths.script_path) / "outputs" + imgs_from_txt_dir = str(talker_path / "txt2img-images/") + imgs = glob.glob(imgs_from_txt_dir+'/*/*.png') + imgs.sort(key=lambda x:os.path.getmtime(os.path.join(imgs_from_txt_dir, x))) + img_from_txt_path = os.path.join(imgs_from_txt_dir, imgs[-1]) + return img_from_txt_path, img_from_txt_path + +def get_img_from_img2img(x): + talker_path = Path(paths.script_path) / "outputs" + imgs_from_img_dir = str(talker_path / "img2img-images/") + imgs = glob.glob(imgs_from_img_dir+'/*/*.png') + imgs.sort(key=lambda x:os.path.getmtime(os.path.join(imgs_from_img_dir, x))) + img_from_img_path = os.path.join(imgs_from_img_dir, imgs[-1]) + return img_from_img_path, img_from_img_path + +def get_default_checkpoint_path(): + # check the path of models/checkpoints and extensions/ + checkpoint_path = Path(paths.script_path) / "models"/ "SadTalker" + extension_checkpoint_path = Path(paths.script_path) / "extensions"/ "SadTalker" / "checkpoints" + + if check_all_files_safetensor(checkpoint_path): + # print('founding sadtalker checkpoint in ' + str(checkpoint_path)) + return checkpoint_path + + if check_all_files_safetensor(extension_checkpoint_path): + # print('founding sadtalker checkpoint in ' + str(extension_checkpoint_path)) + return extension_checkpoint_path + + if check_all_files(checkpoint_path): + # print('founding sadtalker checkpoint in ' + str(checkpoint_path)) + return checkpoint_path + + if check_all_files(extension_checkpoint_path): + # print('founding sadtalker checkpoint in ' + str(extension_checkpoint_path)) + return extension_checkpoint_path + + return None + + + +def install(): + + kv = { + "face_alignment": "face-alignment==1.3.5", + "imageio": "imageio==2.19.3", + "imageio_ffmpeg": "imageio-ffmpeg==0.4.7", + "librosa":"librosa==0.8.0", + "pydub":"pydub==0.25.1", + "scipy":"scipy==1.8.1", + "tqdm": "tqdm", + "yacs":"yacs==0.1.8", + "yaml": "pyyaml", + "av":"av", + "gfpgan": "gfpgan", + } + + # # dlib is not necessary currently + # if 'darwin' in sys.platform: + # kv['dlib'] = "dlib" + # else: + # kv['dlib'] = 'dlib-bin' + + # #### we need to have a newer version of imageio for our method. + # launch.run_pip("install imageio==2.19.3", "requirements for SadTalker") + + for k,v in kv.items(): + if not launch.is_installed(k): + print(k, launch.is_installed(k)) + launch.run_pip("install "+ v, "requirements for SadTalker") + + if os.getenv('SADTALKER_CHECKPOINTS'): + print('load Sadtalker Checkpoints from '+ os.getenv('SADTALKER_CHECKPOINTS')) + + elif get_default_checkpoint_path() is not None: + os.environ['SADTALKER_CHECKPOINTS'] = str(get_default_checkpoint_path()) + else: + + print( + """" + SadTalker will not support download all the files from hugging face, which will take a long time. + + please manually set the SADTALKER_CHECKPOINTS in `webui_user.bat`(windows) or `webui_user.sh`(linux) + """ + ) + + # python = sys.executable + + # launch.run(f'"{python}" -m pip uninstall -y huggingface_hub', live=True) + # launch.run(f'"{python}" -m pip install --upgrade git+https://github.com/huggingface/huggingface_hub@main', live=True) + # ### run the scripts to downlod models to correct localtion. + # # print('download models for SadTalker') + # # launch.run("cd " + paths.script_path+"/extensions/SadTalker && bash ./scripts/download_models.sh", live=True) + # # print('SadTalker is successfully installed!') + # download_model(paths.script_path+'/extensions/SadTalker/checkpoints') + + +def on_ui_tabs(): + install() + + sys.path.extend([paths.script_path+'/extensions/SadTalker']) + + repo_dir = paths.script_path+'/extensions/SadTalker/' + + result_dir = opts.sadtalker_result_dir + os.makedirs(result_dir, exist_ok=True) + + from app_sadtalker import sadtalker_demo + + if os.getenv('SADTALKER_CHECKPOINTS'): + checkpoint_path = os.getenv('SADTALKER_CHECKPOINTS') + else: + checkpoint_path = repo_dir+'checkpoints/' + + audio_to_video = sadtalker_demo(checkpoint_path=checkpoint_path, config_path=repo_dir+'src/config', warpfn = wrap_queued_call) + + return [(audio_to_video, "SadTalker", "extension")] + +def on_ui_settings(): + talker_path = Path(paths.script_path) / "outputs" + section = ('extension', "SadTalker") + opts.add_option("sadtalker_result_dir", OptionInfo(str(talker_path / "SadTalker/"), "Path to save results of sadtalker", section=section)) + +script_callbacks.on_ui_settings(on_ui_settings) +script_callbacks.on_ui_tabs(on_ui_tabs) diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..bcfecfde94951c8feec231c14c30a685674a284a --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,21 @@ +# ### some test command before commit. +# python inference.py --preprocess crop --size 256 +# python inference.py --preprocess crop --size 512 + +# python inference.py --preprocess extcrop --size 256 +# python inference.py --preprocess extcrop --size 512 + +# python inference.py --preprocess resize --size 256 +# python inference.py --preprocess resize --size 512 + +# python inference.py --preprocess full --size 256 +# python inference.py --preprocess full --size 512 + +# python inference.py --preprocess extfull --size 256 +# python inference.py --preprocess extfull --size 512 + +python inference.py --preprocess full --size 256 --enhancer gfpgan +python inference.py --preprocess full --size 512 --enhancer gfpgan + +python inference.py --preprocess full --size 256 --enhancer gfpgan --still +python inference.py --preprocess full --size 512 --enhancer gfpgan --still diff --git a/speaker_encoder/__init__.py b/speaker_encoder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/speaker_encoder/__init__.py @@ -0,0 +1 @@ + diff --git a/speaker_encoder/__pycache__/__init__.cpython-310.pyc b/speaker_encoder/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1336b8b5ef290c79d32788d8dd0014a3f74bbe74 Binary files /dev/null and b/speaker_encoder/__pycache__/__init__.cpython-310.pyc differ diff --git a/speaker_encoder/__pycache__/audio.cpython-310.pyc b/speaker_encoder/__pycache__/audio.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1edcc0311aeee2ed9348bf8a80c78abfe433c62 Binary files /dev/null and b/speaker_encoder/__pycache__/audio.cpython-310.pyc differ diff --git a/speaker_encoder/__pycache__/hparams.cpython-310.pyc b/speaker_encoder/__pycache__/hparams.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1438b3c864b16eedb06960a789f934cc1b526f0c Binary files /dev/null and b/speaker_encoder/__pycache__/hparams.cpython-310.pyc differ diff --git a/speaker_encoder/__pycache__/params_data.cpython-310.pyc b/speaker_encoder/__pycache__/params_data.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88cac9392f5c6bf9f1cfce138fb92a3f7f0371dd Binary files /dev/null and b/speaker_encoder/__pycache__/params_data.cpython-310.pyc differ diff --git a/speaker_encoder/__pycache__/voice_encoder.cpython-310.pyc b/speaker_encoder/__pycache__/voice_encoder.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d71825c9fb4603b9b2a2a876b777bf706df61113 Binary files /dev/null and b/speaker_encoder/__pycache__/voice_encoder.cpython-310.pyc differ diff --git a/speaker_encoder/audio.py b/speaker_encoder/audio.py new file mode 100644 index 0000000000000000000000000000000000000000..2fcb77ad1d3a85f523e24f84691886736a5686cb --- /dev/null +++ b/speaker_encoder/audio.py @@ -0,0 +1,107 @@ +from scipy.ndimage.morphology import binary_dilation +from speaker_encoder.params_data import * +from pathlib import Path +from typing import Optional, Union +import numpy as np +import webrtcvad +import librosa +import struct + +int16_max = (2 ** 15) - 1 + + +def preprocess_wav(fpath_or_wav: Union[str, Path, np.ndarray], + source_sr: Optional[int] = None): + """ + Applies the preprocessing operations used in training the Speaker Encoder to a waveform + either on disk or in memory. The waveform will be resampled to match the data hyperparameters. + + :param fpath_or_wav: either a filepath to an audio file (many extensions are supported, not + just .wav), either the waveform as a numpy array of floats. + :param source_sr: if passing an audio waveform, the sampling rate of the waveform before + preprocessing. After preprocessing, the waveform's sampling rate will match the data + hyperparameters. If passing a filepath, the sampling rate will be automatically detected and + this argument will be ignored. + """ + # Load the wav from disk if needed + if isinstance(fpath_or_wav, str) or isinstance(fpath_or_wav, Path): + wav, source_sr = librosa.load(fpath_or_wav, sr=None) + else: + wav = fpath_or_wav + + # Resample the wav if needed + if source_sr is not None and source_sr != sampling_rate: + wav = librosa.resample(wav, source_sr, sampling_rate) + + # Apply the preprocessing: normalize volume and shorten long silences + wav = normalize_volume(wav, audio_norm_target_dBFS, increase_only=True) + wav = trim_long_silences(wav) + + return wav + + +def wav_to_mel_spectrogram(wav): + """ + Derives a mel spectrogram ready to be used by the encoder from a preprocessed audio waveform. + Note: this not a log-mel spectrogram. + """ + frames = librosa.feature.melspectrogram( + y=wav, + sr=sampling_rate, + n_fft=int(sampling_rate * mel_window_length / 1000), + hop_length=int(sampling_rate * mel_window_step / 1000), + n_mels=mel_n_channels + ) + return frames.astype(np.float32).T + + +def trim_long_silences(wav): + """ + Ensures that segments without voice in the waveform remain no longer than a + threshold determined by the VAD parameters in params.py. + + :param wav: the raw waveform as a numpy array of floats + :return: the same waveform with silences trimmed away (length <= original wav length) + """ + # Compute the voice detection window size + samples_per_window = (vad_window_length * sampling_rate) // 1000 + + # Trim the end of the audio to have a multiple of the window size + wav = wav[:len(wav) - (len(wav) % samples_per_window)] + + # Convert the float waveform to 16-bit mono PCM + pcm_wave = struct.pack("%dh" % len(wav), *(np.round(wav * int16_max)).astype(np.int16)) + + # Perform voice activation detection + voice_flags = [] + vad = webrtcvad.Vad(mode=3) + for window_start in range(0, len(wav), samples_per_window): + window_end = window_start + samples_per_window + voice_flags.append(vad.is_speech(pcm_wave[window_start * 2:window_end * 2], + sample_rate=sampling_rate)) + voice_flags = np.array(voice_flags) + + # Smooth the voice detection with a moving average + def moving_average(array, width): + array_padded = np.concatenate((np.zeros((width - 1) // 2), array, np.zeros(width // 2))) + ret = np.cumsum(array_padded, dtype=float) + ret[width:] = ret[width:] - ret[:-width] + return ret[width - 1:] / width + + audio_mask = moving_average(voice_flags, vad_moving_average_width) + audio_mask = np.round(audio_mask).astype(np.bool) + + # Dilate the voiced regions + audio_mask = binary_dilation(audio_mask, np.ones(vad_max_silence_length + 1)) + audio_mask = np.repeat(audio_mask, samples_per_window) + + return wav[audio_mask == True] + + +def normalize_volume(wav, target_dBFS, increase_only=False, decrease_only=False): + if increase_only and decrease_only: + raise ValueError("Both increase only and decrease only are set") + dBFS_change = target_dBFS - 10 * np.log10(np.mean(wav ** 2)) + if (dBFS_change < 0 and increase_only) or (dBFS_change > 0 and decrease_only): + return wav + return wav * (10 ** (dBFS_change / 20)) diff --git a/speaker_encoder/ckpt/__init__.py b/speaker_encoder/ckpt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/speaker_encoder/ckpt/__init__.py @@ -0,0 +1 @@ + diff --git a/speaker_encoder/ckpt/pretrained_bak_5805000.pt b/speaker_encoder/ckpt/pretrained_bak_5805000.pt new file mode 100644 index 0000000000000000000000000000000000000000..662d22b686114b4b6124330a688007d9495d22c8 --- /dev/null +++ b/speaker_encoder/ckpt/pretrained_bak_5805000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc7ff82ef75becd495aab2ede3a8220da393a717f178ae9534df355a6173bbca +size 17090379 diff --git a/speaker_encoder/compute_embed.py b/speaker_encoder/compute_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..2fee33db0168f40efc42145c06fa62016e3e008e --- /dev/null +++ b/speaker_encoder/compute_embed.py @@ -0,0 +1,40 @@ +from speaker_encoder import inference as encoder +from multiprocessing.pool import Pool +from functools import partial +from pathlib import Path +# from utils import logmmse +# from tqdm import tqdm +# import numpy as np +# import librosa + + +def embed_utterance(fpaths, encoder_model_fpath): + if not encoder.is_loaded(): + encoder.load_model(encoder_model_fpath) + + # Compute the speaker embedding of the utterance + wav_fpath, embed_fpath = fpaths + wav = np.load(wav_fpath) + wav = encoder.preprocess_wav(wav) + embed = encoder.embed_utterance(wav) + np.save(embed_fpath, embed, allow_pickle=False) + + +def create_embeddings(outdir_root: Path, wav_dir: Path, encoder_model_fpath: Path, n_processes: int): + + wav_dir = outdir_root.joinpath("audio") + metadata_fpath = synthesizer_root.joinpath("train.txt") + assert wav_dir.exists() and metadata_fpath.exists() + embed_dir = synthesizer_root.joinpath("embeds") + embed_dir.mkdir(exist_ok=True) + + # Gather the input wave filepath and the target output embed filepath + with metadata_fpath.open("r") as metadata_file: + metadata = [line.split("|") for line in metadata_file] + fpaths = [(wav_dir.joinpath(m[0]), embed_dir.joinpath(m[2])) for m in metadata] + + # TODO: improve on the multiprocessing, it's terrible. Disk I/O is the bottleneck here. + # Embed the utterances in separate threads + func = partial(embed_utterance, encoder_model_fpath=encoder_model_fpath) + job = Pool(n_processes).imap(func, fpaths) + list(tqdm(job, "Embedding", len(fpaths), unit="utterances")) \ No newline at end of file diff --git a/speaker_encoder/config.py b/speaker_encoder/config.py new file mode 100644 index 0000000000000000000000000000000000000000..1c21312f3de971bfa008254c6035cebc09f05e4c --- /dev/null +++ b/speaker_encoder/config.py @@ -0,0 +1,45 @@ +librispeech_datasets = { + "train": { + "clean": ["LibriSpeech/train-clean-100", "LibriSpeech/train-clean-360"], + "other": ["LibriSpeech/train-other-500"] + }, + "test": { + "clean": ["LibriSpeech/test-clean"], + "other": ["LibriSpeech/test-other"] + }, + "dev": { + "clean": ["LibriSpeech/dev-clean"], + "other": ["LibriSpeech/dev-other"] + }, +} +libritts_datasets = { + "train": { + "clean": ["LibriTTS/train-clean-100", "LibriTTS/train-clean-360"], + "other": ["LibriTTS/train-other-500"] + }, + "test": { + "clean": ["LibriTTS/test-clean"], + "other": ["LibriTTS/test-other"] + }, + "dev": { + "clean": ["LibriTTS/dev-clean"], + "other": ["LibriTTS/dev-other"] + }, +} +voxceleb_datasets = { + "voxceleb1" : { + "train": ["VoxCeleb1/wav"], + "test": ["VoxCeleb1/test_wav"] + }, + "voxceleb2" : { + "train": ["VoxCeleb2/dev/aac"], + "test": ["VoxCeleb2/test_wav"] + } +} + +other_datasets = [ + "LJSpeech-1.1", + "VCTK-Corpus/wav48", +] + +anglophone_nationalites = ["australia", "canada", "ireland", "uk", "usa"] diff --git a/speaker_encoder/data_objects/__init__.py b/speaker_encoder/data_objects/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..030317a1d9a328d452bf29bc7a802e29629b1a42 --- /dev/null +++ b/speaker_encoder/data_objects/__init__.py @@ -0,0 +1,2 @@ +from speaker_encoder.data_objects.speaker_verification_dataset import SpeakerVerificationDataset +from speaker_encoder.data_objects.speaker_verification_dataset import SpeakerVerificationDataLoader diff --git a/speaker_encoder/data_objects/random_cycler.py b/speaker_encoder/data_objects/random_cycler.py new file mode 100644 index 0000000000000000000000000000000000000000..c405db6b27f46d874d8feb37e3f9c1e12c251109 --- /dev/null +++ b/speaker_encoder/data_objects/random_cycler.py @@ -0,0 +1,37 @@ +import random + +class RandomCycler: + """ + Creates an internal copy of a sequence and allows access to its items in a constrained random + order. For a source sequence of n items and one or several consecutive queries of a total + of m items, the following guarantees hold (one implies the other): + - Each item will be returned between m // n and ((m - 1) // n) + 1 times. + - Between two appearances of the same item, there may be at most 2 * (n - 1) other items. + """ + + def __init__(self, source): + if len(source) == 0: + raise Exception("Can't create RandomCycler from an empty collection") + self.all_items = list(source) + self.next_items = [] + + def sample(self, count: int): + shuffle = lambda l: random.sample(l, len(l)) + + out = [] + while count > 0: + if count >= len(self.all_items): + out.extend(shuffle(list(self.all_items))) + count -= len(self.all_items) + continue + n = min(count, len(self.next_items)) + out.extend(self.next_items[:n]) + count -= n + self.next_items = self.next_items[n:] + if len(self.next_items) == 0: + self.next_items = shuffle(list(self.all_items)) + return out + + def __next__(self): + return self.sample(1)[0] + diff --git a/speaker_encoder/data_objects/speaker.py b/speaker_encoder/data_objects/speaker.py new file mode 100644 index 0000000000000000000000000000000000000000..07379847a854d85623db02ce5e5409c1566eb80c --- /dev/null +++ b/speaker_encoder/data_objects/speaker.py @@ -0,0 +1,40 @@ +from speaker_encoder.data_objects.random_cycler import RandomCycler +from speaker_encoder.data_objects.utterance import Utterance +from pathlib import Path + +# Contains the set of utterances of a single speaker +class Speaker: + def __init__(self, root: Path): + self.root = root + self.name = root.name + self.utterances = None + self.utterance_cycler = None + + def _load_utterances(self): + with self.root.joinpath("_sources.txt").open("r") as sources_file: + sources = [l.split(",") for l in sources_file] + sources = {frames_fname: wave_fpath for frames_fname, wave_fpath in sources} + self.utterances = [Utterance(self.root.joinpath(f), w) for f, w in sources.items()] + self.utterance_cycler = RandomCycler(self.utterances) + + def random_partial(self, count, n_frames): + """ + Samples a batch of unique partial utterances from the disk in a way that all + utterances come up at least once every two cycles and in a random order every time. + + :param count: The number of partial utterances to sample from the set of utterances from + that speaker. Utterances are guaranteed not to be repeated if is not larger than + the number of utterances available. + :param n_frames: The number of frames in the partial utterance. + :return: A list of tuples (utterance, frames, range) where utterance is an Utterance, + frames are the frames of the partial utterances and range is the range of the partial + utterance with regard to the complete utterance. + """ + if self.utterances is None: + self._load_utterances() + + utterances = self.utterance_cycler.sample(count) + + a = [(u,) + u.random_partial(n_frames) for u in utterances] + + return a diff --git a/speaker_encoder/data_objects/speaker_batch.py b/speaker_encoder/data_objects/speaker_batch.py new file mode 100644 index 0000000000000000000000000000000000000000..4485605e3ece5b491d1e7d0f223c543b6c91eb96 --- /dev/null +++ b/speaker_encoder/data_objects/speaker_batch.py @@ -0,0 +1,12 @@ +import numpy as np +from typing import List +from speaker_encoder.data_objects.speaker import Speaker + +class SpeakerBatch: + def __init__(self, speakers: List[Speaker], utterances_per_speaker: int, n_frames: int): + self.speakers = speakers + self.partials = {s: s.random_partial(utterances_per_speaker, n_frames) for s in speakers} + + # Array of shape (n_speakers * n_utterances, n_frames, mel_n), e.g. for 3 speakers with + # 4 utterances each of 160 frames of 40 mel coefficients: (12, 160, 40) + self.data = np.array([frames for s in speakers for _, frames, _ in self.partials[s]]) diff --git a/speaker_encoder/data_objects/speaker_verification_dataset.py b/speaker_encoder/data_objects/speaker_verification_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..cecd8ed8ac100b80d5087fa47f22f92c84fea032 --- /dev/null +++ b/speaker_encoder/data_objects/speaker_verification_dataset.py @@ -0,0 +1,56 @@ +from speaker_encoder.data_objects.random_cycler import RandomCycler +from speaker_encoder.data_objects.speaker_batch import SpeakerBatch +from speaker_encoder.data_objects.speaker import Speaker +from speaker_encoder.params_data import partials_n_frames +from torch.utils.data import Dataset, DataLoader +from pathlib import Path + +# TODO: improve with a pool of speakers for data efficiency + +class SpeakerVerificationDataset(Dataset): + def __init__(self, datasets_root: Path): + self.root = datasets_root + speaker_dirs = [f for f in self.root.glob("*") if f.is_dir()] + if len(speaker_dirs) == 0: + raise Exception("No speakers found. Make sure you are pointing to the directory " + "containing all preprocessed speaker directories.") + self.speakers = [Speaker(speaker_dir) for speaker_dir in speaker_dirs] + self.speaker_cycler = RandomCycler(self.speakers) + + def __len__(self): + return int(1e10) + + def __getitem__(self, index): + return next(self.speaker_cycler) + + def get_logs(self): + log_string = "" + for log_fpath in self.root.glob("*.txt"): + with log_fpath.open("r") as log_file: + log_string += "".join(log_file.readlines()) + return log_string + + +class SpeakerVerificationDataLoader(DataLoader): + def __init__(self, dataset, speakers_per_batch, utterances_per_speaker, sampler=None, + batch_sampler=None, num_workers=0, pin_memory=False, timeout=0, + worker_init_fn=None): + self.utterances_per_speaker = utterances_per_speaker + + super().__init__( + dataset=dataset, + batch_size=speakers_per_batch, + shuffle=False, + sampler=sampler, + batch_sampler=batch_sampler, + num_workers=num_workers, + collate_fn=self.collate, + pin_memory=pin_memory, + drop_last=False, + timeout=timeout, + worker_init_fn=worker_init_fn + ) + + def collate(self, speakers): + return SpeakerBatch(speakers, self.utterances_per_speaker, partials_n_frames) + \ No newline at end of file diff --git a/speaker_encoder/data_objects/utterance.py b/speaker_encoder/data_objects/utterance.py new file mode 100644 index 0000000000000000000000000000000000000000..0768c3420f422a7464f305b4c1fb6752c57ceda7 --- /dev/null +++ b/speaker_encoder/data_objects/utterance.py @@ -0,0 +1,26 @@ +import numpy as np + + +class Utterance: + def __init__(self, frames_fpath, wave_fpath): + self.frames_fpath = frames_fpath + self.wave_fpath = wave_fpath + + def get_frames(self): + return np.load(self.frames_fpath) + + def random_partial(self, n_frames): + """ + Crops the frames into a partial utterance of n_frames + + :param n_frames: The number of frames of the partial utterance + :return: the partial utterance frames and a tuple indicating the start and end of the + partial utterance in the complete utterance. + """ + frames = self.get_frames() + if frames.shape[0] == n_frames: + start = 0 + else: + start = np.random.randint(0, frames.shape[0] - n_frames) + end = start + n_frames + return frames[start:end], (start, end) \ No newline at end of file diff --git a/speaker_encoder/hparams.py b/speaker_encoder/hparams.py new file mode 100644 index 0000000000000000000000000000000000000000..9a8c16471903b0c92253b1d70fcd6a61d10e085f --- /dev/null +++ b/speaker_encoder/hparams.py @@ -0,0 +1,31 @@ +## Mel-filterbank +mel_window_length = 25 # In milliseconds +mel_window_step = 10 # In milliseconds +mel_n_channels = 40 + + +## Audio +sampling_rate = 16000 +# Number of spectrogram frames in a partial utterance +partials_n_frames = 160 # 1600 ms + + +## Voice Activation Detection +# Window size of the VAD. Must be either 10, 20 or 30 milliseconds. +# This sets the granularity of the VAD. Should not need to be changed. +vad_window_length = 30 # In milliseconds +# Number of frames to average together when performing the moving average smoothing. +# The larger this value, the larger the VAD variations must be to not get smoothed out. +vad_moving_average_width = 8 +# Maximum number of consecutive silent frames a segment can have. +vad_max_silence_length = 6 + + +## Audio volume normalization +audio_norm_target_dBFS = -30 + + +## Model parameters +model_hidden_size = 256 +model_embedding_size = 256 +model_num_layers = 3 \ No newline at end of file diff --git a/speaker_encoder/inference.py b/speaker_encoder/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..15e6bf16ba9e551473cd6b179bb518f0704ac33d --- /dev/null +++ b/speaker_encoder/inference.py @@ -0,0 +1,177 @@ +from speaker_encoder.params_data import * +from speaker_encoder.model import SpeakerEncoder +from speaker_encoder.audio import preprocess_wav # We want to expose this function from here +from matplotlib import cm +from speaker_encoder import audio +from pathlib import Path +import matplotlib.pyplot as plt +import numpy as np +import torch + +_model = None # type: SpeakerEncoder +_device = None # type: torch.device + + +def load_model(weights_fpath: Path, device=None): + """ + Loads the model in memory. If this function is not explicitely called, it will be run on the + first call to embed_frames() with the default weights file. + + :param weights_fpath: the path to saved model weights. + :param device: either a torch device or the name of a torch device (e.g. "cpu", "cuda"). The + model will be loaded and will run on this device. Outputs will however always be on the cpu. + If None, will default to your GPU if it"s available, otherwise your CPU. + """ + # TODO: I think the slow loading of the encoder might have something to do with the device it + # was saved on. Worth investigating. + global _model, _device + if device is None: + _device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + elif isinstance(device, str): + _device = torch.device(device) + _model = SpeakerEncoder(_device, torch.device("cpu")) + checkpoint = torch.load(weights_fpath) + _model.load_state_dict(checkpoint["model_state"]) + _model.eval() + print("Loaded encoder \"%s\" trained to step %d" % (weights_fpath.name, checkpoint["step"])) + + +def is_loaded(): + return _model is not None + + +def embed_frames_batch(frames_batch): + """ + Computes embeddings for a batch of mel spectrogram. + + :param frames_batch: a batch mel of spectrogram as a numpy array of float32 of shape + (batch_size, n_frames, n_channels) + :return: the embeddings as a numpy array of float32 of shape (batch_size, model_embedding_size) + """ + if _model is None: + raise Exception("Model was not loaded. Call load_model() before inference.") + + frames = torch.from_numpy(frames_batch).to(_device) + embed = _model.forward(frames).detach().cpu().numpy() + return embed + + +def compute_partial_slices(n_samples, partial_utterance_n_frames=partials_n_frames, + min_pad_coverage=0.75, overlap=0.5): + """ + Computes where to split an utterance waveform and its corresponding mel spectrogram to obtain + partial utterances of each. Both the waveform and the mel + spectrogram slices are returned, so as to make each partial utterance waveform correspond to + its spectrogram. This function assumes that the mel spectrogram parameters used are those + defined in params_data.py. + + The returned ranges may be indexing further than the length of the waveform. It is + recommended that you pad the waveform with zeros up to wave_slices[-1].stop. + + :param n_samples: the number of samples in the waveform + :param partial_utterance_n_frames: the number of mel spectrogram frames in each partial + utterance + :param min_pad_coverage: when reaching the last partial utterance, it may or may not have + enough frames. If at least of are present, + then the last partial utterance will be considered, as if we padded the audio. Otherwise, + it will be discarded, as if we trimmed the audio. If there aren't enough frames for 1 partial + utterance, this parameter is ignored so that the function always returns at least 1 slice. + :param overlap: by how much the partial utterance should overlap. If set to 0, the partial + utterances are entirely disjoint. + :return: the waveform slices and mel spectrogram slices as lists of array slices. Index + respectively the waveform and the mel spectrogram with these slices to obtain the partial + utterances. + """ + assert 0 <= overlap < 1 + assert 0 < min_pad_coverage <= 1 + + samples_per_frame = int((sampling_rate * mel_window_step / 1000)) + n_frames = int(np.ceil((n_samples + 1) / samples_per_frame)) + frame_step = max(int(np.round(partial_utterance_n_frames * (1 - overlap))), 1) + + # Compute the slices + wav_slices, mel_slices = [], [] + steps = max(1, n_frames - partial_utterance_n_frames + frame_step + 1) + for i in range(0, steps, frame_step): + mel_range = np.array([i, i + partial_utterance_n_frames]) + wav_range = mel_range * samples_per_frame + mel_slices.append(slice(*mel_range)) + wav_slices.append(slice(*wav_range)) + + # Evaluate whether extra padding is warranted or not + last_wav_range = wav_slices[-1] + coverage = (n_samples - last_wav_range.start) / (last_wav_range.stop - last_wav_range.start) + if coverage < min_pad_coverage and len(mel_slices) > 1: + mel_slices = mel_slices[:-1] + wav_slices = wav_slices[:-1] + + return wav_slices, mel_slices + + +def embed_utterance(wav, using_partials=True, return_partials=False, **kwargs): + """ + Computes an embedding for a single utterance. + + # TODO: handle multiple wavs to benefit from batching on GPU + :param wav: a preprocessed (see audio.py) utterance waveform as a numpy array of float32 + :param using_partials: if True, then the utterance is split in partial utterances of + frames and the utterance embedding is computed from their + normalized average. If False, the utterance is instead computed from feeding the entire + spectogram to the network. + :param return_partials: if True, the partial embeddings will also be returned along with the + wav slices that correspond to the partial embeddings. + :param kwargs: additional arguments to compute_partial_splits() + :return: the embedding as a numpy array of float32 of shape (model_embedding_size,). If + is True, the partial utterances as a numpy array of float32 of shape + (n_partials, model_embedding_size) and the wav partials as a list of slices will also be + returned. If is simultaneously set to False, both these values will be None + instead. + """ + # Process the entire utterance if not using partials + if not using_partials: + frames = audio.wav_to_mel_spectrogram(wav) + embed = embed_frames_batch(frames[None, ...])[0] + if return_partials: + return embed, None, None + return embed + + # Compute where to split the utterance into partials and pad if necessary + wave_slices, mel_slices = compute_partial_slices(len(wav), **kwargs) + max_wave_length = wave_slices[-1].stop + if max_wave_length >= len(wav): + wav = np.pad(wav, (0, max_wave_length - len(wav)), "constant") + + # Split the utterance into partials + frames = audio.wav_to_mel_spectrogram(wav) + frames_batch = np.array([frames[s] for s in mel_slices]) + partial_embeds = embed_frames_batch(frames_batch) + + # Compute the utterance embedding from the partial embeddings + raw_embed = np.mean(partial_embeds, axis=0) + embed = raw_embed / np.linalg.norm(raw_embed, 2) + + if return_partials: + return embed, partial_embeds, wave_slices + return embed + + +def embed_speaker(wavs, **kwargs): + raise NotImplemented() + + +def plot_embedding_as_heatmap(embed, ax=None, title="", shape=None, color_range=(0, 0.30)): + if ax is None: + ax = plt.gca() + + if shape is None: + height = int(np.sqrt(len(embed))) + shape = (height, -1) + embed = embed.reshape(shape) + + cmap = cm.get_cmap() + mappable = ax.imshow(embed, cmap=cmap) + cbar = plt.colorbar(mappable, ax=ax, fraction=0.046, pad=0.04) + cbar.set_clim(*color_range) + + ax.set_xticks([]), ax.set_yticks([]) + ax.set_title(title) diff --git a/speaker_encoder/model.py b/speaker_encoder/model.py new file mode 100644 index 0000000000000000000000000000000000000000..aefe6c84cd0de2031daf6b69a942e406594ad187 --- /dev/null +++ b/speaker_encoder/model.py @@ -0,0 +1,135 @@ +from speaker_encoder.params_model import * +from speaker_encoder.params_data import * +from scipy.interpolate import interp1d +from sklearn.metrics import roc_curve +from torch.nn.utils import clip_grad_norm_ +from scipy.optimize import brentq +from torch import nn +import numpy as np +import torch + + +class SpeakerEncoder(nn.Module): + def __init__(self, device, loss_device): + super().__init__() + self.loss_device = loss_device + + # Network defition + self.lstm = nn.LSTM(input_size=mel_n_channels, # 40 + hidden_size=model_hidden_size, # 256 + num_layers=model_num_layers, # 3 + batch_first=True).to(device) + self.linear = nn.Linear(in_features=model_hidden_size, + out_features=model_embedding_size).to(device) + self.relu = torch.nn.ReLU().to(device) + + # Cosine similarity scaling (with fixed initial parameter values) + self.similarity_weight = nn.Parameter(torch.tensor([10.])).to(loss_device) + self.similarity_bias = nn.Parameter(torch.tensor([-5.])).to(loss_device) + + # Loss + self.loss_fn = nn.CrossEntropyLoss().to(loss_device) + + def do_gradient_ops(self): + # Gradient scale + self.similarity_weight.grad *= 0.01 + self.similarity_bias.grad *= 0.01 + + # Gradient clipping + clip_grad_norm_(self.parameters(), 3, norm_type=2) + + def forward(self, utterances, hidden_init=None): + """ + Computes the embeddings of a batch of utterance spectrograms. + + :param utterances: batch of mel-scale filterbanks of same duration as a tensor of shape + (batch_size, n_frames, n_channels) + :param hidden_init: initial hidden state of the LSTM as a tensor of shape (num_layers, + batch_size, hidden_size). Will default to a tensor of zeros if None. + :return: the embeddings as a tensor of shape (batch_size, embedding_size) + """ + # Pass the input through the LSTM layers and retrieve all outputs, the final hidden state + # and the final cell state. + out, (hidden, cell) = self.lstm(utterances, hidden_init) + + # We take only the hidden state of the last layer + embeds_raw = self.relu(self.linear(hidden[-1])) + + # L2-normalize it + embeds = embeds_raw / torch.norm(embeds_raw, dim=1, keepdim=True) + + return embeds + + def similarity_matrix(self, embeds): + """ + Computes the similarity matrix according the section 2.1 of GE2E. + + :param embeds: the embeddings as a tensor of shape (speakers_per_batch, + utterances_per_speaker, embedding_size) + :return: the similarity matrix as a tensor of shape (speakers_per_batch, + utterances_per_speaker, speakers_per_batch) + """ + speakers_per_batch, utterances_per_speaker = embeds.shape[:2] + + # Inclusive centroids (1 per speaker). Cloning is needed for reverse differentiation + centroids_incl = torch.mean(embeds, dim=1, keepdim=True) + centroids_incl = centroids_incl.clone() / torch.norm(centroids_incl, dim=2, keepdim=True) + + # Exclusive centroids (1 per utterance) + centroids_excl = (torch.sum(embeds, dim=1, keepdim=True) - embeds) + centroids_excl /= (utterances_per_speaker - 1) + centroids_excl = centroids_excl.clone() / torch.norm(centroids_excl, dim=2, keepdim=True) + + # Similarity matrix. The cosine similarity of already 2-normed vectors is simply the dot + # product of these vectors (which is just an element-wise multiplication reduced by a sum). + # We vectorize the computation for efficiency. + sim_matrix = torch.zeros(speakers_per_batch, utterances_per_speaker, + speakers_per_batch).to(self.loss_device) + mask_matrix = 1 - np.eye(speakers_per_batch, dtype=np.int) + for j in range(speakers_per_batch): + mask = np.where(mask_matrix[j])[0] + sim_matrix[mask, :, j] = (embeds[mask] * centroids_incl[j]).sum(dim=2) + sim_matrix[j, :, j] = (embeds[j] * centroids_excl[j]).sum(dim=1) + + ## Even more vectorized version (slower maybe because of transpose) + # sim_matrix2 = torch.zeros(speakers_per_batch, speakers_per_batch, utterances_per_speaker + # ).to(self.loss_device) + # eye = np.eye(speakers_per_batch, dtype=np.int) + # mask = np.where(1 - eye) + # sim_matrix2[mask] = (embeds[mask[0]] * centroids_incl[mask[1]]).sum(dim=2) + # mask = np.where(eye) + # sim_matrix2[mask] = (embeds * centroids_excl).sum(dim=2) + # sim_matrix2 = sim_matrix2.transpose(1, 2) + + sim_matrix = sim_matrix * self.similarity_weight + self.similarity_bias + return sim_matrix + + def loss(self, embeds): + """ + Computes the softmax loss according the section 2.1 of GE2E. + + :param embeds: the embeddings as a tensor of shape (speakers_per_batch, + utterances_per_speaker, embedding_size) + :return: the loss and the EER for this batch of embeddings. + """ + speakers_per_batch, utterances_per_speaker = embeds.shape[:2] + + # Loss + sim_matrix = self.similarity_matrix(embeds) + sim_matrix = sim_matrix.reshape((speakers_per_batch * utterances_per_speaker, + speakers_per_batch)) + ground_truth = np.repeat(np.arange(speakers_per_batch), utterances_per_speaker) + target = torch.from_numpy(ground_truth).long().to(self.loss_device) + loss = self.loss_fn(sim_matrix, target) + + # EER (not backpropagated) + with torch.no_grad(): + inv_argmax = lambda i: np.eye(1, speakers_per_batch, i, dtype=np.int)[0] + labels = np.array([inv_argmax(i) for i in ground_truth]) + preds = sim_matrix.detach().cpu().numpy() + + # Snippet from https://yangcha.github.io/EER-ROC/ + fpr, tpr, thresholds = roc_curve(labels.flatten(), preds.flatten()) + eer = brentq(lambda x: 1. - x - interp1d(fpr, tpr)(x), 0., 1.) + + return loss, eer diff --git a/speaker_encoder/params_data.py b/speaker_encoder/params_data.py new file mode 100644 index 0000000000000000000000000000000000000000..bdb1716ed45617f2b127a7fb8885afe6cc74fb71 --- /dev/null +++ b/speaker_encoder/params_data.py @@ -0,0 +1,29 @@ + +## Mel-filterbank +mel_window_length = 25 # In milliseconds +mel_window_step = 10 # In milliseconds +mel_n_channels = 40 + + +## Audio +sampling_rate = 16000 +# Number of spectrogram frames in a partial utterance +partials_n_frames = 160 # 1600 ms +# Number of spectrogram frames at inference +inference_n_frames = 80 # 800 ms + + +## Voice Activation Detection +# Window size of the VAD. Must be either 10, 20 or 30 milliseconds. +# This sets the granularity of the VAD. Should not need to be changed. +vad_window_length = 30 # In milliseconds +# Number of frames to average together when performing the moving average smoothing. +# The larger this value, the larger the VAD variations must be to not get smoothed out. +vad_moving_average_width = 8 +# Maximum number of consecutive silent frames a segment can have. +vad_max_silence_length = 6 + + +## Audio volume normalization +audio_norm_target_dBFS = -30 + diff --git a/speaker_encoder/params_model.py b/speaker_encoder/params_model.py new file mode 100644 index 0000000000000000000000000000000000000000..3e356472fb5a27f370cb3920976a11d12a76c1b7 --- /dev/null +++ b/speaker_encoder/params_model.py @@ -0,0 +1,11 @@ + +## Model parameters +model_hidden_size = 256 +model_embedding_size = 256 +model_num_layers = 3 + + +## Training parameters +learning_rate_init = 1e-4 +speakers_per_batch = 64 +utterances_per_speaker = 10 diff --git a/speaker_encoder/preprocess.py b/speaker_encoder/preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..fe5ab25ef7cb4adeb76cad11962f179d6a38edcc --- /dev/null +++ b/speaker_encoder/preprocess.py @@ -0,0 +1,285 @@ +from multiprocess.pool import ThreadPool +from speaker_encoder.params_data import * +from speaker_encoder.config import librispeech_datasets, anglophone_nationalites +from datetime import datetime +from speaker_encoder import audio +from pathlib import Path +from tqdm import tqdm +import numpy as np + + +class DatasetLog: + """ + Registers metadata about the dataset in a text file. + """ + def __init__(self, root, name): + self.text_file = open(Path(root, "Log_%s.txt" % name.replace("/", "_")), "w") + self.sample_data = dict() + + start_time = str(datetime.now().strftime("%A %d %B %Y at %H:%M")) + self.write_line("Creating dataset %s on %s" % (name, start_time)) + self.write_line("-----") + self._log_params() + + def _log_params(self): + from speaker_encoder import params_data + self.write_line("Parameter values:") + for param_name in (p for p in dir(params_data) if not p.startswith("__")): + value = getattr(params_data, param_name) + self.write_line("\t%s: %s" % (param_name, value)) + self.write_line("-----") + + def write_line(self, line): + self.text_file.write("%s\n" % line) + + def add_sample(self, **kwargs): + for param_name, value in kwargs.items(): + if not param_name in self.sample_data: + self.sample_data[param_name] = [] + self.sample_data[param_name].append(value) + + def finalize(self): + self.write_line("Statistics:") + for param_name, values in self.sample_data.items(): + self.write_line("\t%s:" % param_name) + self.write_line("\t\tmin %.3f, max %.3f" % (np.min(values), np.max(values))) + self.write_line("\t\tmean %.3f, median %.3f" % (np.mean(values), np.median(values))) + self.write_line("-----") + end_time = str(datetime.now().strftime("%A %d %B %Y at %H:%M")) + self.write_line("Finished on %s" % end_time) + self.text_file.close() + + +def _init_preprocess_dataset(dataset_name, datasets_root, out_dir) -> (Path, DatasetLog): + dataset_root = datasets_root.joinpath(dataset_name) + if not dataset_root.exists(): + print("Couldn\'t find %s, skipping this dataset." % dataset_root) + return None, None + return dataset_root, DatasetLog(out_dir, dataset_name) + + +def _preprocess_speaker_dirs(speaker_dirs, dataset_name, datasets_root, out_dir, extension, + skip_existing, logger): + print("%s: Preprocessing data for %d speakers." % (dataset_name, len(speaker_dirs))) + + # Function to preprocess utterances for one speaker + def preprocess_speaker(speaker_dir: Path): + # Give a name to the speaker that includes its dataset + speaker_name = "_".join(speaker_dir.relative_to(datasets_root).parts) + + # Create an output directory with that name, as well as a txt file containing a + # reference to each source file. + speaker_out_dir = out_dir.joinpath(speaker_name) + speaker_out_dir.mkdir(exist_ok=True) + sources_fpath = speaker_out_dir.joinpath("_sources.txt") + + # There's a possibility that the preprocessing was interrupted earlier, check if + # there already is a sources file. + if sources_fpath.exists(): + try: + with sources_fpath.open("r") as sources_file: + existing_fnames = {line.split(",")[0] for line in sources_file} + except: + existing_fnames = {} + else: + existing_fnames = {} + + # Gather all audio files for that speaker recursively + sources_file = sources_fpath.open("a" if skip_existing else "w") + for in_fpath in speaker_dir.glob("**/*.%s" % extension): + # Check if the target output file already exists + out_fname = "_".join(in_fpath.relative_to(speaker_dir).parts) + out_fname = out_fname.replace(".%s" % extension, ".npy") + if skip_existing and out_fname in existing_fnames: + continue + + # Load and preprocess the waveform + wav = audio.preprocess_wav(in_fpath) + if len(wav) == 0: + continue + + # Create the mel spectrogram, discard those that are too short + frames = audio.wav_to_mel_spectrogram(wav) + if len(frames) < partials_n_frames: + continue + + out_fpath = speaker_out_dir.joinpath(out_fname) + np.save(out_fpath, frames) + logger.add_sample(duration=len(wav) / sampling_rate) + sources_file.write("%s,%s\n" % (out_fname, in_fpath)) + + sources_file.close() + + # Process the utterances for each speaker + with ThreadPool(8) as pool: + list(tqdm(pool.imap(preprocess_speaker, speaker_dirs), dataset_name, len(speaker_dirs), + unit="speakers")) + logger.finalize() + print("Done preprocessing %s.\n" % dataset_name) + + +# Function to preprocess utterances for one speaker +def __preprocess_speaker(speaker_dir: Path, datasets_root: Path, out_dir: Path, extension: str, skip_existing: bool): + # Give a name to the speaker that includes its dataset + speaker_name = "_".join(speaker_dir.relative_to(datasets_root).parts) + + # Create an output directory with that name, as well as a txt file containing a + # reference to each source file. + speaker_out_dir = out_dir.joinpath(speaker_name) + speaker_out_dir.mkdir(exist_ok=True) + sources_fpath = speaker_out_dir.joinpath("_sources.txt") + + # There's a possibility that the preprocessing was interrupted earlier, check if + # there already is a sources file. + # if sources_fpath.exists(): + # try: + # with sources_fpath.open("r") as sources_file: + # existing_fnames = {line.split(",")[0] for line in sources_file} + # except: + # existing_fnames = {} + # else: + # existing_fnames = {} + existing_fnames = {} + # Gather all audio files for that speaker recursively + sources_file = sources_fpath.open("a" if skip_existing else "w") + + for in_fpath in speaker_dir.glob("**/*.%s" % extension): + # Check if the target output file already exists + out_fname = "_".join(in_fpath.relative_to(speaker_dir).parts) + out_fname = out_fname.replace(".%s" % extension, ".npy") + if skip_existing and out_fname in existing_fnames: + continue + + # Load and preprocess the waveform + wav = audio.preprocess_wav(in_fpath) + if len(wav) == 0: + continue + + # Create the mel spectrogram, discard those that are too short + frames = audio.wav_to_mel_spectrogram(wav) + if len(frames) < partials_n_frames: + continue + + out_fpath = speaker_out_dir.joinpath(out_fname) + np.save(out_fpath, frames) + # logger.add_sample(duration=len(wav) / sampling_rate) + sources_file.write("%s,%s\n" % (out_fname, in_fpath)) + + sources_file.close() + return len(wav) + +def _preprocess_speaker_dirs_vox2(speaker_dirs, dataset_name, datasets_root, out_dir, extension, + skip_existing, logger): + # from multiprocessing import Pool, cpu_count + from pathos.multiprocessing import ProcessingPool as Pool + # Function to preprocess utterances for one speaker + def __preprocess_speaker(speaker_dir: Path): + # Give a name to the speaker that includes its dataset + speaker_name = "_".join(speaker_dir.relative_to(datasets_root).parts) + + # Create an output directory with that name, as well as a txt file containing a + # reference to each source file. + speaker_out_dir = out_dir.joinpath(speaker_name) + speaker_out_dir.mkdir(exist_ok=True) + sources_fpath = speaker_out_dir.joinpath("_sources.txt") + + existing_fnames = {} + # Gather all audio files for that speaker recursively + sources_file = sources_fpath.open("a" if skip_existing else "w") + wav_lens = [] + for in_fpath in speaker_dir.glob("**/*.%s" % extension): + # Check if the target output file already exists + out_fname = "_".join(in_fpath.relative_to(speaker_dir).parts) + out_fname = out_fname.replace(".%s" % extension, ".npy") + if skip_existing and out_fname in existing_fnames: + continue + + # Load and preprocess the waveform + wav = audio.preprocess_wav(in_fpath) + if len(wav) == 0: + continue + + # Create the mel spectrogram, discard those that are too short + frames = audio.wav_to_mel_spectrogram(wav) + if len(frames) < partials_n_frames: + continue + + out_fpath = speaker_out_dir.joinpath(out_fname) + np.save(out_fpath, frames) + # logger.add_sample(duration=len(wav) / sampling_rate) + sources_file.write("%s,%s\n" % (out_fname, in_fpath)) + wav_lens.append(len(wav)) + sources_file.close() + return wav_lens + + print("%s: Preprocessing data for %d speakers." % (dataset_name, len(speaker_dirs))) + # Process the utterances for each speaker + # with ThreadPool(8) as pool: + # list(tqdm(pool.imap(preprocess_speaker, speaker_dirs), dataset_name, len(speaker_dirs), + # unit="speakers")) + pool = Pool(processes=20) + for i, wav_lens in enumerate(pool.map(__preprocess_speaker, speaker_dirs), 1): + for wav_len in wav_lens: + logger.add_sample(duration=wav_len / sampling_rate) + print(f'{i}/{len(speaker_dirs)} \r') + + logger.finalize() + print("Done preprocessing %s.\n" % dataset_name) + + +def preprocess_librispeech(datasets_root: Path, out_dir: Path, skip_existing=False): + for dataset_name in librispeech_datasets["train"]["other"]: + # Initialize the preprocessing + dataset_root, logger = _init_preprocess_dataset(dataset_name, datasets_root, out_dir) + if not dataset_root: + return + + # Preprocess all speakers + speaker_dirs = list(dataset_root.glob("*")) + _preprocess_speaker_dirs(speaker_dirs, dataset_name, datasets_root, out_dir, "flac", + skip_existing, logger) + + +def preprocess_voxceleb1(datasets_root: Path, out_dir: Path, skip_existing=False): + # Initialize the preprocessing + dataset_name = "VoxCeleb1" + dataset_root, logger = _init_preprocess_dataset(dataset_name, datasets_root, out_dir) + if not dataset_root: + return + + # Get the contents of the meta file + with dataset_root.joinpath("vox1_meta.csv").open("r") as metafile: + metadata = [line.split("\t") for line in metafile][1:] + + # Select the ID and the nationality, filter out non-anglophone speakers + nationalities = {line[0]: line[3] for line in metadata} + # keep_speaker_ids = [speaker_id for speaker_id, nationality in nationalities.items() if + # nationality.lower() in anglophone_nationalites] + keep_speaker_ids = [speaker_id for speaker_id, nationality in nationalities.items()] + print("VoxCeleb1: using samples from %d (presumed anglophone) speakers out of %d." % + (len(keep_speaker_ids), len(nationalities))) + + # Get the speaker directories for anglophone speakers only + speaker_dirs = dataset_root.joinpath("wav").glob("*") + speaker_dirs = [speaker_dir for speaker_dir in speaker_dirs if + speaker_dir.name in keep_speaker_ids] + print("VoxCeleb1: found %d anglophone speakers on the disk, %d missing (this is normal)." % + (len(speaker_dirs), len(keep_speaker_ids) - len(speaker_dirs))) + + # Preprocess all speakers + _preprocess_speaker_dirs(speaker_dirs, dataset_name, datasets_root, out_dir, "wav", + skip_existing, logger) + + +def preprocess_voxceleb2(datasets_root: Path, out_dir: Path, skip_existing=False): + # Initialize the preprocessing + dataset_name = "VoxCeleb2" + dataset_root, logger = _init_preprocess_dataset(dataset_name, datasets_root, out_dir) + if not dataset_root: + return + + # Get the speaker directories + # Preprocess all speakers + speaker_dirs = list(dataset_root.joinpath("dev", "aac").glob("*")) + _preprocess_speaker_dirs_vox2(speaker_dirs, dataset_name, datasets_root, out_dir, "m4a", + skip_existing, logger) diff --git a/speaker_encoder/train.py b/speaker_encoder/train.py new file mode 100644 index 0000000000000000000000000000000000000000..2e9485afbeead6a063b5ef69a85f05757d6c91ff --- /dev/null +++ b/speaker_encoder/train.py @@ -0,0 +1,125 @@ +from speaker_encoder.visualizations import Visualizations +from speaker_encoder.data_objects import SpeakerVerificationDataLoader, SpeakerVerificationDataset +from speaker_encoder.params_model import * +from speaker_encoder.model import SpeakerEncoder +from utils.profiler import Profiler +from pathlib import Path +import torch + +def sync(device: torch.device): + # FIXME + return + # For correct profiling (cuda operations are async) + if device.type == "cuda": + torch.cuda.synchronize(device) + +def train(run_id: str, clean_data_root: Path, models_dir: Path, umap_every: int, save_every: int, + backup_every: int, vis_every: int, force_restart: bool, visdom_server: str, + no_visdom: bool): + # Create a dataset and a dataloader + dataset = SpeakerVerificationDataset(clean_data_root) + loader = SpeakerVerificationDataLoader( + dataset, + speakers_per_batch, # 64 + utterances_per_speaker, # 10 + num_workers=8, + ) + + # Setup the device on which to run the forward pass and the loss. These can be different, + # because the forward pass is faster on the GPU whereas the loss is often (depending on your + # hyperparameters) faster on the CPU. + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + # FIXME: currently, the gradient is None if loss_device is cuda + loss_device = torch.device("cpu") + + # Create the model and the optimizer + model = SpeakerEncoder(device, loss_device) + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate_init) + init_step = 1 + + # Configure file path for the model + state_fpath = models_dir.joinpath(run_id + ".pt") + backup_dir = models_dir.joinpath(run_id + "_backups") + + # Load any existing model + if not force_restart: + if state_fpath.exists(): + print("Found existing model \"%s\", loading it and resuming training." % run_id) + checkpoint = torch.load(state_fpath) + init_step = checkpoint["step"] + model.load_state_dict(checkpoint["model_state"]) + optimizer.load_state_dict(checkpoint["optimizer_state"]) + optimizer.param_groups[0]["lr"] = learning_rate_init + else: + print("No model \"%s\" found, starting training from scratch." % run_id) + else: + print("Starting the training from scratch.") + model.train() + + # Initialize the visualization environment + vis = Visualizations(run_id, vis_every, server=visdom_server, disabled=no_visdom) + vis.log_dataset(dataset) + vis.log_params() + device_name = str(torch.cuda.get_device_name(0) if torch.cuda.is_available() else "CPU") + vis.log_implementation({"Device": device_name}) + + # Training loop + profiler = Profiler(summarize_every=10, disabled=False) + for step, speaker_batch in enumerate(loader, init_step): + profiler.tick("Blocking, waiting for batch (threaded)") + + # Forward pass + inputs = torch.from_numpy(speaker_batch.data).to(device) + sync(device) + profiler.tick("Data to %s" % device) + embeds = model(inputs) + sync(device) + profiler.tick("Forward pass") + embeds_loss = embeds.view((speakers_per_batch, utterances_per_speaker, -1)).to(loss_device) + loss, eer = model.loss(embeds_loss) + sync(loss_device) + profiler.tick("Loss") + + # Backward pass + model.zero_grad() + loss.backward() + profiler.tick("Backward pass") + model.do_gradient_ops() + optimizer.step() + profiler.tick("Parameter update") + + # Update visualizations + # learning_rate = optimizer.param_groups[0]["lr"] + vis.update(loss.item(), eer, step) + + # Draw projections and save them to the backup folder + if umap_every != 0 and step % umap_every == 0: + print("Drawing and saving projections (step %d)" % step) + backup_dir.mkdir(exist_ok=True) + projection_fpath = backup_dir.joinpath("%s_umap_%06d.png" % (run_id, step)) + embeds = embeds.detach().cpu().numpy() + vis.draw_projections(embeds, utterances_per_speaker, step, projection_fpath) + vis.save() + + # Overwrite the latest version of the model + if save_every != 0 and step % save_every == 0: + print("Saving the model (step %d)" % step) + torch.save({ + "step": step + 1, + "model_state": model.state_dict(), + "optimizer_state": optimizer.state_dict(), + }, state_fpath) + + # Make a backup + if backup_every != 0 and step % backup_every == 0: + print("Making a backup (step %d)" % step) + backup_dir.mkdir(exist_ok=True) + backup_fpath = backup_dir.joinpath("%s_bak_%06d.pt" % (run_id, step)) + torch.save({ + "step": step + 1, + "model_state": model.state_dict(), + "optimizer_state": optimizer.state_dict(), + }, backup_fpath) + + profiler.tick("Extras (visualizations, saving)") + diff --git a/speaker_encoder/visualizations.py b/speaker_encoder/visualizations.py new file mode 100644 index 0000000000000000000000000000000000000000..ec00fc64d6e9fda2bb8e613531066ac824df1451 --- /dev/null +++ b/speaker_encoder/visualizations.py @@ -0,0 +1,178 @@ +from speaker_encoder.data_objects.speaker_verification_dataset import SpeakerVerificationDataset +from datetime import datetime +from time import perf_counter as timer +import matplotlib.pyplot as plt +import numpy as np +# import webbrowser +import visdom +import umap + +colormap = np.array([ + [76, 255, 0], + [0, 127, 70], + [255, 0, 0], + [255, 217, 38], + [0, 135, 255], + [165, 0, 165], + [255, 167, 255], + [0, 255, 255], + [255, 96, 38], + [142, 76, 0], + [33, 0, 127], + [0, 0, 0], + [183, 183, 183], +], dtype=np.float) / 255 + + +class Visualizations: + def __init__(self, env_name=None, update_every=10, server="http://localhost", disabled=False): + # Tracking data + self.last_update_timestamp = timer() + self.update_every = update_every + self.step_times = [] + self.losses = [] + self.eers = [] + print("Updating the visualizations every %d steps." % update_every) + + # If visdom is disabled TODO: use a better paradigm for that + self.disabled = disabled + if self.disabled: + return + + # Set the environment name + now = str(datetime.now().strftime("%d-%m %Hh%M")) + if env_name is None: + self.env_name = now + else: + self.env_name = "%s (%s)" % (env_name, now) + + # Connect to visdom and open the corresponding window in the browser + try: + self.vis = visdom.Visdom(server, env=self.env_name, raise_exceptions=True) + except ConnectionError: + raise Exception("No visdom server detected. Run the command \"visdom\" in your CLI to " + "start it.") + # webbrowser.open("http://localhost:8097/env/" + self.env_name) + + # Create the windows + self.loss_win = None + self.eer_win = None + # self.lr_win = None + self.implementation_win = None + self.projection_win = None + self.implementation_string = "" + + def log_params(self): + if self.disabled: + return + from speaker_encoder import params_data + from speaker_encoder import params_model + param_string = "Model parameters:
" + for param_name in (p for p in dir(params_model) if not p.startswith("__")): + value = getattr(params_model, param_name) + param_string += "\t%s: %s
" % (param_name, value) + param_string += "Data parameters:
" + for param_name in (p for p in dir(params_data) if not p.startswith("__")): + value = getattr(params_data, param_name) + param_string += "\t%s: %s
" % (param_name, value) + self.vis.text(param_string, opts={"title": "Parameters"}) + + def log_dataset(self, dataset: SpeakerVerificationDataset): + if self.disabled: + return + dataset_string = "" + dataset_string += "Speakers: %s\n" % len(dataset.speakers) + dataset_string += "\n" + dataset.get_logs() + dataset_string = dataset_string.replace("\n", "
") + self.vis.text(dataset_string, opts={"title": "Dataset"}) + + def log_implementation(self, params): + if self.disabled: + return + implementation_string = "" + for param, value in params.items(): + implementation_string += "%s: %s\n" % (param, value) + implementation_string = implementation_string.replace("\n", "
") + self.implementation_string = implementation_string + self.implementation_win = self.vis.text( + implementation_string, + opts={"title": "Training implementation"} + ) + + def update(self, loss, eer, step): + # Update the tracking data + now = timer() + self.step_times.append(1000 * (now - self.last_update_timestamp)) + self.last_update_timestamp = now + self.losses.append(loss) + self.eers.append(eer) + print(".", end="") + + # Update the plots every steps + if step % self.update_every != 0: + return + time_string = "Step time: mean: %5dms std: %5dms" % \ + (int(np.mean(self.step_times)), int(np.std(self.step_times))) + print("\nStep %6d Loss: %.4f EER: %.4f %s" % + (step, np.mean(self.losses), np.mean(self.eers), time_string)) + if not self.disabled: + self.loss_win = self.vis.line( + [np.mean(self.losses)], + [step], + win=self.loss_win, + update="append" if self.loss_win else None, + opts=dict( + legend=["Avg. loss"], + xlabel="Step", + ylabel="Loss", + title="Loss", + ) + ) + self.eer_win = self.vis.line( + [np.mean(self.eers)], + [step], + win=self.eer_win, + update="append" if self.eer_win else None, + opts=dict( + legend=["Avg. EER"], + xlabel="Step", + ylabel="EER", + title="Equal error rate" + ) + ) + if self.implementation_win is not None: + self.vis.text( + self.implementation_string + ("%s" % time_string), + win=self.implementation_win, + opts={"title": "Training implementation"}, + ) + + # Reset the tracking + self.losses.clear() + self.eers.clear() + self.step_times.clear() + + def draw_projections(self, embeds, utterances_per_speaker, step, out_fpath=None, + max_speakers=10): + max_speakers = min(max_speakers, len(colormap)) + embeds = embeds[:max_speakers * utterances_per_speaker] + + n_speakers = len(embeds) // utterances_per_speaker + ground_truth = np.repeat(np.arange(n_speakers), utterances_per_speaker) + colors = [colormap[i] for i in ground_truth] + + reducer = umap.UMAP() + projected = reducer.fit_transform(embeds) + plt.scatter(projected[:, 0], projected[:, 1], c=colors) + plt.gca().set_aspect("equal", "datalim") + plt.title("UMAP projection (step %d)" % step) + if not self.disabled: + self.projection_win = self.vis.matplot(plt, win=self.projection_win) + if out_fpath is not None: + plt.savefig(out_fpath) + plt.clf() + + def save(self): + if not self.disabled: + self.vis.save([self.env_name]) + \ No newline at end of file diff --git a/speaker_encoder/voice_encoder.py b/speaker_encoder/voice_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..88cdee2de76b72db58c5dd19a888597e0fe12fbb --- /dev/null +++ b/speaker_encoder/voice_encoder.py @@ -0,0 +1,173 @@ +from speaker_encoder.hparams import * +from speaker_encoder import audio +from pathlib import Path +from typing import Union, List +from torch import nn +from time import perf_counter as timer +import numpy as np +import torch + + +class SpeakerEncoder(nn.Module): + def __init__(self, weights_fpath, device: Union[str, torch.device]=None, verbose=True): + """ + :param device: either a torch device or the name of a torch device (e.g. "cpu", "cuda"). + If None, defaults to cuda if it is available on your machine, otherwise the model will + run on cpu. Outputs are always returned on the cpu, as numpy arrays. + """ + super().__init__() + + # Define the network + self.lstm = nn.LSTM(mel_n_channels, model_hidden_size, model_num_layers, batch_first=True) + self.linear = nn.Linear(model_hidden_size, model_embedding_size) + self.relu = nn.ReLU() + + # Get the target device + if device is None: + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + elif isinstance(device, str): + device = torch.device(device) + self.device = device + + # Load the pretrained model'speaker weights + # weights_fpath = Path(__file__).resolve().parent.joinpath("pretrained.pt") + # if not weights_fpath.exists(): + # raise Exception("Couldn't find the voice encoder pretrained model at %s." % + # weights_fpath) + + start = timer() + checkpoint = torch.load(weights_fpath, map_location="cpu") + + self.load_state_dict(checkpoint["model_state"], strict=False) + self.to(device) + + if verbose: + print("Loaded the voice encoder model on %s in %.2f seconds." % + (device.type, timer() - start)) + + def forward(self, mels: torch.FloatTensor): + """ + Computes the embeddings of a batch of utterance spectrograms. + :param mels: a batch of mel spectrograms of same duration as a float32 tensor of shape + (batch_size, n_frames, n_channels) + :return: the embeddings as a float 32 tensor of shape (batch_size, embedding_size). + Embeddings are positive and L2-normed, thus they lay in the range [0, 1]. + """ + # Pass the input through the LSTM layers and retrieve the final hidden state of the last + # layer. Apply a cutoff to 0 for negative values and L2 normalize the embeddings. + _, (hidden, _) = self.lstm(mels) + embeds_raw = self.relu(self.linear(hidden[-1])) + return embeds_raw / torch.norm(embeds_raw, dim=1, keepdim=True) + + @staticmethod + def compute_partial_slices(n_samples: int, rate, min_coverage): + """ + Computes where to split an utterance waveform and its corresponding mel spectrogram to + obtain partial utterances of each. Both the waveform and the + mel spectrogram slices are returned, so as to make each partial utterance waveform + correspond to its spectrogram. + + The returned ranges may be indexing further than the length of the waveform. It is + recommended that you pad the waveform with zeros up to wav_slices[-1].stop. + + :param n_samples: the number of samples in the waveform + :param rate: how many partial utterances should occur per second. Partial utterances must + cover the span of the entire utterance, thus the rate should not be lower than the inverse + of the duration of a partial utterance. By default, partial utterances are 1.6s long and + the minimum rate is thus 0.625. + :param min_coverage: when reaching the last partial utterance, it may or may not have + enough frames. If at least of are present, + then the last partial utterance will be considered by zero-padding the audio. Otherwise, + it will be discarded. If there aren't enough frames for one partial utterance, + this parameter is ignored so that the function always returns at least one slice. + :return: the waveform slices and mel spectrogram slices as lists of array slices. Index + respectively the waveform and the mel spectrogram with these slices to obtain the partial + utterances. + """ + assert 0 < min_coverage <= 1 + + # Compute how many frames separate two partial utterances + samples_per_frame = int((sampling_rate * mel_window_step / 1000)) + n_frames = int(np.ceil((n_samples + 1) / samples_per_frame)) + frame_step = int(np.round((sampling_rate / rate) / samples_per_frame)) + assert 0 < frame_step, "The rate is too high" + assert frame_step <= partials_n_frames, "The rate is too low, it should be %f at least" % \ + (sampling_rate / (samples_per_frame * partials_n_frames)) + + # Compute the slices + wav_slices, mel_slices = [], [] + steps = max(1, n_frames - partials_n_frames + frame_step + 1) + for i in range(0, steps, frame_step): + mel_range = np.array([i, i + partials_n_frames]) + wav_range = mel_range * samples_per_frame + mel_slices.append(slice(*mel_range)) + wav_slices.append(slice(*wav_range)) + + # Evaluate whether extra padding is warranted or not + last_wav_range = wav_slices[-1] + coverage = (n_samples - last_wav_range.start) / (last_wav_range.stop - last_wav_range.start) + if coverage < min_coverage and len(mel_slices) > 1: + mel_slices = mel_slices[:-1] + wav_slices = wav_slices[:-1] + + return wav_slices, mel_slices + + def embed_utterance(self, wav: np.ndarray, return_partials=False, rate=1.3, min_coverage=0.75): + """ + Computes an embedding for a single utterance. The utterance is divided in partial + utterances and an embedding is computed for each. The complete utterance embedding is the + L2-normed average embedding of the partial utterances. + + TODO: independent batched version of this function + + :param wav: a preprocessed utterance waveform as a numpy array of float32 + :param return_partials: if True, the partial embeddings will also be returned along with + the wav slices corresponding to each partial utterance. + :param rate: how many partial utterances should occur per second. Partial utterances must + cover the span of the entire utterance, thus the rate should not be lower than the inverse + of the duration of a partial utterance. By default, partial utterances are 1.6s long and + the minimum rate is thus 0.625. + :param min_coverage: when reaching the last partial utterance, it may or may not have + enough frames. If at least of are present, + then the last partial utterance will be considered by zero-padding the audio. Otherwise, + it will be discarded. If there aren't enough frames for one partial utterance, + this parameter is ignored so that the function always returns at least one slice. + :return: the embedding as a numpy array of float32 of shape (model_embedding_size,). If + is True, the partial utterances as a numpy array of float32 of shape + (n_partials, model_embedding_size) and the wav partials as a list of slices will also be + returned. + """ + # Compute where to split the utterance into partials and pad the waveform with zeros if + # the partial utterances cover a larger range. + wav_slices, mel_slices = self.compute_partial_slices(len(wav), rate, min_coverage) + max_wave_length = wav_slices[-1].stop + if max_wave_length >= len(wav): + wav = np.pad(wav, (0, max_wave_length - len(wav)), "constant") + + # Split the utterance into partials and forward them through the model + mel = audio.wav_to_mel_spectrogram(wav) + mels = np.array([mel[s] for s in mel_slices]) + with torch.no_grad(): + mels = torch.from_numpy(mels).to(self.device) + partial_embeds = self(mels).cpu().numpy() + + # Compute the utterance embedding from the partial embeddings + raw_embed = np.mean(partial_embeds, axis=0) + embed = raw_embed / np.linalg.norm(raw_embed, 2) + + if return_partials: + return embed, partial_embeds, wav_slices + return embed + + def embed_speaker(self, wavs: List[np.ndarray], **kwargs): + """ + Compute the embedding of a collection of wavs (presumably from the same speaker) by + averaging their embedding and L2-normalizing it. + + :param wavs: list of wavs a numpy arrays of float32. + :param kwargs: extra arguments to embed_utterance() + :return: the embedding as a numpy array of float32 of shape (model_embedding_size,). + """ + raw_embed = np.mean([self.embed_utterance(wav, return_partials=False, **kwargs) \ + for wav in wavs], axis=0) + return raw_embed / np.linalg.norm(raw_embed, 2) \ No newline at end of file diff --git a/src/__pycache__/generate_batch.cpython-310.pyc b/src/__pycache__/generate_batch.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..58d86073980c2c0367d0fb6f8f2d305f84152e74 Binary files /dev/null and b/src/__pycache__/generate_batch.cpython-310.pyc differ diff --git a/src/__pycache__/generate_facerender_batch.cpython-310.pyc b/src/__pycache__/generate_facerender_batch.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..11763cad316fc2856e721dd864f18568616cb281 Binary files /dev/null and b/src/__pycache__/generate_facerender_batch.cpython-310.pyc differ diff --git a/src/__pycache__/gradio_demo.cpython-310.pyc b/src/__pycache__/gradio_demo.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..139cb6b1c52349bf3012877ded435afd95813d67 Binary files /dev/null and b/src/__pycache__/gradio_demo.cpython-310.pyc differ diff --git a/src/__pycache__/test_audio2coeff.cpython-310.pyc b/src/__pycache__/test_audio2coeff.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6a25863f31c832fe488716800366bb384c9e5cf Binary files /dev/null and b/src/__pycache__/test_audio2coeff.cpython-310.pyc differ diff --git a/src/audio2exp_models/__pycache__/audio2exp.cpython-310.pyc b/src/audio2exp_models/__pycache__/audio2exp.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a471643a5db54fe2987e992732b2bcca23992d89 Binary files /dev/null and b/src/audio2exp_models/__pycache__/audio2exp.cpython-310.pyc differ diff --git a/src/audio2exp_models/__pycache__/networks.cpython-310.pyc b/src/audio2exp_models/__pycache__/networks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2fb5e0272e8277240b7a8c211b205cf7e5c167f4 Binary files /dev/null and b/src/audio2exp_models/__pycache__/networks.cpython-310.pyc differ diff --git a/src/audio2exp_models/audio2exp.py b/src/audio2exp_models/audio2exp.py new file mode 100644 index 0000000000000000000000000000000000000000..9e79a929560592687a505e13188796e2b0ca8772 --- /dev/null +++ b/src/audio2exp_models/audio2exp.py @@ -0,0 +1,41 @@ +from tqdm import tqdm +import torch +from torch import nn + + +class Audio2Exp(nn.Module): + def __init__(self, netG, cfg, device, prepare_training_loss=False): + super(Audio2Exp, self).__init__() + self.cfg = cfg + self.device = device + self.netG = netG.to(device) + + def test(self, batch): + + mel_input = batch['indiv_mels'] # bs T 1 80 16 + bs = mel_input.shape[0] + T = mel_input.shape[1] + + exp_coeff_pred = [] + + for i in tqdm(range(0, T, 10),'audio2exp:'): # every 10 frames + + current_mel_input = mel_input[:,i:i+10] + + #ref = batch['ref'][:, :, :64].repeat((1,current_mel_input.shape[1],1)) #bs T 64 + ref = batch['ref'][:, :, :64][:, i:i+10] + ratio = batch['ratio_gt'][:, i:i+10] #bs T + + audiox = current_mel_input.view(-1, 1, 80, 16) # bs*T 1 80 16 + + curr_exp_coeff_pred = self.netG(audiox, ref, ratio) # bs T 64 + + exp_coeff_pred += [curr_exp_coeff_pred] + + # BS x T x 64 + results_dict = { + 'exp_coeff_pred': torch.cat(exp_coeff_pred, axis=1) + } + return results_dict + + diff --git a/src/audio2exp_models/networks.py b/src/audio2exp_models/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..f052e18101f5446a527ae354b3621e7d0d4991cc --- /dev/null +++ b/src/audio2exp_models/networks.py @@ -0,0 +1,74 @@ +import torch +import torch.nn.functional as F +from torch import nn + +class Conv2d(nn.Module): + def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, use_act = True, *args, **kwargs): + super().__init__(*args, **kwargs) + self.conv_block = nn.Sequential( + nn.Conv2d(cin, cout, kernel_size, stride, padding), + nn.BatchNorm2d(cout) + ) + self.act = nn.ReLU() + self.residual = residual + self.use_act = use_act + + def forward(self, x): + out = self.conv_block(x) + if self.residual: + out += x + + if self.use_act: + return self.act(out) + else: + return out + +class SimpleWrapperV2(nn.Module): + def __init__(self) -> None: + super().__init__() + self.audio_encoder = nn.Sequential( + Conv2d(1, 32, kernel_size=3, stride=1, padding=1), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(64, 128, kernel_size=3, stride=3, padding=1), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(256, 512, kernel_size=3, stride=1, padding=0), + Conv2d(512, 512, kernel_size=1, stride=1, padding=0), + ) + + #### load the pre-trained audio_encoder + #self.audio_encoder = self.audio_encoder.to(device) + ''' + wav2lip_state_dict = torch.load('/apdcephfs_cq2/share_1290939/wenxuazhang/checkpoints/wav2lip.pth')['state_dict'] + state_dict = self.audio_encoder.state_dict() + + for k,v in wav2lip_state_dict.items(): + if 'audio_encoder' in k: + print('init:', k) + state_dict[k.replace('module.audio_encoder.', '')] = v + self.audio_encoder.load_state_dict(state_dict) + ''' + + self.mapping1 = nn.Linear(512+64+1, 64) + #self.mapping2 = nn.Linear(30, 64) + #nn.init.constant_(self.mapping1.weight, 0.) + nn.init.constant_(self.mapping1.bias, 0.) + + def forward(self, x, ref, ratio): + x = self.audio_encoder(x).view(x.size(0), -1) + ref_reshape = ref.reshape(x.size(0), -1) + ratio = ratio.reshape(x.size(0), -1) + + y = self.mapping1(torch.cat([x, ref_reshape, ratio], dim=1)) + out = y.reshape(ref.shape[0], ref.shape[1], -1) #+ ref # resudial + return out diff --git a/src/audio2pose_models/__pycache__/audio2pose.cpython-310.pyc b/src/audio2pose_models/__pycache__/audio2pose.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa74857960866aec54c3968edaa5c0c71f7472a3 Binary files /dev/null and b/src/audio2pose_models/__pycache__/audio2pose.cpython-310.pyc differ diff --git a/src/audio2pose_models/__pycache__/audio_encoder.cpython-310.pyc b/src/audio2pose_models/__pycache__/audio_encoder.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57dd68a89f16907a8aa883bab557a1c4753e417d Binary files /dev/null and b/src/audio2pose_models/__pycache__/audio_encoder.cpython-310.pyc differ diff --git a/src/audio2pose_models/__pycache__/cvae.cpython-310.pyc b/src/audio2pose_models/__pycache__/cvae.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6c3384c9855f291021ab88ab3b2c73350fa64da Binary files /dev/null and b/src/audio2pose_models/__pycache__/cvae.cpython-310.pyc differ diff --git a/src/audio2pose_models/__pycache__/discriminator.cpython-310.pyc b/src/audio2pose_models/__pycache__/discriminator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..681216850018cff31f73699f1b5b1373f0a314ed Binary files /dev/null and b/src/audio2pose_models/__pycache__/discriminator.cpython-310.pyc differ diff --git a/src/audio2pose_models/__pycache__/networks.cpython-310.pyc b/src/audio2pose_models/__pycache__/networks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..67df9cbcc437519b452c91db9fa3d3171b1bcadb Binary files /dev/null and b/src/audio2pose_models/__pycache__/networks.cpython-310.pyc differ diff --git a/src/audio2pose_models/__pycache__/res_unet.cpython-310.pyc b/src/audio2pose_models/__pycache__/res_unet.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf2ed55d212e2f838b206976a0912f7b40de9e84 Binary files /dev/null and b/src/audio2pose_models/__pycache__/res_unet.cpython-310.pyc differ diff --git a/src/audio2pose_models/audio2pose.py b/src/audio2pose_models/audio2pose.py new file mode 100644 index 0000000000000000000000000000000000000000..2b8cd1427038460a7679260a424d2f01d2bcf2c5 --- /dev/null +++ b/src/audio2pose_models/audio2pose.py @@ -0,0 +1,94 @@ +import torch +from torch import nn +from src.audio2pose_models.cvae import CVAE +from src.audio2pose_models.discriminator import PoseSequenceDiscriminator +from src.audio2pose_models.audio_encoder import AudioEncoder + +class Audio2Pose(nn.Module): + def __init__(self, cfg, wav2lip_checkpoint, device='cuda'): + super().__init__() + self.cfg = cfg + self.seq_len = cfg.MODEL.CVAE.SEQ_LEN + self.latent_dim = cfg.MODEL.CVAE.LATENT_SIZE + self.device = device + + self.audio_encoder = AudioEncoder(wav2lip_checkpoint, device) + self.audio_encoder.eval() + for param in self.audio_encoder.parameters(): + param.requires_grad = False + + self.netG = CVAE(cfg) + self.netD_motion = PoseSequenceDiscriminator(cfg) + + + def forward(self, x): + + batch = {} + coeff_gt = x['gt'].cuda().squeeze(0) #bs frame_len+1 73 + batch['pose_motion_gt'] = coeff_gt[:, 1:, 64:70] - coeff_gt[:, :1, 64:70] #bs frame_len 6 + batch['ref'] = coeff_gt[:, 0, 64:70] #bs 6 + batch['class'] = x['class'].squeeze(0).cuda() # bs + indiv_mels= x['indiv_mels'].cuda().squeeze(0) # bs seq_len+1 80 16 + + # forward + audio_emb_list = [] + audio_emb = self.audio_encoder(indiv_mels[:, 1:, :, :].unsqueeze(2)) #bs seq_len 512 + batch['audio_emb'] = audio_emb + batch = self.netG(batch) + + pose_motion_pred = batch['pose_motion_pred'] # bs frame_len 6 + pose_gt = coeff_gt[:, 1:, 64:70].clone() # bs frame_len 6 + pose_pred = coeff_gt[:, :1, 64:70] + pose_motion_pred # bs frame_len 6 + + batch['pose_pred'] = pose_pred + batch['pose_gt'] = pose_gt + + return batch + + def test(self, x): + + batch = {} + ref = x['ref'] #bs 1 70 + batch['ref'] = x['ref'][:,0,-6:] + batch['class'] = x['class'] + bs = ref.shape[0] + + indiv_mels= x['indiv_mels'] # bs T 1 80 16 + indiv_mels_use = indiv_mels[:, 1:] # we regard the ref as the first frame + num_frames = x['num_frames'] + num_frames = int(num_frames) - 1 + + # + div = num_frames//self.seq_len + re = num_frames%self.seq_len + audio_emb_list = [] + pose_motion_pred_list = [torch.zeros(batch['ref'].unsqueeze(1).shape, dtype=batch['ref'].dtype, + device=batch['ref'].device)] + + for i in range(div): + z = torch.randn(bs, self.latent_dim).to(ref.device) + batch['z'] = z + audio_emb = self.audio_encoder(indiv_mels_use[:, i*self.seq_len:(i+1)*self.seq_len,:,:,:]) #bs seq_len 512 + batch['audio_emb'] = audio_emb + batch = self.netG.test(batch) + pose_motion_pred_list.append(batch['pose_motion_pred']) #list of bs seq_len 6 + + if re != 0: + z = torch.randn(bs, self.latent_dim).to(ref.device) + batch['z'] = z + audio_emb = self.audio_encoder(indiv_mels_use[:, -1*self.seq_len:,:,:,:]) #bs seq_len 512 + if audio_emb.shape[1] != self.seq_len: + pad_dim = self.seq_len-audio_emb.shape[1] + pad_audio_emb = audio_emb[:, :1].repeat(1, pad_dim, 1) + audio_emb = torch.cat([pad_audio_emb, audio_emb], 1) + batch['audio_emb'] = audio_emb + batch = self.netG.test(batch) + pose_motion_pred_list.append(batch['pose_motion_pred'][:,-1*re:,:]) + + pose_motion_pred = torch.cat(pose_motion_pred_list, dim = 1) + batch['pose_motion_pred'] = pose_motion_pred + + pose_pred = ref[:, :1, -6:] + pose_motion_pred # bs T 6 + + batch['pose_pred'] = pose_pred + return batch diff --git a/src/audio2pose_models/audio_encoder.py b/src/audio2pose_models/audio_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..6279d2014a2e786a6c549f084339e18d00e50331 --- /dev/null +++ b/src/audio2pose_models/audio_encoder.py @@ -0,0 +1,64 @@ +import torch +from torch import nn +from torch.nn import functional as F + +class Conv2d(nn.Module): + def __init__(self, cin, cout, kernel_size, stride, padding, residual=False, *args, **kwargs): + super().__init__(*args, **kwargs) + self.conv_block = nn.Sequential( + nn.Conv2d(cin, cout, kernel_size, stride, padding), + nn.BatchNorm2d(cout) + ) + self.act = nn.ReLU() + self.residual = residual + + def forward(self, x): + out = self.conv_block(x) + if self.residual: + out += x + return self.act(out) + +class AudioEncoder(nn.Module): + def __init__(self, wav2lip_checkpoint, device): + super(AudioEncoder, self).__init__() + + self.audio_encoder = nn.Sequential( + Conv2d(1, 32, kernel_size=3, stride=1, padding=1), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(32, 32, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(32, 64, kernel_size=3, stride=(3, 1), padding=1), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(64, 64, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(64, 128, kernel_size=3, stride=3, padding=1), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + Conv2d(128, 128, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(128, 256, kernel_size=3, stride=(3, 2), padding=1), + Conv2d(256, 256, kernel_size=3, stride=1, padding=1, residual=True), + + Conv2d(256, 512, kernel_size=3, stride=1, padding=0), + Conv2d(512, 512, kernel_size=1, stride=1, padding=0),) + + #### load the pre-trained audio_encoder, we do not need to load wav2lip model here. + # wav2lip_state_dict = torch.load(wav2lip_checkpoint, map_location=torch.device(device))['state_dict'] + # state_dict = self.audio_encoder.state_dict() + + # for k,v in wav2lip_state_dict.items(): + # if 'audio_encoder' in k: + # state_dict[k.replace('module.audio_encoder.', '')] = v + # self.audio_encoder.load_state_dict(state_dict) + + + def forward(self, audio_sequences): + # audio_sequences = (B, T, 1, 80, 16) + B = audio_sequences.size(0) + + audio_sequences = torch.cat([audio_sequences[:, i] for i in range(audio_sequences.size(1))], dim=0) + + audio_embedding = self.audio_encoder(audio_sequences) # B, 512, 1, 1 + dim = audio_embedding.shape[1] + audio_embedding = audio_embedding.reshape((B, -1, dim, 1, 1)) + + return audio_embedding.squeeze(-1).squeeze(-1) #B seq_len+1 512 diff --git a/src/audio2pose_models/cvae.py b/src/audio2pose_models/cvae.py new file mode 100644 index 0000000000000000000000000000000000000000..d017ce865a03bae40dfe066dbcd82e29839d89dc --- /dev/null +++ b/src/audio2pose_models/cvae.py @@ -0,0 +1,149 @@ +import torch +import torch.nn.functional as F +from torch import nn +from src.audio2pose_models.res_unet import ResUnet + +def class2onehot(idx, class_num): + + assert torch.max(idx).item() < class_num + onehot = torch.zeros(idx.size(0), class_num).to(idx.device) + onehot.scatter_(1, idx, 1) + return onehot + +class CVAE(nn.Module): + def __init__(self, cfg): + super().__init__() + encoder_layer_sizes = cfg.MODEL.CVAE.ENCODER_LAYER_SIZES + decoder_layer_sizes = cfg.MODEL.CVAE.DECODER_LAYER_SIZES + latent_size = cfg.MODEL.CVAE.LATENT_SIZE + num_classes = cfg.DATASET.NUM_CLASSES + audio_emb_in_size = cfg.MODEL.CVAE.AUDIO_EMB_IN_SIZE + audio_emb_out_size = cfg.MODEL.CVAE.AUDIO_EMB_OUT_SIZE + seq_len = cfg.MODEL.CVAE.SEQ_LEN + + self.latent_size = latent_size + + self.encoder = ENCODER(encoder_layer_sizes, latent_size, num_classes, + audio_emb_in_size, audio_emb_out_size, seq_len) + self.decoder = DECODER(decoder_layer_sizes, latent_size, num_classes, + audio_emb_in_size, audio_emb_out_size, seq_len) + def reparameterize(self, mu, logvar): + std = torch.exp(0.5 * logvar) + eps = torch.randn_like(std) + return mu + eps * std + + def forward(self, batch): + batch = self.encoder(batch) + mu = batch['mu'] + logvar = batch['logvar'] + z = self.reparameterize(mu, logvar) + batch['z'] = z + return self.decoder(batch) + + def test(self, batch): + ''' + class_id = batch['class'] + z = torch.randn([class_id.size(0), self.latent_size]).to(class_id.device) + batch['z'] = z + ''' + return self.decoder(batch) + +class ENCODER(nn.Module): + def __init__(self, layer_sizes, latent_size, num_classes, + audio_emb_in_size, audio_emb_out_size, seq_len): + super().__init__() + + self.resunet = ResUnet() + self.num_classes = num_classes + self.seq_len = seq_len + + self.MLP = nn.Sequential() + layer_sizes[0] += latent_size + seq_len*audio_emb_out_size + 6 + for i, (in_size, out_size) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])): + self.MLP.add_module( + name="L{:d}".format(i), module=nn.Linear(in_size, out_size)) + self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU()) + + self.linear_means = nn.Linear(layer_sizes[-1], latent_size) + self.linear_logvar = nn.Linear(layer_sizes[-1], latent_size) + self.linear_audio = nn.Linear(audio_emb_in_size, audio_emb_out_size) + + self.classbias = nn.Parameter(torch.randn(self.num_classes, latent_size)) + + def forward(self, batch): + class_id = batch['class'] + pose_motion_gt = batch['pose_motion_gt'] #bs seq_len 6 + ref = batch['ref'] #bs 6 + bs = pose_motion_gt.shape[0] + audio_in = batch['audio_emb'] # bs seq_len audio_emb_in_size + + #pose encode + pose_emb = self.resunet(pose_motion_gt.unsqueeze(1)) #bs 1 seq_len 6 + pose_emb = pose_emb.reshape(bs, -1) #bs seq_len*6 + + #audio mapping + print(audio_in.shape) + audio_out = self.linear_audio(audio_in) # bs seq_len audio_emb_out_size + audio_out = audio_out.reshape(bs, -1) + + class_bias = self.classbias[class_id] #bs latent_size + x_in = torch.cat([ref, pose_emb, audio_out, class_bias], dim=-1) #bs seq_len*(audio_emb_out_size+6)+latent_size + x_out = self.MLP(x_in) + + mu = self.linear_means(x_out) + logvar = self.linear_means(x_out) #bs latent_size + + batch.update({'mu':mu, 'logvar':logvar}) + return batch + +class DECODER(nn.Module): + def __init__(self, layer_sizes, latent_size, num_classes, + audio_emb_in_size, audio_emb_out_size, seq_len): + super().__init__() + + self.resunet = ResUnet() + self.num_classes = num_classes + self.seq_len = seq_len + + self.MLP = nn.Sequential() + input_size = latent_size + seq_len*audio_emb_out_size + 6 + for i, (in_size, out_size) in enumerate(zip([input_size]+layer_sizes[:-1], layer_sizes)): + self.MLP.add_module( + name="L{:d}".format(i), module=nn.Linear(in_size, out_size)) + if i+1 < len(layer_sizes): + self.MLP.add_module(name="A{:d}".format(i), module=nn.ReLU()) + else: + self.MLP.add_module(name="sigmoid", module=nn.Sigmoid()) + + self.pose_linear = nn.Linear(6, 6) + self.linear_audio = nn.Linear(audio_emb_in_size, audio_emb_out_size) + + self.classbias = nn.Parameter(torch.randn(self.num_classes, latent_size)) + + def forward(self, batch): + + z = batch['z'] #bs latent_size + bs = z.shape[0] + class_id = batch['class'] + ref = batch['ref'] #bs 6 + audio_in = batch['audio_emb'] # bs seq_len audio_emb_in_size + #print('audio_in: ', audio_in[:, :, :10]) + + audio_out = self.linear_audio(audio_in) # bs seq_len audio_emb_out_size + #print('audio_out: ', audio_out[:, :, :10]) + audio_out = audio_out.reshape([bs, -1]) # bs seq_len*audio_emb_out_size + class_bias = self.classbias[class_id] #bs latent_size + + z = z + class_bias + x_in = torch.cat([ref, z, audio_out], dim=-1) + x_out = self.MLP(x_in) # bs layer_sizes[-1] + x_out = x_out.reshape((bs, self.seq_len, -1)) + + #print('x_out: ', x_out) + + pose_emb = self.resunet(x_out.unsqueeze(1)) #bs 1 seq_len 6 + + pose_motion_pred = self.pose_linear(pose_emb.squeeze(1)) #bs seq_len 6 + + batch.update({'pose_motion_pred':pose_motion_pred}) + return batch diff --git a/src/audio2pose_models/discriminator.py b/src/audio2pose_models/discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..339c38e4812ff38a810f0f3a1c01812f6d5d78db --- /dev/null +++ b/src/audio2pose_models/discriminator.py @@ -0,0 +1,76 @@ +import torch +import torch.nn.functional as F +from torch import nn + +class ConvNormRelu(nn.Module): + def __init__(self, conv_type='1d', in_channels=3, out_channels=64, downsample=False, + kernel_size=None, stride=None, padding=None, norm='BN', leaky=False): + super().__init__() + if kernel_size is None: + if downsample: + kernel_size, stride, padding = 4, 2, 1 + else: + kernel_size, stride, padding = 3, 1, 1 + + if conv_type == '2d': + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride, + padding, + bias=False, + ) + if norm == 'BN': + self.norm = nn.BatchNorm2d(out_channels) + elif norm == 'IN': + self.norm = nn.InstanceNorm2d(out_channels) + else: + raise NotImplementedError + elif conv_type == '1d': + self.conv = nn.Conv1d( + in_channels, + out_channels, + kernel_size, + stride, + padding, + bias=False, + ) + if norm == 'BN': + self.norm = nn.BatchNorm1d(out_channels) + elif norm == 'IN': + self.norm = nn.InstanceNorm1d(out_channels) + else: + raise NotImplementedError + nn.init.kaiming_normal_(self.conv.weight) + + self.act = nn.LeakyReLU(negative_slope=0.2, inplace=False) if leaky else nn.ReLU(inplace=True) + + def forward(self, x): + x = self.conv(x) + if isinstance(self.norm, nn.InstanceNorm1d): + x = self.norm(x.permute((0, 2, 1))).permute((0, 2, 1)) # normalize on [C] + else: + x = self.norm(x) + x = self.act(x) + return x + + +class PoseSequenceDiscriminator(nn.Module): + def __init__(self, cfg): + super().__init__() + self.cfg = cfg + leaky = self.cfg.MODEL.DISCRIMINATOR.LEAKY_RELU + + self.seq = nn.Sequential( + ConvNormRelu('1d', cfg.MODEL.DISCRIMINATOR.INPUT_CHANNELS, 256, downsample=True, leaky=leaky), # B, 256, 64 + ConvNormRelu('1d', 256, 512, downsample=True, leaky=leaky), # B, 512, 32 + ConvNormRelu('1d', 512, 1024, kernel_size=3, stride=1, padding=1, leaky=leaky), # B, 1024, 16 + nn.Conv1d(1024, 1, kernel_size=3, stride=1, padding=1, bias=True) # B, 1, 16 + ) + + def forward(self, x): + x = x.reshape(x.size(0), x.size(1), -1).transpose(1, 2) + x = self.seq(x) + x = x.squeeze(1) + return x \ No newline at end of file diff --git a/src/audio2pose_models/networks.py b/src/audio2pose_models/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..8aa0b1390e7b4bb0e16057ac94d2fe84f48421af --- /dev/null +++ b/src/audio2pose_models/networks.py @@ -0,0 +1,140 @@ +import torch.nn as nn +import torch + + +class ResidualConv(nn.Module): + def __init__(self, input_dim, output_dim, stride, padding): + super(ResidualConv, self).__init__() + + self.conv_block = nn.Sequential( + nn.BatchNorm2d(input_dim), + nn.ReLU(), + nn.Conv2d( + input_dim, output_dim, kernel_size=3, stride=stride, padding=padding + ), + nn.BatchNorm2d(output_dim), + nn.ReLU(), + nn.Conv2d(output_dim, output_dim, kernel_size=3, padding=1), + ) + self.conv_skip = nn.Sequential( + nn.Conv2d(input_dim, output_dim, kernel_size=3, stride=stride, padding=1), + nn.BatchNorm2d(output_dim), + ) + + def forward(self, x): + + return self.conv_block(x) + self.conv_skip(x) + + +class Upsample(nn.Module): + def __init__(self, input_dim, output_dim, kernel, stride): + super(Upsample, self).__init__() + + self.upsample = nn.ConvTranspose2d( + input_dim, output_dim, kernel_size=kernel, stride=stride + ) + + def forward(self, x): + return self.upsample(x) + + +class Squeeze_Excite_Block(nn.Module): + def __init__(self, channel, reduction=16): + super(Squeeze_Excite_Block, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential( + nn.Linear(channel, channel // reduction, bias=False), + nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel, bias=False), + nn.Sigmoid(), + ) + + def forward(self, x): + b, c, _, _ = x.size() + y = self.avg_pool(x).view(b, c) + y = self.fc(y).view(b, c, 1, 1) + return x * y.expand_as(x) + + +class ASPP(nn.Module): + def __init__(self, in_dims, out_dims, rate=[6, 12, 18]): + super(ASPP, self).__init__() + + self.aspp_block1 = nn.Sequential( + nn.Conv2d( + in_dims, out_dims, 3, stride=1, padding=rate[0], dilation=rate[0] + ), + nn.ReLU(inplace=True), + nn.BatchNorm2d(out_dims), + ) + self.aspp_block2 = nn.Sequential( + nn.Conv2d( + in_dims, out_dims, 3, stride=1, padding=rate[1], dilation=rate[1] + ), + nn.ReLU(inplace=True), + nn.BatchNorm2d(out_dims), + ) + self.aspp_block3 = nn.Sequential( + nn.Conv2d( + in_dims, out_dims, 3, stride=1, padding=rate[2], dilation=rate[2] + ), + nn.ReLU(inplace=True), + nn.BatchNorm2d(out_dims), + ) + + self.output = nn.Conv2d(len(rate) * out_dims, out_dims, 1) + self._init_weights() + + def forward(self, x): + x1 = self.aspp_block1(x) + x2 = self.aspp_block2(x) + x3 = self.aspp_block3(x) + out = torch.cat([x1, x2, x3], dim=1) + return self.output(out) + + def _init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + +class Upsample_(nn.Module): + def __init__(self, scale=2): + super(Upsample_, self).__init__() + + self.upsample = nn.Upsample(mode="bilinear", scale_factor=scale) + + def forward(self, x): + return self.upsample(x) + + +class AttentionBlock(nn.Module): + def __init__(self, input_encoder, input_decoder, output_dim): + super(AttentionBlock, self).__init__() + + self.conv_encoder = nn.Sequential( + nn.BatchNorm2d(input_encoder), + nn.ReLU(), + nn.Conv2d(input_encoder, output_dim, 3, padding=1), + nn.MaxPool2d(2, 2), + ) + + self.conv_decoder = nn.Sequential( + nn.BatchNorm2d(input_decoder), + nn.ReLU(), + nn.Conv2d(input_decoder, output_dim, 3, padding=1), + ) + + self.conv_attn = nn.Sequential( + nn.BatchNorm2d(output_dim), + nn.ReLU(), + nn.Conv2d(output_dim, 1, 1), + ) + + def forward(self, x1, x2): + out = self.conv_encoder(x1) + self.conv_decoder(x2) + out = self.conv_attn(out) + return out * x2 \ No newline at end of file diff --git a/src/audio2pose_models/res_unet.py b/src/audio2pose_models/res_unet.py new file mode 100644 index 0000000000000000000000000000000000000000..f2611e1d1a9bf233507427b34928fca60e094224 --- /dev/null +++ b/src/audio2pose_models/res_unet.py @@ -0,0 +1,65 @@ +import torch +import torch.nn as nn +from src.audio2pose_models.networks import ResidualConv, Upsample + + +class ResUnet(nn.Module): + def __init__(self, channel=1, filters=[32, 64, 128, 256]): + super(ResUnet, self).__init__() + + self.input_layer = nn.Sequential( + nn.Conv2d(channel, filters[0], kernel_size=3, padding=1), + nn.BatchNorm2d(filters[0]), + nn.ReLU(), + nn.Conv2d(filters[0], filters[0], kernel_size=3, padding=1), + ) + self.input_skip = nn.Sequential( + nn.Conv2d(channel, filters[0], kernel_size=3, padding=1) + ) + + self.residual_conv_1 = ResidualConv(filters[0], filters[1], stride=(2,1), padding=1) + self.residual_conv_2 = ResidualConv(filters[1], filters[2], stride=(2,1), padding=1) + + self.bridge = ResidualConv(filters[2], filters[3], stride=(2,1), padding=1) + + self.upsample_1 = Upsample(filters[3], filters[3], kernel=(2,1), stride=(2,1)) + self.up_residual_conv1 = ResidualConv(filters[3] + filters[2], filters[2], stride=1, padding=1) + + self.upsample_2 = Upsample(filters[2], filters[2], kernel=(2,1), stride=(2,1)) + self.up_residual_conv2 = ResidualConv(filters[2] + filters[1], filters[1], stride=1, padding=1) + + self.upsample_3 = Upsample(filters[1], filters[1], kernel=(2,1), stride=(2,1)) + self.up_residual_conv3 = ResidualConv(filters[1] + filters[0], filters[0], stride=1, padding=1) + + self.output_layer = nn.Sequential( + nn.Conv2d(filters[0], 1, 1, 1), + nn.Sigmoid(), + ) + + def forward(self, x): + # Encode + x1 = self.input_layer(x) + self.input_skip(x) + x2 = self.residual_conv_1(x1) + x3 = self.residual_conv_2(x2) + # Bridge + x4 = self.bridge(x3) + + # Decode + x4 = self.upsample_1(x4) + x5 = torch.cat([x4, x3], dim=1) + + x6 = self.up_residual_conv1(x5) + + x6 = self.upsample_2(x6) + x7 = torch.cat([x6, x2], dim=1) + + x8 = self.up_residual_conv2(x7) + + x8 = self.upsample_3(x8) + x9 = torch.cat([x8, x1], dim=1) + + x10 = self.up_residual_conv3(x9) + + output = self.output_layer(x10) + + return output \ No newline at end of file diff --git a/src/config/auido2exp.yaml b/src/config/auido2exp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7369dbf350476e14a1d600507f1f8b7d8aa6ecd3 --- /dev/null +++ b/src/config/auido2exp.yaml @@ -0,0 +1,58 @@ +DATASET: + TRAIN_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/file_list/train.txt + EVAL_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/file_list/val.txt + TRAIN_BATCH_SIZE: 32 + EVAL_BATCH_SIZE: 32 + EXP: True + EXP_DIM: 64 + FRAME_LEN: 32 + COEFF_LEN: 73 + NUM_CLASSES: 46 + AUDIO_ROOT_PATH: /apdcephfs_cq2/share_1290939/wenxuazhang/voxceleb1/wav + COEFF_ROOT_PATH: /apdcephfs_cq2/share_1290939/wenxuazhang/voxceleb1/wav2lip_3dmm + LMDB_PATH: /apdcephfs_cq2/share_1290939/shadowcun/datasets/VoxCeleb/v1/imdb + DEBUG: True + NUM_REPEATS: 2 + T: 40 + + +MODEL: + FRAMEWORK: V2 + AUDIOENCODER: + LEAKY_RELU: True + NORM: 'IN' + DISCRIMINATOR: + LEAKY_RELU: False + INPUT_CHANNELS: 6 + CVAE: + AUDIO_EMB_IN_SIZE: 512 + AUDIO_EMB_OUT_SIZE: 128 + SEQ_LEN: 32 + LATENT_SIZE: 256 + ENCODER_LAYER_SIZES: [192, 1024] + DECODER_LAYER_SIZES: [1024, 192] + + +TRAIN: + MAX_EPOCH: 300 + GENERATOR: + LR: 2.0e-5 + DISCRIMINATOR: + LR: 1.0e-5 + LOSS: + W_FEAT: 0 + W_COEFF_EXP: 2 + W_LM: 1.0e-2 + W_LM_MOUTH: 0 + W_REG: 0 + W_SYNC: 0 + W_COLOR: 0 + W_EXPRESSION: 0 + W_LIPREADING: 0.01 + W_LIPREADING_VV: 0 + W_EYE_BLINK: 4 + +TAG: + NAME: small_dataset + + diff --git a/src/config/auido2pose.yaml b/src/config/auido2pose.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc61f94d12f406f2d8d02545e55b61075051484d --- /dev/null +++ b/src/config/auido2pose.yaml @@ -0,0 +1,49 @@ +DATASET: + TRAIN_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/audio2pose_unet_noAudio/dataset/train_33.txt + EVAL_FILE_LIST: /apdcephfs_cq2/share_1290939/wenxuazhang/code/audio2pose_unet_noAudio/dataset/val.txt + TRAIN_BATCH_SIZE: 64 + EVAL_BATCH_SIZE: 1 + EXP: True + EXP_DIM: 64 + FRAME_LEN: 32 + COEFF_LEN: 73 + NUM_CLASSES: 46 + AUDIO_ROOT_PATH: /apdcephfs_cq2/share_1290939/wenxuazhang/voxceleb1/wav + COEFF_ROOT_PATH: /apdcephfs_cq2/share_1290939/shadowcun/datasets/VoxCeleb/v1/imdb + DEBUG: True + + +MODEL: + AUDIOENCODER: + LEAKY_RELU: True + NORM: 'IN' + DISCRIMINATOR: + LEAKY_RELU: False + INPUT_CHANNELS: 6 + CVAE: + AUDIO_EMB_IN_SIZE: 512 + AUDIO_EMB_OUT_SIZE: 6 + SEQ_LEN: 32 + LATENT_SIZE: 64 + ENCODER_LAYER_SIZES: [192, 128] + DECODER_LAYER_SIZES: [128, 192] + + +TRAIN: + MAX_EPOCH: 150 + GENERATOR: + LR: 1.0e-4 + DISCRIMINATOR: + LR: 1.0e-4 + LOSS: + LAMBDA_REG: 1 + LAMBDA_LANDMARKS: 0 + LAMBDA_VERTICES: 0 + LAMBDA_GAN_MOTION: 0.7 + LAMBDA_GAN_COEFF: 0 + LAMBDA_KL: 1 + +TAG: + NAME: cvae_UNET_useAudio_usewav2lipAudioEncoder + + diff --git a/src/config/facerender.yaml b/src/config/facerender.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9494ef82dfa16b16b7aa0b848ebdd6b23e739e2a --- /dev/null +++ b/src/config/facerender.yaml @@ -0,0 +1,45 @@ +model_params: + common_params: + num_kp: 15 + image_channel: 3 + feature_channel: 32 + estimate_jacobian: False # True + kp_detector_params: + temperature: 0.1 + block_expansion: 32 + max_features: 1024 + scale_factor: 0.25 # 0.25 + num_blocks: 5 + reshape_channel: 16384 # 16384 = 1024 * 16 + reshape_depth: 16 + he_estimator_params: + block_expansion: 64 + max_features: 2048 + num_bins: 66 + generator_params: + block_expansion: 64 + max_features: 512 + num_down_blocks: 2 + reshape_channel: 32 + reshape_depth: 16 # 512 = 32 * 16 + num_resblocks: 6 + estimate_occlusion_map: True + dense_motion_params: + block_expansion: 32 + max_features: 1024 + num_blocks: 5 + reshape_depth: 16 + compress: 4 + discriminator_params: + scales: [1] + block_expansion: 32 + max_features: 512 + num_blocks: 4 + sn: True + mapping_params: + coeff_nc: 70 + descriptor_nc: 1024 + layer: 3 + num_kp: 15 + num_bins: 66 + diff --git a/src/config/facerender_still.yaml b/src/config/facerender_still.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b4d66dade3e655ac4cfc25a994ca28e53821d80 --- /dev/null +++ b/src/config/facerender_still.yaml @@ -0,0 +1,45 @@ +model_params: + common_params: + num_kp: 15 + image_channel: 3 + feature_channel: 32 + estimate_jacobian: False # True + kp_detector_params: + temperature: 0.1 + block_expansion: 32 + max_features: 1024 + scale_factor: 0.25 # 0.25 + num_blocks: 5 + reshape_channel: 16384 # 16384 = 1024 * 16 + reshape_depth: 16 + he_estimator_params: + block_expansion: 64 + max_features: 2048 + num_bins: 66 + generator_params: + block_expansion: 64 + max_features: 512 + num_down_blocks: 2 + reshape_channel: 32 + reshape_depth: 16 # 512 = 32 * 16 + num_resblocks: 6 + estimate_occlusion_map: True + dense_motion_params: + block_expansion: 32 + max_features: 1024 + num_blocks: 5 + reshape_depth: 16 + compress: 4 + discriminator_params: + scales: [1] + block_expansion: 32 + max_features: 512 + num_blocks: 4 + sn: True + mapping_params: + coeff_nc: 73 + descriptor_nc: 1024 + layer: 3 + num_kp: 15 + num_bins: 66 + diff --git a/src/config/similarity_Lm3D_all.mat b/src/config/similarity_Lm3D_all.mat new file mode 100644 index 0000000000000000000000000000000000000000..a0e23588302bc71fc899eef53ff06df5f4df4c1d Binary files /dev/null and b/src/config/similarity_Lm3D_all.mat differ diff --git a/src/face3d/__pycache__/extract_kp_videos_safe.cpython-310.pyc b/src/face3d/__pycache__/extract_kp_videos_safe.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..370acde35340c7b78b81f2b8030215a15467b01e Binary files /dev/null and b/src/face3d/__pycache__/extract_kp_videos_safe.cpython-310.pyc differ diff --git a/src/face3d/data/__init__.py b/src/face3d/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9a9761c518a1b07c5996165869742af0a52c82bc --- /dev/null +++ b/src/face3d/data/__init__.py @@ -0,0 +1,116 @@ +"""This package includes all the modules related to data loading and preprocessing + + To add a custom dataset class called 'dummy', you need to add a file called 'dummy_dataset.py' and define a subclass 'DummyDataset' inherited from BaseDataset. + You need to implement four functions: + -- <__init__>: initialize the class, first call BaseDataset.__init__(self, opt). + -- <__len__>: return the size of dataset. + -- <__getitem__>: get a data point from data loader. + -- : (optionally) add dataset-specific options and set default options. + +Now you can use the dataset class by specifying flag '--dataset_mode dummy'. +See our template dataset class 'template_dataset.py' for more details. +""" +import numpy as np +import importlib +import torch.utils.data +from face3d.data.base_dataset import BaseDataset + + +def find_dataset_using_name(dataset_name): + """Import the module "data/[dataset_name]_dataset.py". + + In the file, the class called DatasetNameDataset() will + be instantiated. It has to be a subclass of BaseDataset, + and it is case-insensitive. + """ + dataset_filename = "data." + dataset_name + "_dataset" + datasetlib = importlib.import_module(dataset_filename) + + dataset = None + target_dataset_name = dataset_name.replace('_', '') + 'dataset' + for name, cls in datasetlib.__dict__.items(): + if name.lower() == target_dataset_name.lower() \ + and issubclass(cls, BaseDataset): + dataset = cls + + if dataset is None: + raise NotImplementedError("In %s.py, there should be a subclass of BaseDataset with class name that matches %s in lowercase." % (dataset_filename, target_dataset_name)) + + return dataset + + +def get_option_setter(dataset_name): + """Return the static method of the dataset class.""" + dataset_class = find_dataset_using_name(dataset_name) + return dataset_class.modify_commandline_options + + +def create_dataset(opt, rank=0): + """Create a dataset given the option. + + This function wraps the class CustomDatasetDataLoader. + This is the main interface between this package and 'train.py'/'test.py' + + Example: + >>> from data import create_dataset + >>> dataset = create_dataset(opt) + """ + data_loader = CustomDatasetDataLoader(opt, rank=rank) + dataset = data_loader.load_data() + return dataset + +class CustomDatasetDataLoader(): + """Wrapper class of Dataset class that performs multi-threaded data loading""" + + def __init__(self, opt, rank=0): + """Initialize this class + + Step 1: create a dataset instance given the name [dataset_mode] + Step 2: create a multi-threaded data loader. + """ + self.opt = opt + dataset_class = find_dataset_using_name(opt.dataset_mode) + self.dataset = dataset_class(opt) + self.sampler = None + print("rank %d %s dataset [%s] was created" % (rank, self.dataset.name, type(self.dataset).__name__)) + if opt.use_ddp and opt.isTrain: + world_size = opt.world_size + self.sampler = torch.utils.data.distributed.DistributedSampler( + self.dataset, + num_replicas=world_size, + rank=rank, + shuffle=not opt.serial_batches + ) + self.dataloader = torch.utils.data.DataLoader( + self.dataset, + sampler=self.sampler, + num_workers=int(opt.num_threads / world_size), + batch_size=int(opt.batch_size / world_size), + drop_last=True) + else: + self.dataloader = torch.utils.data.DataLoader( + self.dataset, + batch_size=opt.batch_size, + shuffle=(not opt.serial_batches) and opt.isTrain, + num_workers=int(opt.num_threads), + drop_last=True + ) + + def set_epoch(self, epoch): + self.dataset.current_epoch = epoch + if self.sampler is not None: + self.sampler.set_epoch(epoch) + + def load_data(self): + return self + + def __len__(self): + """Return the number of data in the dataset""" + return min(len(self.dataset), self.opt.max_dataset_size) + + def __iter__(self): + """Return a batch of data""" + for i, data in enumerate(self.dataloader): + if i * self.opt.batch_size >= self.opt.max_dataset_size: + break + yield data diff --git a/src/face3d/data/base_dataset.py b/src/face3d/data/base_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..1bd57d082d519f512d7114b4f867b6695fb7de06 --- /dev/null +++ b/src/face3d/data/base_dataset.py @@ -0,0 +1,125 @@ +"""This module implements an abstract base class (ABC) 'BaseDataset' for datasets. + +It also includes common transformation functions (e.g., get_transform, __scale_width), which can be later used in subclasses. +""" +import random +import numpy as np +import torch.utils.data as data +from PIL import Image +import torchvision.transforms as transforms +from abc import ABC, abstractmethod + + +class BaseDataset(data.Dataset, ABC): + """This class is an abstract base class (ABC) for datasets. + + To create a subclass, you need to implement the following four functions: + -- <__init__>: initialize the class, first call BaseDataset.__init__(self, opt). + -- <__len__>: return the size of dataset. + -- <__getitem__>: get a data point. + -- : (optionally) add dataset-specific options and set default options. + """ + + def __init__(self, opt): + """Initialize the class; save the options in the class + + Parameters: + opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions + """ + self.opt = opt + # self.root = opt.dataroot + self.current_epoch = 0 + + @staticmethod + def modify_commandline_options(parser, is_train): + """Add new dataset-specific options, and rewrite default values for existing options. + + Parameters: + parser -- original option parser + is_train (bool) -- whether training phase or test phase. You can use this flag to add training-specific or test-specific options. + + Returns: + the modified parser. + """ + return parser + + @abstractmethod + def __len__(self): + """Return the total number of images in the dataset.""" + return 0 + + @abstractmethod + def __getitem__(self, index): + """Return a data point and its metadata information. + + Parameters: + index - - a random integer for data indexing + + Returns: + a dictionary of data with their names. It ususally contains the data itself and its metadata information. + """ + pass + + +def get_transform(grayscale=False): + transform_list = [] + if grayscale: + transform_list.append(transforms.Grayscale(1)) + transform_list += [transforms.ToTensor()] + return transforms.Compose(transform_list) + +def get_affine_mat(opt, size): + shift_x, shift_y, scale, rot_angle, flip = 0., 0., 1., 0., False + w, h = size + + if 'shift' in opt.preprocess: + shift_pixs = int(opt.shift_pixs) + shift_x = random.randint(-shift_pixs, shift_pixs) + shift_y = random.randint(-shift_pixs, shift_pixs) + if 'scale' in opt.preprocess: + scale = 1 + opt.scale_delta * (2 * random.random() - 1) + if 'rot' in opt.preprocess: + rot_angle = opt.rot_angle * (2 * random.random() - 1) + rot_rad = -rot_angle * np.pi/180 + if 'flip' in opt.preprocess: + flip = random.random() > 0.5 + + shift_to_origin = np.array([1, 0, -w//2, 0, 1, -h//2, 0, 0, 1]).reshape([3, 3]) + flip_mat = np.array([-1 if flip else 1, 0, 0, 0, 1, 0, 0, 0, 1]).reshape([3, 3]) + shift_mat = np.array([1, 0, shift_x, 0, 1, shift_y, 0, 0, 1]).reshape([3, 3]) + rot_mat = np.array([np.cos(rot_rad), np.sin(rot_rad), 0, -np.sin(rot_rad), np.cos(rot_rad), 0, 0, 0, 1]).reshape([3, 3]) + scale_mat = np.array([scale, 0, 0, 0, scale, 0, 0, 0, 1]).reshape([3, 3]) + shift_to_center = np.array([1, 0, w//2, 0, 1, h//2, 0, 0, 1]).reshape([3, 3]) + + affine = shift_to_center @ scale_mat @ rot_mat @ shift_mat @ flip_mat @ shift_to_origin + affine_inv = np.linalg.inv(affine) + return affine, affine_inv, flip + +def apply_img_affine(img, affine_inv, method=Image.BICUBIC): + return img.transform(img.size, Image.AFFINE, data=affine_inv.flatten()[:6], resample=Image.BICUBIC) + +def apply_lm_affine(landmark, affine, flip, size): + _, h = size + lm = landmark.copy() + lm[:, 1] = h - 1 - lm[:, 1] + lm = np.concatenate((lm, np.ones([lm.shape[0], 1])), -1) + lm = lm @ np.transpose(affine) + lm[:, :2] = lm[:, :2] / lm[:, 2:] + lm = lm[:, :2] + lm[:, 1] = h - 1 - lm[:, 1] + if flip: + lm_ = lm.copy() + lm_[:17] = lm[16::-1] + lm_[17:22] = lm[26:21:-1] + lm_[22:27] = lm[21:16:-1] + lm_[31:36] = lm[35:30:-1] + lm_[36:40] = lm[45:41:-1] + lm_[40:42] = lm[47:45:-1] + lm_[42:46] = lm[39:35:-1] + lm_[46:48] = lm[41:39:-1] + lm_[48:55] = lm[54:47:-1] + lm_[55:60] = lm[59:54:-1] + lm_[60:65] = lm[64:59:-1] + lm_[65:68] = lm[67:64:-1] + lm = lm_ + return lm diff --git a/src/face3d/data/flist_dataset.py b/src/face3d/data/flist_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..c0b6945c80aa756074a5d3c02b9443b15ddcfc57 --- /dev/null +++ b/src/face3d/data/flist_dataset.py @@ -0,0 +1,125 @@ +"""This script defines the custom dataset for Deep3DFaceRecon_pytorch +""" + +import os.path +from data.base_dataset import BaseDataset, get_transform, get_affine_mat, apply_img_affine, apply_lm_affine +from data.image_folder import make_dataset +from PIL import Image +import random +import util.util as util +import numpy as np +import json +import torch +from scipy.io import loadmat, savemat +import pickle +from util.preprocess import align_img, estimate_norm +from util.load_mats import load_lm3d + + +def default_flist_reader(flist): + """ + flist format: impath label\nimpath label\n ...(same to caffe's filelist) + """ + imlist = [] + with open(flist, 'r') as rf: + for line in rf.readlines(): + impath = line.strip() + imlist.append(impath) + + return imlist + +def jason_flist_reader(flist): + with open(flist, 'r') as fp: + info = json.load(fp) + return info + +def parse_label(label): + return torch.tensor(np.array(label).astype(np.float32)) + + +class FlistDataset(BaseDataset): + """ + It requires one directories to host training images '/path/to/data/train' + You can train the model with the dataset flag '--dataroot /path/to/data'. + """ + + def __init__(self, opt): + """Initialize this dataset class. + + Parameters: + opt (Option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions + """ + BaseDataset.__init__(self, opt) + + self.lm3d_std = load_lm3d(opt.bfm_folder) + + msk_names = default_flist_reader(opt.flist) + self.msk_paths = [os.path.join(opt.data_root, i) for i in msk_names] + + self.size = len(self.msk_paths) + self.opt = opt + + self.name = 'train' if opt.isTrain else 'val' + if '_' in opt.flist: + self.name += '_' + opt.flist.split(os.sep)[-1].split('_')[0] + + + def __getitem__(self, index): + """Return a data point and its metadata information. + + Parameters: + index (int) -- a random integer for data indexing + + Returns a dictionary that contains A, B, A_paths and B_paths + img (tensor) -- an image in the input domain + msk (tensor) -- its corresponding attention mask + lm (tensor) -- its corresponding 3d landmarks + im_paths (str) -- image paths + aug_flag (bool) -- a flag used to tell whether its raw or augmented + """ + msk_path = self.msk_paths[index % self.size] # make sure index is within then range + img_path = msk_path.replace('mask/', '') + lm_path = '.'.join(msk_path.replace('mask', 'landmarks').split('.')[:-1]) + '.txt' + + raw_img = Image.open(img_path).convert('RGB') + raw_msk = Image.open(msk_path).convert('RGB') + raw_lm = np.loadtxt(lm_path).astype(np.float32) + + _, img, lm, msk = align_img(raw_img, raw_lm, self.lm3d_std, raw_msk) + + aug_flag = self.opt.use_aug and self.opt.isTrain + if aug_flag: + img, lm, msk = self._augmentation(img, lm, self.opt, msk) + + _, H = img.size + M = estimate_norm(lm, H) + transform = get_transform() + img_tensor = transform(img) + msk_tensor = transform(msk)[:1, ...] + lm_tensor = parse_label(lm) + M_tensor = parse_label(M) + + + return {'imgs': img_tensor, + 'lms': lm_tensor, + 'msks': msk_tensor, + 'M': M_tensor, + 'im_paths': img_path, + 'aug_flag': aug_flag, + 'dataset': self.name} + + def _augmentation(self, img, lm, opt, msk=None): + affine, affine_inv, flip = get_affine_mat(opt, img.size) + img = apply_img_affine(img, affine_inv) + lm = apply_lm_affine(lm, affine, flip, img.size) + if msk is not None: + msk = apply_img_affine(msk, affine_inv, method=Image.BILINEAR) + return img, lm, msk + + + + + def __len__(self): + """Return the total number of images in the dataset. + """ + return self.size diff --git a/src/face3d/data/image_folder.py b/src/face3d/data/image_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..efadc2ecbe2fb4b53b78230aba25ec505eff0e55 --- /dev/null +++ b/src/face3d/data/image_folder.py @@ -0,0 +1,66 @@ +"""A modified image folder class + +We modify the official PyTorch image folder (https://github.com/pytorch/vision/blob/master/torchvision/datasets/folder.py) +so that this class can load images from both current directory and its subdirectories. +""" +import numpy as np +import torch.utils.data as data + +from PIL import Image +import os +import os.path + +IMG_EXTENSIONS = [ + '.jpg', '.JPG', '.jpeg', '.JPEG', + '.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', + '.tif', '.TIF', '.tiff', '.TIFF', +] + + +def is_image_file(filename): + return any(filename.endswith(extension) for extension in IMG_EXTENSIONS) + + +def make_dataset(dir, max_dataset_size=float("inf")): + images = [] + assert os.path.isdir(dir) or os.path.islink(dir), '%s is not a valid directory' % dir + + for root, _, fnames in sorted(os.walk(dir, followlinks=True)): + for fname in fnames: + if is_image_file(fname): + path = os.path.join(root, fname) + images.append(path) + return images[:min(max_dataset_size, len(images))] + + +def default_loader(path): + return Image.open(path).convert('RGB') + + +class ImageFolder(data.Dataset): + + def __init__(self, root, transform=None, return_paths=False, + loader=default_loader): + imgs = make_dataset(root) + if len(imgs) == 0: + raise(RuntimeError("Found 0 images in: " + root + "\n" + "Supported image extensions are: " + ",".join(IMG_EXTENSIONS))) + + self.root = root + self.imgs = imgs + self.transform = transform + self.return_paths = return_paths + self.loader = loader + + def __getitem__(self, index): + path = self.imgs[index] + img = self.loader(path) + if self.transform is not None: + img = self.transform(img) + if self.return_paths: + return img, path + else: + return img + + def __len__(self): + return len(self.imgs) diff --git a/src/face3d/data/template_dataset.py b/src/face3d/data/template_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..bfdf16be2a8a834b204c45d88c86857b37b9bd25 --- /dev/null +++ b/src/face3d/data/template_dataset.py @@ -0,0 +1,75 @@ +"""Dataset class template + +This module provides a template for users to implement custom datasets. +You can specify '--dataset_mode template' to use this dataset. +The class name should be consistent with both the filename and its dataset_mode option. +The filename should be _dataset.py +The class name should be Dataset.py +You need to implement the following functions: + -- :　Add dataset-specific options and rewrite default values for existing options. + -- <__init__>: Initialize this dataset class. + -- <__getitem__>: Return a data point and its metadata information. + -- <__len__>: Return the number of images. +""" +from data.base_dataset import BaseDataset, get_transform +# from data.image_folder import make_dataset +# from PIL import Image + + +class TemplateDataset(BaseDataset): + """A template dataset class for you to implement custom datasets.""" + @staticmethod + def modify_commandline_options(parser, is_train): + """Add new dataset-specific options, and rewrite default values for existing options. + + Parameters: + parser -- original option parser + is_train (bool) -- whether training phase or test phase. You can use this flag to add training-specific or test-specific options. + + Returns: + the modified parser. + """ + parser.add_argument('--new_dataset_option', type=float, default=1.0, help='new dataset option') + parser.set_defaults(max_dataset_size=10, new_dataset_option=2.0) # specify dataset-specific default values + return parser + + def __init__(self, opt): + """Initialize this dataset class. + + Parameters: + opt (Option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions + + A few things can be done here. + - save the options (have been done in BaseDataset) + - get image paths and meta information of the dataset. + - define the image transformation. + """ + # save the option and dataset root + BaseDataset.__init__(self, opt) + # get the image paths of your dataset; + self.image_paths = [] # You can call sorted(make_dataset(self.root, opt.max_dataset_size)) to get all the image paths under the directory self.root + # define the default transform function. You can use ; You can also define your custom transform function + self.transform = get_transform(opt) + + def __getitem__(self, index): + """Return a data point and its metadata information. + + Parameters: + index -- a random integer for data indexing + + Returns: + a dictionary of data with their names. It usually contains the data itself and its metadata information. + + Step 1: get a random image path: e.g., path = self.image_paths[index] + Step 2: load your data from the disk: e.g., image = Image.open(path).convert('RGB'). + Step 3: convert your data to a PyTorch tensor. You can use helpder functions such as self.transform. e.g., data = self.transform(image) + Step 4: return a data point as a dictionary. + """ + path = 'temp' # needs to be a string + data_A = None # needs to be a tensor + data_B = None # needs to be a tensor + return {'data_A': data_A, 'data_B': data_B, 'path': path} + + def __len__(self): + """Return the total number of images.""" + return len(self.image_paths) diff --git a/src/face3d/extract_kp_videos.py b/src/face3d/extract_kp_videos.py new file mode 100644 index 0000000000000000000000000000000000000000..21616a3b4b5077ffdce99621395237b4edcff58c --- /dev/null +++ b/src/face3d/extract_kp_videos.py @@ -0,0 +1,108 @@ +import os +import cv2 +import time +import glob +import argparse +import face_alignment +import numpy as np +from PIL import Image +from tqdm import tqdm +from itertools import cycle + +from torch.multiprocessing import Pool, Process, set_start_method + +class KeypointExtractor(): + def __init__(self, device): + self.detector = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, + device=device) + + def extract_keypoint(self, images, name=None, info=True): + if isinstance(images, list): + keypoints = [] + if info: + i_range = tqdm(images,desc='landmark Det:') + else: + i_range = images + + for image in i_range: + current_kp = self.extract_keypoint(image) + if np.mean(current_kp) == -1 and keypoints: + keypoints.append(keypoints[-1]) + else: + keypoints.append(current_kp[None]) + + keypoints = np.concatenate(keypoints, 0) + np.savetxt(os.path.splitext(name)[0]+'.txt', keypoints.reshape(-1)) + return keypoints + else: + while True: + try: + keypoints = self.detector.get_landmarks_from_image(np.array(images))[0] + break + except RuntimeError as e: + if str(e).startswith('CUDA'): + print("Warning: out of memory, sleep for 1s") + time.sleep(1) + else: + print(e) + break + except TypeError: + print('No face detected in this image') + shape = [68, 2] + keypoints = -1. * np.ones(shape) + break + if name is not None: + np.savetxt(os.path.splitext(name)[0]+'.txt', keypoints.reshape(-1)) + return keypoints + +def read_video(filename): + frames = [] + cap = cv2.VideoCapture(filename) + while cap.isOpened(): + ret, frame = cap.read() + if ret: + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + frame = Image.fromarray(frame) + frames.append(frame) + else: + break + cap.release() + return frames + +def run(data): + filename, opt, device = data + os.environ['CUDA_VISIBLE_DEVICES'] = device + kp_extractor = KeypointExtractor() + images = read_video(filename) + name = filename.split('/')[-2:] + os.makedirs(os.path.join(opt.output_dir, name[-2]), exist_ok=True) + kp_extractor.extract_keypoint( + images, + name=os.path.join(opt.output_dir, name[-2], name[-1]) + ) + +if __name__ == '__main__': + set_start_method('spawn') + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--input_dir', type=str, help='the folder of the input files') + parser.add_argument('--output_dir', type=str, help='the folder of the output files') + parser.add_argument('--device_ids', type=str, default='0,1') + parser.add_argument('--workers', type=int, default=4) + + opt = parser.parse_args() + filenames = list() + VIDEO_EXTENSIONS_LOWERCASE = {'mp4'} + VIDEO_EXTENSIONS = VIDEO_EXTENSIONS_LOWERCASE.union({f.upper() for f in VIDEO_EXTENSIONS_LOWERCASE}) + extensions = VIDEO_EXTENSIONS + + for ext in extensions: + os.listdir(f'{opt.input_dir}') + print(f'{opt.input_dir}/*.{ext}') + filenames = sorted(glob.glob(f'{opt.input_dir}/*.{ext}')) + print('Total number of videos:', len(filenames)) + pool = Pool(opt.workers) + args_list = cycle([opt]) + device_ids = opt.device_ids.split(",") + device_ids = cycle(device_ids) + for data in tqdm(pool.imap_unordered(run, zip(filenames, args_list, device_ids))): + None diff --git a/src/face3d/extract_kp_videos_safe.py b/src/face3d/extract_kp_videos_safe.py new file mode 100644 index 0000000000000000000000000000000000000000..5141ba3adfdd62b6205909dca519d66271c425ad --- /dev/null +++ b/src/face3d/extract_kp_videos_safe.py @@ -0,0 +1,151 @@ +import os +import cv2 +import time +import glob +import argparse +import numpy as np +from PIL import Image +import torch +from tqdm import tqdm +from itertools import cycle +from torch.multiprocessing import Pool, Process, set_start_method + +from facexlib.alignment import landmark_98_to_68 +from facexlib.detection import init_detection_model + +from facexlib.utils import load_file_from_url +from src.face3d.util.my_awing_arch import FAN + +def init_alignment_model(model_name, half=False, device='cuda', model_rootpath=None): + if model_name == 'awing_fan': + model = FAN(num_modules=4, num_landmarks=98, device=device) + model_url = 'https://github.com/xinntao/facexlib/releases/download/v0.1.0/alignment_WFLW_4HG.pth' + else: + raise NotImplementedError(f'{model_name} is not implemented.') + + model_path = load_file_from_url( + url=model_url, model_dir='facexlib/weights', progress=True, file_name=None, save_dir=model_rootpath) + model.load_state_dict(torch.load(model_path, map_location=device)['state_dict'], strict=True) + model.eval() + model = model.to(device) + return model + + +class KeypointExtractor(): + def __init__(self, device='cuda'): + + ### gfpgan/weights + try: + import webui # in webui + root_path = 'extensions/SadTalker/gfpgan/weights' + + except: + root_path = 'gfpgan/weights' + + self.detector = init_alignment_model('awing_fan',device=device, model_rootpath=root_path) + self.det_net = init_detection_model('retinaface_resnet50', half=False,device=device, model_rootpath=root_path) + + def extract_keypoint(self, images, name=None, info=True): + if isinstance(images, list): + keypoints = [] + if info: + i_range = tqdm(images,desc='landmark Det:') + else: + i_range = images + + for image in i_range: + current_kp = self.extract_keypoint(image) + # current_kp = self.detector.get_landmarks(np.array(image)) + if np.mean(current_kp) == -1 and keypoints: + keypoints.append(keypoints[-1]) + else: + keypoints.append(current_kp[None]) + + keypoints = np.concatenate(keypoints, 0) + np.savetxt(os.path.splitext(name)[0]+'.txt', keypoints.reshape(-1)) + return keypoints + else: + while True: + try: + with torch.no_grad(): + # face detection -> face alignment. + img = np.array(images) + bboxes = self.det_net.detect_faces(images, 0.97) + + bboxes = bboxes[0] + img = img[int(bboxes[1]):int(bboxes[3]), int(bboxes[0]):int(bboxes[2]), :] + + keypoints = landmark_98_to_68(self.detector.get_landmarks(img)) # [0] + + #### keypoints to the original location + keypoints[:,0] += int(bboxes[0]) + keypoints[:,1] += int(bboxes[1]) + + break + except RuntimeError as e: + if str(e).startswith('CUDA'): + print("Warning: out of memory, sleep for 1s") + time.sleep(1) + else: + print(e) + break + except TypeError: + print('No face detected in this image') + shape = [68, 2] + keypoints = -1. * np.ones(shape) + break + if name is not None: + np.savetxt(os.path.splitext(name)[0]+'.txt', keypoints.reshape(-1)) + return keypoints + +def read_video(filename): + frames = [] + cap = cv2.VideoCapture(filename) + while cap.isOpened(): + ret, frame = cap.read() + if ret: + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + frame = Image.fromarray(frame) + frames.append(frame) + else: + break + cap.release() + return frames + +def run(data): + filename, opt, device = data + os.environ['CUDA_VISIBLE_DEVICES'] = device + kp_extractor = KeypointExtractor() + images = read_video(filename) + name = filename.split('/')[-2:] + os.makedirs(os.path.join(opt.output_dir, name[-2]), exist_ok=True) + kp_extractor.extract_keypoint( + images, + name=os.path.join(opt.output_dir, name[-2], name[-1]) + ) + +if __name__ == '__main__': + set_start_method('spawn') + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--input_dir', type=str, help='the folder of the input files') + parser.add_argument('--output_dir', type=str, help='the folder of the output files') + parser.add_argument('--device_ids', type=str, default='0,1') + parser.add_argument('--workers', type=int, default=4) + + opt = parser.parse_args() + filenames = list() + VIDEO_EXTENSIONS_LOWERCASE = {'mp4'} + VIDEO_EXTENSIONS = VIDEO_EXTENSIONS_LOWERCASE.union({f.upper() for f in VIDEO_EXTENSIONS_LOWERCASE}) + extensions = VIDEO_EXTENSIONS + + for ext in extensions: + os.listdir(f'{opt.input_dir}') + print(f'{opt.input_dir}/*.{ext}') + filenames = sorted(glob.glob(f'{opt.input_dir}/*.{ext}')) + print('Total number of videos:', len(filenames)) + pool = Pool(opt.workers) + args_list = cycle([opt]) + device_ids = opt.device_ids.split(",") + device_ids = cycle(device_ids) + for data in tqdm(pool.imap_unordered(run, zip(filenames, args_list, device_ids))): + None diff --git a/src/face3d/models/__init__.py b/src/face3d/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5a7986c7ad2ec48f404adf81fea5aa06aaf1eeb4 --- /dev/null +++ b/src/face3d/models/__init__.py @@ -0,0 +1,67 @@ +"""This package contains modules related to objective functions, optimizations, and network architectures. + +To add a custom model class called 'dummy', you need to add a file called 'dummy_model.py' and define a subclass DummyModel inherited from BaseModel. +You need to implement the following five functions: + -- <__init__>: initialize the class; first call BaseModel.__init__(self, opt). + -- : unpack data from dataset and apply preprocessing. + -- : produce intermediate results. + -- : calculate loss, gradients, and update network weights. + -- : (optionally) add model-specific options and set default options. + +In the function <__init__>, you need to define four lists: + -- self.loss_names (str list): specify the training losses that you want to plot and save. + -- self.model_names (str list): define networks used in our training. + -- self.visual_names (str list): specify the images that you want to display and save. + -- self.optimizers (optimizer list): define and initialize optimizers. You can define one optimizer for each network. If two networks are updated at the same time, you can use itertools.chain to group them. See cycle_gan_model.py for an usage. + +Now you can use the model class by specifying flag '--model dummy'. +See our template model class 'template_model.py' for more details. +""" + +import importlib +from src.face3d.models.base_model import BaseModel + + +def find_model_using_name(model_name): + """Import the module "models/[model_name]_model.py". + + In the file, the class called DatasetNameModel() will + be instantiated. It has to be a subclass of BaseModel, + and it is case-insensitive. + """ + model_filename = "face3d.models." + model_name + "_model" + modellib = importlib.import_module(model_filename) + model = None + target_model_name = model_name.replace('_', '') + 'model' + for name, cls in modellib.__dict__.items(): + if name.lower() == target_model_name.lower() \ + and issubclass(cls, BaseModel): + model = cls + + if model is None: + print("In %s.py, there should be a subclass of BaseModel with class name that matches %s in lowercase." % (model_filename, target_model_name)) + exit(0) + + return model + + +def get_option_setter(model_name): + """Return the static method of the model class.""" + model_class = find_model_using_name(model_name) + return model_class.modify_commandline_options + + +def create_model(opt): + """Create a model given the option. + + This function warps the class CustomDatasetDataLoader. + This is the main interface between this package and 'train.py'/'test.py' + + Example: + >>> from models import create_model + >>> model = create_model(opt) + """ + model = find_model_using_name(opt.model) + instance = model(opt) + print("model [%s] was created" % type(instance).__name__) + return instance diff --git a/src/face3d/models/__pycache__/__init__.cpython-310.pyc b/src/face3d/models/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f794954ef6093f350722728a51304cc091684e45 Binary files /dev/null and b/src/face3d/models/__pycache__/__init__.cpython-310.pyc differ diff --git a/src/face3d/models/__pycache__/base_model.cpython-310.pyc b/src/face3d/models/__pycache__/base_model.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6580c87310abd0bbf7778cf91eef258949719d72 Binary files /dev/null and b/src/face3d/models/__pycache__/base_model.cpython-310.pyc differ diff --git a/src/face3d/models/__pycache__/networks.cpython-310.pyc b/src/face3d/models/__pycache__/networks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e810fe993d6d68aa07aaff000a48e5a93c0fb2b Binary files /dev/null and b/src/face3d/models/__pycache__/networks.cpython-310.pyc differ diff --git a/src/face3d/models/arcface_torch/README.md b/src/face3d/models/arcface_torch/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2ee63a861229b68873561fa39bfa7c9a8b53b947 --- /dev/null +++ b/src/face3d/models/arcface_torch/README.md @@ -0,0 +1,164 @@ +# Distributed Arcface Training in Pytorch + +This is a deep learning library that makes face recognition efficient, and effective, which can train tens of millions +identity on a single server. + +## Requirements + +- Install [pytorch](http://pytorch.org) (torch>=1.6.0), our doc for [install.md](docs/install.md). +- `pip install -r requirements.txt`. +- Download the dataset + from [https://github.com/deepinsight/insightface/tree/master/recognition/_datasets_](https://github.com/deepinsight/insightface/tree/master/recognition/_datasets_) + . + +## How to Training + +To train a model, run `train.py` with the path to the configs: + +### 1. Single node, 8 GPUs: + +```shell +python -m torch.distributed.launch --nproc_per_node=8 --nnodes=1 --node_rank=0 --master_addr="127.0.0.1" --master_port=1234 train.py configs/ms1mv3_r50 +``` + +### 2. Multiple nodes, each node 8 GPUs: + +Node 0: + +```shell +python -m torch.distributed.launch --nproc_per_node=8 --nnodes=2 --node_rank=0 --master_addr="ip1" --master_port=1234 train.py train.py configs/ms1mv3_r50 +``` + +Node 1: + +```shell +python -m torch.distributed.launch --nproc_per_node=8 --nnodes=2 --node_rank=1 --master_addr="ip1" --master_port=1234 train.py train.py configs/ms1mv3_r50 +``` + +### 3.Training resnet2060 with 8 GPUs: + +```shell +python -m torch.distributed.launch --nproc_per_node=8 --nnodes=1 --node_rank=0 --master_addr="127.0.0.1" --master_port=1234 train.py configs/ms1mv3_r2060.py +``` + +## Model Zoo + +- The models are available for non-commercial research purposes only. +- All models can be found in here. +- [Baidu Yun Pan](https://pan.baidu.com/s/1CL-l4zWqsI1oDuEEYVhj-g): e8pw +- [onedrive](https://1drv.ms/u/s!AswpsDO2toNKq0lWY69vN58GR6mw?e=p9Ov5d) + +### Performance on [**ICCV2021-MFR**](http://iccv21-mfr.com/) + +ICCV2021-MFR testset consists of non-celebrities so we can ensure that it has very few overlap with public available face +recognition training set, such as MS1M and CASIA as they mostly collected from online celebrities. +As the result, we can evaluate the FAIR performance for different algorithms. + +For **ICCV2021-MFR-ALL** set, TAR is measured on all-to-all 1:1 protocal, with FAR less than 0.000001(e-6). The +globalised multi-racial testset contains 242,143 identities and 1,624,305 images. + +For **ICCV2021-MFR-MASK** set, TAR is measured on mask-to-nonmask 1:1 protocal, with FAR less than 0.0001(e-4). +Mask testset contains 6,964 identities, 6,964 masked images and 13,928 non-masked images. +There are totally 13,928 positive pairs and 96,983,824 negative pairs. + +| Datasets | backbone | Training throughout | Size / MB | **ICCV2021-MFR-MASK** | **ICCV2021-MFR-ALL** | +| :---: | :--- | :--- | :--- |:--- |:--- | +| MS1MV3 | r18 | - | 91 | **47.85** | **68.33** | +| Glint360k | r18 | 8536 | 91 | **53.32** | **72.07** | +| MS1MV3 | r34 | - | 130 | **58.72** | **77.36** | +| Glint360k | r34 | 6344 | 130 | **65.10** | **83.02** | +| MS1MV3 | r50 | 5500 | 166 | **63.85** | **80.53** | +| Glint360k | r50 | 5136 | 166 | **70.23** | **87.08** | +| MS1MV3 | r100 | - | 248 | **69.09** | **84.31** | +| Glint360k | r100 | 3332 | 248 | **75.57** | **90.66** | +| MS1MV3 | mobilefacenet | 12185 | 7.8 | **41.52** | **65.26** | +| Glint360k | mobilefacenet | 11197 | 7.8 | **44.52** | **66.48** | + +### Performance on IJB-C and Verification Datasets + +| Datasets | backbone | IJBC(1e-05) | IJBC(1e-04) | agedb30 | cfp_fp | lfw | log | +| :---: | :--- | :--- | :--- | :--- |:--- |:--- |:--- | +| MS1MV3 | r18 | 92.07 | 94.66 | 97.77 | 97.73 | 99.77 |[log](https://raw.githubusercontent.com/anxiangsir/insightface_arcface_log/master/ms1mv3_arcface_r18_fp16/training.log)| +| MS1MV3 | r34 | 94.10 | 95.90 | 98.10 | 98.67 | 99.80 |[log](https://raw.githubusercontent.com/anxiangsir/insightface_arcface_log/master/ms1mv3_arcface_r34_fp16/training.log)| +| MS1MV3 | r50 | 94.79 | 96.46 | 98.35 | 98.96 | 99.83 |[log](https://raw.githubusercontent.com/anxiangsir/insightface_arcface_log/master/ms1mv3_arcface_r50_fp16/training.log)| +| MS1MV3 | r100 | 95.31 | 96.81 | 98.48 | 99.06 | 99.85 |[log](https://raw.githubusercontent.com/anxiangsir/insightface_arcface_log/master/ms1mv3_arcface_r100_fp16/training.log)| +| MS1MV3 | **r2060**| 95.34 | 97.11 | 98.67 | 99.24 | 99.87 |[log](https://raw.githubusercontent.com/anxiangsir/insightface_arcface_log/master/ms1mv3_arcface_r2060_fp16/training.log)| +| Glint360k |r18-0.1 | 93.16 | 95.33 | 97.72 | 97.73 | 99.77 |[log](https://raw.githubusercontent.com/anxiangsir/insightface_arcface_log/master/glint360k_cosface_r18_fp16_0.1/training.log)| +| Glint360k |r34-0.1 | 95.16 | 96.56 | 98.33 | 98.78 | 99.82 |[log](https://raw.githubusercontent.com/anxiangsir/insightface_arcface_log/master/glint360k_cosface_r34_fp16_0.1/training.log)| +| Glint360k |r50-0.1 | 95.61 | 96.97 | 98.38 | 99.20 | 99.83 |[log](https://raw.githubusercontent.com/anxiangsir/insightface_arcface_log/master/glint360k_cosface_r50_fp16_0.1/training.log)| +| Glint360k |r100-0.1 | 95.88 | 97.32 | 98.48 | 99.29 | 99.82 |[log](https://raw.githubusercontent.com/anxiangsir/insightface_arcface_log/master/glint360k_cosface_r100_fp16_0.1/training.log)| + +[comment]: <> (More details see [model.md](docs/modelzoo.md) in docs.) + + +## [Speed Benchmark](docs/speed_benchmark.md) + +**Arcface Torch** can train large-scale face recognition training set efficiently and quickly. When the number of +classes in training sets is greater than 300K and the training is sufficient, partial fc sampling strategy will get same +accuracy with several times faster training performance and smaller GPU memory. +Partial FC is a sparse variant of the model parallel architecture for large sacle face recognition. Partial FC use a +sparse softmax, where each batch dynamicly sample a subset of class centers for training. In each iteration, only a +sparse part of the parameters will be updated, which can reduce a lot of GPU memory and calculations. With Partial FC, +we can scale trainset of 29 millions identities, the largest to date. Partial FC also supports multi-machine distributed +training and mixed precision training. + +![Image text](https://github.com/anxiangsir/insightface_arcface_log/blob/master/partial_fc_v2.png) + +More details see +[speed_benchmark.md](docs/speed_benchmark.md) in docs. + +### 1. Training speed of different parallel methods (samples / second), Tesla V100 32GB * 8. (Larger is better) + +`-` means training failed because of gpu memory limitations. + +| Number of Identities in Dataset | Data Parallel | Model Parallel | Partial FC 0.1 | +| :--- | :--- | :--- | :--- | +|125000 | 4681 | 4824 | 5004 | +|1400000 | **1672** | 3043 | 4738 | +|5500000 | **-** | **1389** | 3975 | +|8000000 | **-** | **-** | 3565 | +|16000000 | **-** | **-** | 2679 | +|29000000 | **-** | **-** | **1855** | + +### 2. GPU memory cost of different parallel methods (MB per GPU), Tesla V100 32GB * 8. (Smaller is better) + +| Number of Identities in Dataset | Data Parallel | Model Parallel | Partial FC 0.1 | +| :--- | :--- | :--- | :--- | +|125000 | 7358 | 5306 | 4868 | +|1400000 | 32252 | 11178 | 6056 | +|5500000 | **-** | 32188 | 9854 | +|8000000 | **-** | **-** | 12310 | +|16000000 | **-** | **-** | 19950 | +|29000000 | **-** | **-** | 32324 | + +## Evaluation ICCV2021-MFR and IJB-C + +More details see [eval.md](docs/eval.md) in docs. + +## Test + +We tested many versions of PyTorch. Please create an issue if you are having trouble. + +- [x] torch 1.6.0 +- [x] torch 1.7.1 +- [x] torch 1.8.0 +- [x] torch 1.9.0 + +## Citation + +``` +@inproceedings{deng2019arcface, + title={Arcface: Additive angular margin loss for deep face recognition}, + author={Deng, Jiankang and Guo, Jia and Xue, Niannan and Zafeiriou, Stefanos}, + booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition}, + pages={4690--4699}, + year={2019} +} +@inproceedings{an2020partical_fc, + title={Partial FC: Training 10 Million Identities on a Single Machine}, + author={An, Xiang and Zhu, Xuhan and Xiao, Yang and Wu, Lan and Zhang, Ming and Gao, Yuan and Qin, Bin and + Zhang, Debing and Fu Ying}, + booktitle={Arxiv 2010.05222}, + year={2020} +} +``` diff --git a/src/face3d/models/arcface_torch/backbones/__init__.py b/src/face3d/models/arcface_torch/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..55bd4c5d1889a1a998b52eb56793bbc1eef1b691 --- /dev/null +++ b/src/face3d/models/arcface_torch/backbones/__init__.py @@ -0,0 +1,25 @@ +from .iresnet import iresnet18, iresnet34, iresnet50, iresnet100, iresnet200 +from .mobilefacenet import get_mbf + + +def get_model(name, **kwargs): + # resnet + if name == "r18": + return iresnet18(False, **kwargs) + elif name == "r34": + return iresnet34(False, **kwargs) + elif name == "r50": + return iresnet50(False, **kwargs) + elif name == "r100": + return iresnet100(False, **kwargs) + elif name == "r200": + return iresnet200(False, **kwargs) + elif name == "r2060": + from .iresnet2060 import iresnet2060 + return iresnet2060(False, **kwargs) + elif name == "mbf": + fp16 = kwargs.get("fp16", False) + num_features = kwargs.get("num_features", 512) + return get_mbf(fp16=fp16, num_features=num_features) + else: + raise ValueError() \ No newline at end of file diff --git a/src/face3d/models/arcface_torch/backbones/__pycache__/__init__.cpython-310.pyc b/src/face3d/models/arcface_torch/backbones/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..269e519e6bbb32583c3e249f5aea6a0725b7acdf Binary files /dev/null and b/src/face3d/models/arcface_torch/backbones/__pycache__/__init__.cpython-310.pyc differ diff --git a/src/face3d/models/arcface_torch/backbones/__pycache__/iresnet.cpython-310.pyc b/src/face3d/models/arcface_torch/backbones/__pycache__/iresnet.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24a6a0e330f9bf45bd5998d81f5ab7f4fdcc58fd Binary files /dev/null and b/src/face3d/models/arcface_torch/backbones/__pycache__/iresnet.cpython-310.pyc differ diff --git a/src/face3d/models/arcface_torch/backbones/__pycache__/mobilefacenet.cpython-310.pyc b/src/face3d/models/arcface_torch/backbones/__pycache__/mobilefacenet.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..55c8758a904fb0de29c5f2c657d6c9ec8a10e2e1 Binary files /dev/null and b/src/face3d/models/arcface_torch/backbones/__pycache__/mobilefacenet.cpython-310.pyc differ diff --git a/src/face3d/models/arcface_torch/backbones/iresnet.py b/src/face3d/models/arcface_torch/backbones/iresnet.py new file mode 100644 index 0000000000000000000000000000000000000000..c6d3b9c240c24687d432197f976ee01fbf423216 --- /dev/null +++ b/src/face3d/models/arcface_torch/backbones/iresnet.py @@ -0,0 +1,187 @@ +import torch +from torch import nn + +__all__ = ['iresnet18', 'iresnet34', 'iresnet50', 'iresnet100', 'iresnet200'] + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, + out_planes, + kernel_size=1, + stride=stride, + bias=False) + + +class IBasicBlock(nn.Module): + expansion = 1 + def __init__(self, inplanes, planes, stride=1, downsample=None, + groups=1, base_width=64, dilation=1): + super(IBasicBlock, self).__init__() + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + self.bn1 = nn.BatchNorm2d(inplanes, eps=1e-05,) + self.conv1 = conv3x3(inplanes, planes) + self.bn2 = nn.BatchNorm2d(planes, eps=1e-05,) + self.prelu = nn.PReLU(planes) + self.conv2 = conv3x3(planes, planes, stride) + self.bn3 = nn.BatchNorm2d(planes, eps=1e-05,) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + out = self.bn1(x) + out = self.conv1(out) + out = self.bn2(out) + out = self.prelu(out) + out = self.conv2(out) + out = self.bn3(out) + if self.downsample is not None: + identity = self.downsample(x) + out += identity + return out + + +class IResNet(nn.Module): + fc_scale = 7 * 7 + def __init__(self, + block, layers, dropout=0, num_features=512, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, fp16=False): + super(IResNet, self).__init__() + self.fp16 = fp16 + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05) + self.prelu = nn.PReLU(self.inplanes) + self.layer1 = self._make_layer(block, 64, layers[0], stride=2) + self.layer2 = self._make_layer(block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + self.bn2 = nn.BatchNorm2d(512 * block.expansion, eps=1e-05,) + self.dropout = nn.Dropout(p=dropout, inplace=True) + self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features) + self.features = nn.BatchNorm1d(num_features, eps=1e-05) + nn.init.constant_(self.features.weight, 1.0) + self.features.weight.requires_grad = False + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.normal_(m.weight, 0, 0.1) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + if zero_init_residual: + for m in self.modules(): + if isinstance(m, IBasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + nn.BatchNorm2d(planes * block.expansion, eps=1e-05, ), + ) + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block(self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation)) + + return nn.Sequential(*layers) + + def forward(self, x): + with torch.cuda.amp.autocast(self.fp16): + x = self.conv1(x) + x = self.bn1(x) + x = self.prelu(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.bn2(x) + x = torch.flatten(x, 1) + x = self.dropout(x) + x = self.fc(x.float() if self.fp16 else x) + x = self.features(x) + return x + + +def _iresnet(arch, block, layers, pretrained, progress, **kwargs): + model = IResNet(block, layers, **kwargs) + if pretrained: + raise ValueError() + return model + + +def iresnet18(pretrained=False, progress=True, **kwargs): + return _iresnet('iresnet18', IBasicBlock, [2, 2, 2, 2], pretrained, + progress, **kwargs) + + +def iresnet34(pretrained=False, progress=True, **kwargs): + return _iresnet('iresnet34', IBasicBlock, [3, 4, 6, 3], pretrained, + progress, **kwargs) + + +def iresnet50(pretrained=False, progress=True, **kwargs): + return _iresnet('iresnet50', IBasicBlock, [3, 4, 14, 3], pretrained, + progress, **kwargs) + + +def iresnet100(pretrained=False, progress=True, **kwargs): + return _iresnet('iresnet100', IBasicBlock, [3, 13, 30, 3], pretrained, + progress, **kwargs) + + +def iresnet200(pretrained=False, progress=True, **kwargs): + return _iresnet('iresnet200', IBasicBlock, [6, 26, 60, 6], pretrained, + progress, **kwargs) + diff --git a/src/face3d/models/arcface_torch/backbones/iresnet2060.py b/src/face3d/models/arcface_torch/backbones/iresnet2060.py new file mode 100644 index 0000000000000000000000000000000000000000..21d1122144d207637d2444cba1f68fe630c89f31 --- /dev/null +++ b/src/face3d/models/arcface_torch/backbones/iresnet2060.py @@ -0,0 +1,176 @@ +import torch +from torch import nn + +assert torch.__version__ >= "1.8.1" +from torch.utils.checkpoint import checkpoint_sequential + +__all__ = ['iresnet2060'] + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, + out_planes, + kernel_size=1, + stride=stride, + bias=False) + + +class IBasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, + groups=1, base_width=64, dilation=1): + super(IBasicBlock, self).__init__() + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + self.bn1 = nn.BatchNorm2d(inplanes, eps=1e-05, ) + self.conv1 = conv3x3(inplanes, planes) + self.bn2 = nn.BatchNorm2d(planes, eps=1e-05, ) + self.prelu = nn.PReLU(planes) + self.conv2 = conv3x3(planes, planes, stride) + self.bn3 = nn.BatchNorm2d(planes, eps=1e-05, ) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + out = self.bn1(x) + out = self.conv1(out) + out = self.bn2(out) + out = self.prelu(out) + out = self.conv2(out) + out = self.bn3(out) + if self.downsample is not None: + identity = self.downsample(x) + out += identity + return out + + +class IResNet(nn.Module): + fc_scale = 7 * 7 + + def __init__(self, + block, layers, dropout=0, num_features=512, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, fp16=False): + super(IResNet, self).__init__() + self.fp16 = fp16 + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05) + self.prelu = nn.PReLU(self.inplanes) + self.layer1 = self._make_layer(block, 64, layers[0], stride=2) + self.layer2 = self._make_layer(block, + 128, + layers[1], + stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, + 256, + layers[2], + stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, + 512, + layers[3], + stride=2, + dilate=replace_stride_with_dilation[2]) + self.bn2 = nn.BatchNorm2d(512 * block.expansion, eps=1e-05, ) + self.dropout = nn.Dropout(p=dropout, inplace=True) + self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features) + self.features = nn.BatchNorm1d(num_features, eps=1e-05) + nn.init.constant_(self.features.weight, 1.0) + self.features.weight.requires_grad = False + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.normal_(m.weight, 0, 0.1) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + if zero_init_residual: + for m in self.modules(): + if isinstance(m, IBasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + nn.BatchNorm2d(planes * block.expansion, eps=1e-05, ), + ) + layers = [] + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block(self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation)) + + return nn.Sequential(*layers) + + def checkpoint(self, func, num_seg, x): + if self.training: + return checkpoint_sequential(func, num_seg, x) + else: + return func(x) + + def forward(self, x): + with torch.cuda.amp.autocast(self.fp16): + x = self.conv1(x) + x = self.bn1(x) + x = self.prelu(x) + x = self.layer1(x) + x = self.checkpoint(self.layer2, 20, x) + x = self.checkpoint(self.layer3, 100, x) + x = self.layer4(x) + x = self.bn2(x) + x = torch.flatten(x, 1) + x = self.dropout(x) + x = self.fc(x.float() if self.fp16 else x) + x = self.features(x) + return x + + +def _iresnet(arch, block, layers, pretrained, progress, **kwargs): + model = IResNet(block, layers, **kwargs) + if pretrained: + raise ValueError() + return model + + +def iresnet2060(pretrained=False, progress=True, **kwargs): + return _iresnet('iresnet2060', IBasicBlock, [3, 128, 1024 - 128, 3], pretrained, progress, **kwargs) diff --git a/src/face3d/models/arcface_torch/backbones/mobilefacenet.py b/src/face3d/models/arcface_torch/backbones/mobilefacenet.py new file mode 100644 index 0000000000000000000000000000000000000000..87731491d76f9ff61cc70e57bb3f18c54fae308c --- /dev/null +++ b/src/face3d/models/arcface_torch/backbones/mobilefacenet.py @@ -0,0 +1,130 @@ +''' +Adapted from https://github.com/cavalleria/cavaface.pytorch/blob/master/backbone/mobilefacenet.py +Original author cavalleria +''' + +import torch.nn as nn +from torch.nn import Linear, Conv2d, BatchNorm1d, BatchNorm2d, PReLU, Sequential, Module +import torch + + +class Flatten(Module): + def forward(self, x): + return x.view(x.size(0), -1) + + +class ConvBlock(Module): + def __init__(self, in_c, out_c, kernel=(1, 1), stride=(1, 1), padding=(0, 0), groups=1): + super(ConvBlock, self).__init__() + self.layers = nn.Sequential( + Conv2d(in_c, out_c, kernel, groups=groups, stride=stride, padding=padding, bias=False), + BatchNorm2d(num_features=out_c), + PReLU(num_parameters=out_c) + ) + + def forward(self, x): + return self.layers(x) + + +class LinearBlock(Module): + def __init__(self, in_c, out_c, kernel=(1, 1), stride=(1, 1), padding=(0, 0), groups=1): + super(LinearBlock, self).__init__() + self.layers = nn.Sequential( + Conv2d(in_c, out_c, kernel, stride, padding, groups=groups, bias=False), + BatchNorm2d(num_features=out_c) + ) + + def forward(self, x): + return self.layers(x) + + +class DepthWise(Module): + def __init__(self, in_c, out_c, residual=False, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=1): + super(DepthWise, self).__init__() + self.residual = residual + self.layers = nn.Sequential( + ConvBlock(in_c, out_c=groups, kernel=(1, 1), padding=(0, 0), stride=(1, 1)), + ConvBlock(groups, groups, groups=groups, kernel=kernel, padding=padding, stride=stride), + LinearBlock(groups, out_c, kernel=(1, 1), padding=(0, 0), stride=(1, 1)) + ) + + def forward(self, x): + short_cut = None + if self.residual: + short_cut = x + x = self.layers(x) + if self.residual: + output = short_cut + x + else: + output = x + return output + + +class Residual(Module): + def __init__(self, c, num_block, groups, kernel=(3, 3), stride=(1, 1), padding=(1, 1)): + super(Residual, self).__init__() + modules = [] + for _ in range(num_block): + modules.append(DepthWise(c, c, True, kernel, stride, padding, groups)) + self.layers = Sequential(*modules) + + def forward(self, x): + return self.layers(x) + + +class GDC(Module): + def __init__(self, embedding_size): + super(GDC, self).__init__() + self.layers = nn.Sequential( + LinearBlock(512, 512, groups=512, kernel=(7, 7), stride=(1, 1), padding=(0, 0)), + Flatten(), + Linear(512, embedding_size, bias=False), + BatchNorm1d(embedding_size)) + + def forward(self, x): + return self.layers(x) + + +class MobileFaceNet(Module): + def __init__(self, fp16=False, num_features=512): + super(MobileFaceNet, self).__init__() + scale = 2 + self.fp16 = fp16 + self.layers = nn.Sequential( + ConvBlock(3, 64 * scale, kernel=(3, 3), stride=(2, 2), padding=(1, 1)), + ConvBlock(64 * scale, 64 * scale, kernel=(3, 3), stride=(1, 1), padding=(1, 1), groups=64), + DepthWise(64 * scale, 64 * scale, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=128), + Residual(64 * scale, num_block=4, groups=128, kernel=(3, 3), stride=(1, 1), padding=(1, 1)), + DepthWise(64 * scale, 128 * scale, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=256), + Residual(128 * scale, num_block=6, groups=256, kernel=(3, 3), stride=(1, 1), padding=(1, 1)), + DepthWise(128 * scale, 128 * scale, kernel=(3, 3), stride=(2, 2), padding=(1, 1), groups=512), + Residual(128 * scale, num_block=2, groups=256, kernel=(3, 3), stride=(1, 1), padding=(1, 1)), + ) + self.conv_sep = ConvBlock(128 * scale, 512, kernel=(1, 1), stride=(1, 1), padding=(0, 0)) + self.features = GDC(num_features) + self._initialize_weights() + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x): + with torch.cuda.amp.autocast(self.fp16): + x = self.layers(x) + x = self.conv_sep(x.float() if self.fp16 else x) + x = self.features(x) + return x + + +def get_mbf(fp16, num_features): + return MobileFaceNet(fp16, num_features) \ No newline at end of file diff --git a/src/face3d/models/arcface_torch/configs/3millions.py b/src/face3d/models/arcface_torch/configs/3millions.py new file mode 100644 index 0000000000000000000000000000000000000000..c9edc2f1414e35f93abfd3dfe11a61f1f406580e --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/3millions.py @@ -0,0 +1,23 @@ +from easydict import EasyDict as edict + +# configs for test speed + +config = edict() +config.loss = "arcface" +config.network = "r50" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "synthetic" +config.num_classes = 300 * 10000 +config.num_epoch = 30 +config.warmup_epoch = -1 +config.decay_epoch = [10, 16, 22] +config.val_targets = [] diff --git a/src/face3d/models/arcface_torch/configs/3millions_pfc.py b/src/face3d/models/arcface_torch/configs/3millions_pfc.py new file mode 100644 index 0000000000000000000000000000000000000000..77caafdbb300d8109d5bfdb844f131710ef81f20 --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/3millions_pfc.py @@ -0,0 +1,23 @@ +from easydict import EasyDict as edict + +# configs for test speed + +config = edict() +config.loss = "arcface" +config.network = "r50" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 0.1 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "synthetic" +config.num_classes = 300 * 10000 +config.num_epoch = 30 +config.warmup_epoch = -1 +config.decay_epoch = [10, 16, 22] +config.val_targets = [] diff --git a/src/face3d/models/arcface_torch/configs/__init__.py b/src/face3d/models/arcface_torch/configs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/face3d/models/arcface_torch/configs/base.py b/src/face3d/models/arcface_torch/configs/base.py new file mode 100644 index 0000000000000000000000000000000000000000..78e4b36a9142b649ec39a8c59331bb2557f2ad57 --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/base.py @@ -0,0 +1,56 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "arcface" +config.network = "r50" +config.resume = False +config.output = "ms1mv3_arcface_r50" + +config.dataset = "ms1m-retinaface-t1" +config.embedding_size = 512 +config.sample_rate = 1 +config.fp16 = False +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +if config.dataset == "emore": + config.rec = "/train_tmp/faces_emore" + config.num_classes = 85742 + config.num_image = 5822653 + config.num_epoch = 16 + config.warmup_epoch = -1 + config.decay_epoch = [8, 14, ] + config.val_targets = ["lfw", ] + +elif config.dataset == "ms1m-retinaface-t1": + config.rec = "/train_tmp/ms1m-retinaface-t1" + config.num_classes = 93431 + config.num_image = 5179510 + config.num_epoch = 25 + config.warmup_epoch = -1 + config.decay_epoch = [11, 17, 22] + config.val_targets = ["lfw", "cfp_fp", "agedb_30"] + +elif config.dataset == "glint360k": + config.rec = "/train_tmp/glint360k" + config.num_classes = 360232 + config.num_image = 17091657 + config.num_epoch = 20 + config.warmup_epoch = -1 + config.decay_epoch = [8, 12, 15, 18] + config.val_targets = ["lfw", "cfp_fp", "agedb_30"] + +elif config.dataset == "webface": + config.rec = "/train_tmp/faces_webface_112x112" + config.num_classes = 10572 + config.num_image = "forget" + config.num_epoch = 34 + config.warmup_epoch = -1 + config.decay_epoch = [20, 28, 32] + config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/glint360k_mbf.py b/src/face3d/models/arcface_torch/configs/glint360k_mbf.py new file mode 100644 index 0000000000000000000000000000000000000000..46ae777cc97af41a531cba4e5d1ff31f2efcb468 --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/glint360k_mbf.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "cosface" +config.network = "mbf" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 0.1 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 2e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/glint360k" +config.num_classes = 360232 +config.num_image = 17091657 +config.num_epoch = 20 +config.warmup_epoch = -1 +config.decay_epoch = [8, 12, 15, 18] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/glint360k_r100.py b/src/face3d/models/arcface_torch/configs/glint360k_r100.py new file mode 100644 index 0000000000000000000000000000000000000000..93d0701c0094517cec147c382b005e8063938548 --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/glint360k_r100.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "cosface" +config.network = "r100" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/glint360k" +config.num_classes = 360232 +config.num_image = 17091657 +config.num_epoch = 20 +config.warmup_epoch = -1 +config.decay_epoch = [8, 12, 15, 18] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/glint360k_r18.py b/src/face3d/models/arcface_torch/configs/glint360k_r18.py new file mode 100644 index 0000000000000000000000000000000000000000..7a8db34cd547e8e667103c93585296e47a894e97 --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/glint360k_r18.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "cosface" +config.network = "r18" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/glint360k" +config.num_classes = 360232 +config.num_image = 17091657 +config.num_epoch = 20 +config.warmup_epoch = -1 +config.decay_epoch = [8, 12, 15, 18] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/glint360k_r34.py b/src/face3d/models/arcface_torch/configs/glint360k_r34.py new file mode 100644 index 0000000000000000000000000000000000000000..fda2701758a839a7161d09c25f0ca3d26033baff --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/glint360k_r34.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "cosface" +config.network = "r34" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/glint360k" +config.num_classes = 360232 +config.num_image = 17091657 +config.num_epoch = 20 +config.warmup_epoch = -1 +config.decay_epoch = [8, 12, 15, 18] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/glint360k_r50.py b/src/face3d/models/arcface_torch/configs/glint360k_r50.py new file mode 100644 index 0000000000000000000000000000000000000000..37e7922f1f63284e356dcc45a5f979f9c105f25e --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/glint360k_r50.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "cosface" +config.network = "r50" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/glint360k" +config.num_classes = 360232 +config.num_image = 17091657 +config.num_epoch = 20 +config.warmup_epoch = -1 +config.decay_epoch = [8, 12, 15, 18] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/ms1mv3_mbf.py b/src/face3d/models/arcface_torch/configs/ms1mv3_mbf.py new file mode 100644 index 0000000000000000000000000000000000000000..b8a00d6305eeda5a94788017afc1cda0d4a4cd2a --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/ms1mv3_mbf.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "arcface" +config.network = "mbf" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 2e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/ms1m-retinaface-t1" +config.num_classes = 93431 +config.num_image = 5179510 +config.num_epoch = 30 +config.warmup_epoch = -1 +config.decay_epoch = [10, 20, 25] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/ms1mv3_r18.py b/src/face3d/models/arcface_torch/configs/ms1mv3_r18.py new file mode 100644 index 0000000000000000000000000000000000000000..eb4e0d31f1aedf4590628d394e1606920fefb5c9 --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/ms1mv3_r18.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "arcface" +config.network = "r18" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/ms1m-retinaface-t1" +config.num_classes = 93431 +config.num_image = 5179510 +config.num_epoch = 25 +config.warmup_epoch = -1 +config.decay_epoch = [10, 16, 22] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/ms1mv3_r2060.py b/src/face3d/models/arcface_torch/configs/ms1mv3_r2060.py new file mode 100644 index 0000000000000000000000000000000000000000..23ad81e082c4b6390b67b164d0ceb84bb0635684 --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/ms1mv3_r2060.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "arcface" +config.network = "r2060" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 64 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/ms1m-retinaface-t1" +config.num_classes = 93431 +config.num_image = 5179510 +config.num_epoch = 25 +config.warmup_epoch = -1 +config.decay_epoch = [10, 16, 22] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/ms1mv3_r34.py b/src/face3d/models/arcface_torch/configs/ms1mv3_r34.py new file mode 100644 index 0000000000000000000000000000000000000000..5f78337a3d1f9eb6e9145eb5093618796c6842d2 --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/ms1mv3_r34.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "arcface" +config.network = "r34" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/ms1m-retinaface-t1" +config.num_classes = 93431 +config.num_image = 5179510 +config.num_epoch = 25 +config.warmup_epoch = -1 +config.decay_epoch = [10, 16, 22] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/ms1mv3_r50.py b/src/face3d/models/arcface_torch/configs/ms1mv3_r50.py new file mode 100644 index 0000000000000000000000000000000000000000..08ba55dbbea6df0afffddbb3d1ed173efad99604 --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/ms1mv3_r50.py @@ -0,0 +1,26 @@ +from easydict import EasyDict as edict + +# make training faster +# our RAM is 256G +# mount -t tmpfs -o size=140G tmpfs /train_tmp + +config = edict() +config.loss = "arcface" +config.network = "r50" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "/train_tmp/ms1m-retinaface-t1" +config.num_classes = 93431 +config.num_image = 5179510 +config.num_epoch = 25 +config.warmup_epoch = -1 +config.decay_epoch = [10, 16, 22] +config.val_targets = ["lfw", "cfp_fp", "agedb_30"] diff --git a/src/face3d/models/arcface_torch/configs/speed.py b/src/face3d/models/arcface_torch/configs/speed.py new file mode 100644 index 0000000000000000000000000000000000000000..45e95237da65e44f35a172c25ac6dc4e313e4eae --- /dev/null +++ b/src/face3d/models/arcface_torch/configs/speed.py @@ -0,0 +1,23 @@ +from easydict import EasyDict as edict + +# configs for test speed + +config = edict() +config.loss = "arcface" +config.network = "r50" +config.resume = False +config.output = None +config.embedding_size = 512 +config.sample_rate = 1.0 +config.fp16 = True +config.momentum = 0.9 +config.weight_decay = 5e-4 +config.batch_size = 128 +config.lr = 0.1 # batch size is 512 + +config.rec = "synthetic" +config.num_classes = 100 * 10000 +config.num_epoch = 30 +config.warmup_epoch = -1 +config.decay_epoch = [10, 16, 22] +config.val_targets = [] diff --git a/src/face3d/models/arcface_torch/dataset.py b/src/face3d/models/arcface_torch/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..96bbb8bb6da99122f350bc8e1a6390245840e32b --- /dev/null +++ b/src/face3d/models/arcface_torch/dataset.py @@ -0,0 +1,124 @@ +import numbers +import os +import queue as Queue +import threading + +import mxnet as mx +import numpy as np +import torch +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms + + +class BackgroundGenerator(threading.Thread): + def __init__(self, generator, local_rank, max_prefetch=6): + super(BackgroundGenerator, self).__init__() + self.queue = Queue.Queue(max_prefetch) + self.generator = generator + self.local_rank = local_rank + self.daemon = True + self.start() + + def run(self): + torch.cuda.set_device(self.local_rank) + for item in self.generator: + self.queue.put(item) + self.queue.put(None) + + def next(self): + next_item = self.queue.get() + if next_item is None: + raise StopIteration + return next_item + + def __next__(self): + return self.next() + + def __iter__(self): + return self + + +class DataLoaderX(DataLoader): + + def __init__(self, local_rank, **kwargs): + super(DataLoaderX, self).__init__(**kwargs) + self.stream = torch.cuda.Stream(local_rank) + self.local_rank = local_rank + + def __iter__(self): + self.iter = super(DataLoaderX, self).__iter__() + self.iter = BackgroundGenerator(self.iter, self.local_rank) + self.preload() + return self + + def preload(self): + self.batch = next(self.iter, None) + if self.batch is None: + return None + with torch.cuda.stream(self.stream): + for k in range(len(self.batch)): + self.batch[k] = self.batch[k].to(device=self.local_rank, non_blocking=True) + + def __next__(self): + torch.cuda.current_stream().wait_stream(self.stream) + batch = self.batch + if batch is None: + raise StopIteration + self.preload() + return batch + + +class MXFaceDataset(Dataset): + def __init__(self, root_dir, local_rank): + super(MXFaceDataset, self).__init__() + self.transform = transforms.Compose( + [transforms.ToPILImage(), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + ]) + self.root_dir = root_dir + self.local_rank = local_rank + path_imgrec = os.path.join(root_dir, 'train.rec') + path_imgidx = os.path.join(root_dir, 'train.idx') + self.imgrec = mx.recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') + s = self.imgrec.read_idx(0) + header, _ = mx.recordio.unpack(s) + if header.flag > 0: + self.header0 = (int(header.label[0]), int(header.label[1])) + self.imgidx = np.array(range(1, int(header.label[0]))) + else: + self.imgidx = np.array(list(self.imgrec.keys)) + + def __getitem__(self, index): + idx = self.imgidx[index] + s = self.imgrec.read_idx(idx) + header, img = mx.recordio.unpack(s) + label = header.label + if not isinstance(label, numbers.Number): + label = label[0] + label = torch.tensor(label, dtype=torch.long) + sample = mx.image.imdecode(img).asnumpy() + if self.transform is not None: + sample = self.transform(sample) + return sample, label + + def __len__(self): + return len(self.imgidx) + + +class SyntheticDataset(Dataset): + def __init__(self, local_rank): + super(SyntheticDataset, self).__init__() + img = np.random.randint(0, 255, size=(112, 112, 3), dtype=np.int32) + img = np.transpose(img, (2, 0, 1)) + img = torch.from_numpy(img).squeeze(0).float() + img = ((img / 255) - 0.5) / 0.5 + self.img = img + self.label = 1 + + def __getitem__(self, index): + return self.img, self.label + + def __len__(self): + return 1000000 diff --git a/src/face3d/models/arcface_torch/docs/eval.md b/src/face3d/models/arcface_torch/docs/eval.md new file mode 100644 index 0000000000000000000000000000000000000000..dd1d9e257367b6422680966198646c45e5a2671d --- /dev/null +++ b/src/face3d/models/arcface_torch/docs/eval.md @@ -0,0 +1,31 @@ +## Eval on ICCV2021-MFR + +coming soon. + + +## Eval IJBC +You can eval ijbc with pytorch or onnx. + + +1. Eval IJBC With Onnx +```shell +CUDA_VISIBLE_DEVICES=0 python onnx_ijbc.py --model-root ms1mv3_arcface_r50 --image-path IJB_release/IJBC --result-dir ms1mv3_arcface_r50 +``` + +2. Eval IJBC With Pytorch +```shell +CUDA_VISIBLE_DEVICES=0,1 python eval_ijbc.py \ +--model-prefix ms1mv3_arcface_r50/backbone.pth \ +--image-path IJB_release/IJBC \ +--result-dir ms1mv3_arcface_r50 \ +--batch-size 128 \ +--job ms1mv3_arcface_r50 \ +--target IJBC \ +--network iresnet50 +``` + +## Inference + +```shell +python inference.py --weight ms1mv3_arcface_r50/backbone.pth --network r50 +``` diff --git a/src/face3d/models/arcface_torch/docs/install.md b/src/face3d/models/arcface_torch/docs/install.md new file mode 100644 index 0000000000000000000000000000000000000000..6314a40441285e9236438e468caf8b71a407531a --- /dev/null +++ b/src/face3d/models/arcface_torch/docs/install.md @@ -0,0 +1,51 @@ +## v1.8.0 +### Linux and Windows +```shell +# CUDA 11.0 +pip --default-timeout=100 install torch==1.8.0+cu111 torchvision==0.9.0+cu111 torchaudio==0.8.0 -f https://download.pytorch.org/whl/torch_stable.html + +# CUDA 10.2 +pip --default-timeout=100 install torch==1.8.0 torchvision==0.9.0 torchaudio==0.8.0 + +# CPU only +pip --default-timeout=100 install torch==1.8.0+cpu torchvision==0.9.0+cpu torchaudio==0.8.0 -f https://download.pytorch.org/whl/torch_stable.html + +``` + + +## v1.7.1 +### Linux and Windows +```shell +# CUDA 11.0 +pip install torch==1.7.1+cu110 torchvision==0.8.2+cu110 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html + +# CUDA 10.2 +pip install torch==1.7.1 torchvision==0.8.2 torchaudio==0.7.2 + +# CUDA 10.1 +pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html + +# CUDA 9.2 +pip install torch==1.7.1+cu92 torchvision==0.8.2+cu92 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html + +# CPU only +pip install torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html +``` + + +## v1.6.0 + +### Linux and Windows +```shell +# CUDA 10.2 +pip install torch==1.6.0 torchvision==0.7.0 + +# CUDA 10.1 +pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html + +# CUDA 9.2 +pip install torch==1.6.0+cu92 torchvision==0.7.0+cu92 -f https://download.pytorch.org/whl/torch_stable.html + +# CPU only +pip install torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html +``` \ No newline at end of file diff --git a/src/face3d/models/arcface_torch/docs/modelzoo.md b/src/face3d/models/arcface_torch/docs/modelzoo.md new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/face3d/models/arcface_torch/docs/speed_benchmark.md b/src/face3d/models/arcface_torch/docs/speed_benchmark.md new file mode 100644 index 0000000000000000000000000000000000000000..055aee0defe2c43a523ced48260242f0f99b7cea --- /dev/null +++ b/src/face3d/models/arcface_torch/docs/speed_benchmark.md @@ -0,0 +1,93 @@ +## Test Training Speed + +- Test Commands + +You need to use the following two commands to test the Partial FC training performance. +The number of identites is **3 millions** (synthetic data), turn mixed precision training on, backbone is resnet50, +batch size is 1024. +```shell +# Model Parallel +python -m torch.distributed.launch --nproc_per_node=8 --nnodes=1 --node_rank=0 --master_addr="127.0.0.1" --master_port=1234 train.py configs/3millions +# Partial FC 0.1 +python -m torch.distributed.launch --nproc_per_node=8 --nnodes=1 --node_rank=0 --master_addr="127.0.0.1" --master_port=1234 train.py configs/3millions_pfc +``` + +- GPU Memory + +``` +# (Model Parallel) gpustat -i +[0] Tesla V100-SXM2-32GB | 64'C, 94 % | 30338 / 32510 MB +[1] Tesla V100-SXM2-32GB | 60'C, 99 % | 28876 / 32510 MB +[2] Tesla V100-SXM2-32GB | 60'C, 99 % | 28872 / 32510 MB +[3] Tesla V100-SXM2-32GB | 69'C, 99 % | 28872 / 32510 MB +[4] Tesla V100-SXM2-32GB | 66'C, 99 % | 28888 / 32510 MB +[5] Tesla V100-SXM2-32GB | 60'C, 99 % | 28932 / 32510 MB +[6] Tesla V100-SXM2-32GB | 68'C, 100 % | 28916 / 32510 MB +[7] Tesla V100-SXM2-32GB | 65'C, 99 % | 28860 / 32510 MB + +# (Partial FC 0.1) gpustat -i +[0] Tesla V100-SXM2-32GB | 60'C, 95 % | 10488 / 32510 MB │······················· +[1] Tesla V100-SXM2-32GB | 60'C, 97 % | 10344 / 32510 MB │······················· +[2] Tesla V100-SXM2-32GB | 61'C, 95 % | 10340 / 32510 MB │······················· +[3] Tesla V100-SXM2-32GB | 66'C, 95 % | 10340 / 32510 MB │······················· +[4] Tesla V100-SXM2-32GB | 65'C, 94 % | 10356 / 32510 MB │······················· +[5] Tesla V100-SXM2-32GB | 61'C, 95 % | 10400 / 32510 MB │······················· +[6] Tesla V100-SXM2-32GB | 68'C, 96 % | 10384 / 32510 MB │······················· +[7] Tesla V100-SXM2-32GB | 64'C, 95 % | 10328 / 32510 MB │······················· +``` + +- Training Speed + +```python +# (Model Parallel) trainging.log +Training: Speed 2271.33 samples/sec Loss 1.1624 LearningRate 0.2000 Epoch: 0 Global Step: 100 +Training: Speed 2269.94 samples/sec Loss 0.0000 LearningRate 0.2000 Epoch: 0 Global Step: 150 +Training: Speed 2272.67 samples/sec Loss 0.0000 LearningRate 0.2000 Epoch: 0 Global Step: 200 +Training: Speed 2266.55 samples/sec Loss 0.0000 LearningRate 0.2000 Epoch: 0 Global Step: 250 +Training: Speed 2272.54 samples/sec Loss 0.0000 LearningRate 0.2000 Epoch: 0 Global Step: 300 + +# (Partial FC 0.1) trainging.log +Training: Speed 5299.56 samples/sec Loss 1.0965 LearningRate 0.2000 Epoch: 0 Global Step: 100 +Training: Speed 5296.37 samples/sec Loss 0.0000 LearningRate 0.2000 Epoch: 0 Global Step: 150 +Training: Speed 5304.37 samples/sec Loss 0.0000 LearningRate 0.2000 Epoch: 0 Global Step: 200 +Training: Speed 5274.43 samples/sec Loss 0.0000 LearningRate 0.2000 Epoch: 0 Global Step: 250 +Training: Speed 5300.10 samples/sec Loss 0.0000 LearningRate 0.2000 Epoch: 0 Global Step: 300 +``` + +In this test case, Partial FC 0.1 only use1 1/3 of the GPU memory of the model parallel, +and the training speed is 2.5 times faster than the model parallel. + + +## Speed Benchmark + +1. Training speed of different parallel methods (samples/second), Tesla V100 32GB * 8. (Larger is better) + +| Number of Identities in Dataset | Data Parallel | Model Parallel | Partial FC 0.1 | +| :--- | :--- | :--- | :--- | +|125000 | 4681 | 4824 | 5004 | +|250000 | 4047 | 4521 | 4976 | +|500000 | 3087 | 4013 | 4900 | +|1000000 | 2090 | 3449 | 4803 | +|1400000 | 1672 | 3043 | 4738 | +|2000000 | - | 2593 | 4626 | +|4000000 | - | 1748 | 4208 | +|5500000 | - | 1389 | 3975 | +|8000000 | - | - | 3565 | +|16000000 | - | - | 2679 | +|29000000 | - | - | 1855 | + +2. GPU memory cost of different parallel methods (GB per GPU), Tesla V100 32GB * 8. (Smaller is better) + +| Number of Identities in Dataset | Data Parallel | Model Parallel | Partial FC 0.1 | +| :--- | :--- | :--- | :--- | +|125000 | 7358 | 5306 | 4868 | +|250000 | 9940 | 5826 | 5004 | +|500000 | 14220 | 7114 | 5202 | +|1000000 | 23708 | 9966 | 5620 | +|1400000 | 32252 | 11178 | 6056 | +|2000000 | - | 13978 | 6472 | +|4000000 | - | 23238 | 8284 | +|5500000 | - | 32188 | 9854 | +|8000000 | - | - | 12310 | +|16000000 | - | - | 19950 | +|29000000 | - | - | 32324 | diff --git a/src/face3d/models/arcface_torch/eval/__init__.py b/src/face3d/models/arcface_torch/eval/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/face3d/models/arcface_torch/eval/verification.py b/src/face3d/models/arcface_torch/eval/verification.py new file mode 100644 index 0000000000000000000000000000000000000000..253343b83dbf9d1bd154d14ec068e098bf0968db --- /dev/null +++ b/src/face3d/models/arcface_torch/eval/verification.py @@ -0,0 +1,407 @@ +"""Helper for evaluation on the Labeled Faces in the Wild dataset +""" + +# MIT License +# +# Copyright (c) 2016 David Sandberg +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import datetime +import os +import pickle + +import mxnet as mx +import numpy as np +import sklearn +import torch +from mxnet import ndarray as nd +from scipy import interpolate +from sklearn.decomposition import PCA +from sklearn.model_selection import KFold + + +class LFold: + def __init__(self, n_splits=2, shuffle=False): + self.n_splits = n_splits + if self.n_splits > 1: + self.k_fold = KFold(n_splits=n_splits, shuffle=shuffle) + + def split(self, indices): + if self.n_splits > 1: + return self.k_fold.split(indices) + else: + return [(indices, indices)] + + +def calculate_roc(thresholds, + embeddings1, + embeddings2, + actual_issame, + nrof_folds=10, + pca=0): + assert (embeddings1.shape[0] == embeddings2.shape[0]) + assert (embeddings1.shape[1] == embeddings2.shape[1]) + nrof_pairs = min(len(actual_issame), embeddings1.shape[0]) + nrof_thresholds = len(thresholds) + k_fold = LFold(n_splits=nrof_folds, shuffle=False) + + tprs = np.zeros((nrof_folds, nrof_thresholds)) + fprs = np.zeros((nrof_folds, nrof_thresholds)) + accuracy = np.zeros((nrof_folds)) + indices = np.arange(nrof_pairs) + + if pca == 0: + diff = np.subtract(embeddings1, embeddings2) + dist = np.sum(np.square(diff), 1) + + for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)): + if pca > 0: + print('doing pca on', fold_idx) + embed1_train = embeddings1[train_set] + embed2_train = embeddings2[train_set] + _embed_train = np.concatenate((embed1_train, embed2_train), axis=0) + pca_model = PCA(n_components=pca) + pca_model.fit(_embed_train) + embed1 = pca_model.transform(embeddings1) + embed2 = pca_model.transform(embeddings2) + embed1 = sklearn.preprocessing.normalize(embed1) + embed2 = sklearn.preprocessing.normalize(embed2) + diff = np.subtract(embed1, embed2) + dist = np.sum(np.square(diff), 1) + + # Find the best threshold for the fold + acc_train = np.zeros((nrof_thresholds)) + for threshold_idx, threshold in enumerate(thresholds): + _, _, acc_train[threshold_idx] = calculate_accuracy( + threshold, dist[train_set], actual_issame[train_set]) + best_threshold_index = np.argmax(acc_train) + for threshold_idx, threshold in enumerate(thresholds): + tprs[fold_idx, threshold_idx], fprs[fold_idx, threshold_idx], _ = calculate_accuracy( + threshold, dist[test_set], + actual_issame[test_set]) + _, _, accuracy[fold_idx] = calculate_accuracy( + thresholds[best_threshold_index], dist[test_set], + actual_issame[test_set]) + + tpr = np.mean(tprs, 0) + fpr = np.mean(fprs, 0) + return tpr, fpr, accuracy + + +def calculate_accuracy(threshold, dist, actual_issame): + predict_issame = np.less(dist, threshold) + tp = np.sum(np.logical_and(predict_issame, actual_issame)) + fp = np.sum(np.logical_and(predict_issame, np.logical_not(actual_issame))) + tn = np.sum( + np.logical_and(np.logical_not(predict_issame), + np.logical_not(actual_issame))) + fn = np.sum(np.logical_and(np.logical_not(predict_issame), actual_issame)) + + tpr = 0 if (tp + fn == 0) else float(tp) / float(tp + fn) + fpr = 0 if (fp + tn == 0) else float(fp) / float(fp + tn) + acc = float(tp + tn) / dist.size + return tpr, fpr, acc + + +def calculate_val(thresholds, + embeddings1, + embeddings2, + actual_issame, + far_target, + nrof_folds=10): + assert (embeddings1.shape[0] == embeddings2.shape[0]) + assert (embeddings1.shape[1] == embeddings2.shape[1]) + nrof_pairs = min(len(actual_issame), embeddings1.shape[0]) + nrof_thresholds = len(thresholds) + k_fold = LFold(n_splits=nrof_folds, shuffle=False) + + val = np.zeros(nrof_folds) + far = np.zeros(nrof_folds) + + diff = np.subtract(embeddings1, embeddings2) + dist = np.sum(np.square(diff), 1) + indices = np.arange(nrof_pairs) + + for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)): + + # Find the threshold that gives FAR = far_target + far_train = np.zeros(nrof_thresholds) + for threshold_idx, threshold in enumerate(thresholds): + _, far_train[threshold_idx] = calculate_val_far( + threshold, dist[train_set], actual_issame[train_set]) + if np.max(far_train) >= far_target: + f = interpolate.interp1d(far_train, thresholds, kind='slinear') + threshold = f(far_target) + else: + threshold = 0.0 + + val[fold_idx], far[fold_idx] = calculate_val_far( + threshold, dist[test_set], actual_issame[test_set]) + + val_mean = np.mean(val) + far_mean = np.mean(far) + val_std = np.std(val) + return val_mean, val_std, far_mean + + +def calculate_val_far(threshold, dist, actual_issame): + predict_issame = np.less(dist, threshold) + true_accept = np.sum(np.logical_and(predict_issame, actual_issame)) + false_accept = np.sum( + np.logical_and(predict_issame, np.logical_not(actual_issame))) + n_same = np.sum(actual_issame) + n_diff = np.sum(np.logical_not(actual_issame)) + # print(true_accept, false_accept) + # print(n_same, n_diff) + val = float(true_accept) / float(n_same) + far = float(false_accept) / float(n_diff) + return val, far + + +def evaluate(embeddings, actual_issame, nrof_folds=10, pca=0): + # Calculate evaluation metrics + thresholds = np.arange(0, 4, 0.01) + embeddings1 = embeddings[0::2] + embeddings2 = embeddings[1::2] + tpr, fpr, accuracy = calculate_roc(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + nrof_folds=nrof_folds, + pca=pca) + thresholds = np.arange(0, 4, 0.001) + val, val_std, far = calculate_val(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + 1e-3, + nrof_folds=nrof_folds) + return tpr, fpr, accuracy, val, val_std, far + +@torch.no_grad() +def load_bin(path, image_size): + try: + with open(path, 'rb') as f: + bins, issame_list = pickle.load(f) # py2 + except UnicodeDecodeError as e: + with open(path, 'rb') as f: + bins, issame_list = pickle.load(f, encoding='bytes') # py3 + data_list = [] + for flip in [0, 1]: + data = torch.empty((len(issame_list) * 2, 3, image_size[0], image_size[1])) + data_list.append(data) + for idx in range(len(issame_list) * 2): + _bin = bins[idx] + img = mx.image.imdecode(_bin) + if img.shape[1] != image_size[0]: + img = mx.image.resize_short(img, image_size[0]) + img = nd.transpose(img, axes=(2, 0, 1)) + for flip in [0, 1]: + if flip == 1: + img = mx.ndarray.flip(data=img, axis=2) + data_list[flip][idx][:] = torch.from_numpy(img.asnumpy()) + if idx % 1000 == 0: + print('loading bin', idx) + print(data_list[0].shape) + return data_list, issame_list + +@torch.no_grad() +def test(data_set, backbone, batch_size, nfolds=10): + print('testing verification..') + data_list = data_set[0] + issame_list = data_set[1] + embeddings_list = [] + time_consumed = 0.0 + for i in range(len(data_list)): + data = data_list[i] + embeddings = None + ba = 0 + while ba < data.shape[0]: + bb = min(ba + batch_size, data.shape[0]) + count = bb - ba + _data = data[bb - batch_size: bb] + time0 = datetime.datetime.now() + img = ((_data / 255) - 0.5) / 0.5 + net_out: torch.Tensor = backbone(img) + _embeddings = net_out.detach().cpu().numpy() + time_now = datetime.datetime.now() + diff = time_now - time0 + time_consumed += diff.total_seconds() + if embeddings is None: + embeddings = np.zeros((data.shape[0], _embeddings.shape[1])) + embeddings[ba:bb, :] = _embeddings[(batch_size - count):, :] + ba = bb + embeddings_list.append(embeddings) + + _xnorm = 0.0 + _xnorm_cnt = 0 + for embed in embeddings_list: + for i in range(embed.shape[0]): + _em = embed[i] + _norm = np.linalg.norm(_em) + _xnorm += _norm + _xnorm_cnt += 1 + _xnorm /= _xnorm_cnt + + acc1 = 0.0 + std1 = 0.0 + embeddings = embeddings_list[0] + embeddings_list[1] + embeddings = sklearn.preprocessing.normalize(embeddings) + print(embeddings.shape) + print('infer time', time_consumed) + _, _, accuracy, val, val_std, far = evaluate(embeddings, issame_list, nrof_folds=nfolds) + acc2, std2 = np.mean(accuracy), np.std(accuracy) + return acc1, std1, acc2, std2, _xnorm, embeddings_list + + +def dumpR(data_set, + backbone, + batch_size, + name='', + data_extra=None, + label_shape=None): + print('dump verification embedding..') + data_list = data_set[0] + issame_list = data_set[1] + embeddings_list = [] + time_consumed = 0.0 + for i in range(len(data_list)): + data = data_list[i] + embeddings = None + ba = 0 + while ba < data.shape[0]: + bb = min(ba + batch_size, data.shape[0]) + count = bb - ba + + _data = nd.slice_axis(data, axis=0, begin=bb - batch_size, end=bb) + time0 = datetime.datetime.now() + if data_extra is None: + db = mx.io.DataBatch(data=(_data,), label=(_label,)) + else: + db = mx.io.DataBatch(data=(_data, _data_extra), + label=(_label,)) + model.forward(db, is_train=False) + net_out = model.get_outputs() + _embeddings = net_out[0].asnumpy() + time_now = datetime.datetime.now() + diff = time_now - time0 + time_consumed += diff.total_seconds() + if embeddings is None: + embeddings = np.zeros((data.shape[0], _embeddings.shape[1])) + embeddings[ba:bb, :] = _embeddings[(batch_size - count):, :] + ba = bb + embeddings_list.append(embeddings) + embeddings = embeddings_list[0] + embeddings_list[1] + embeddings = sklearn.preprocessing.normalize(embeddings) + actual_issame = np.asarray(issame_list) + outname = os.path.join('temp.bin') + with open(outname, 'wb') as f: + pickle.dump((embeddings, issame_list), + f, + protocol=pickle.HIGHEST_PROTOCOL) + + +# if __name__ == '__main__': +# +# parser = argparse.ArgumentParser(description='do verification') +# # general +# parser.add_argument('--data-dir', default='', help='') +# parser.add_argument('--model', +# default='../model/softmax,50', +# help='path to load model.') +# parser.add_argument('--target', +# default='lfw,cfp_ff,cfp_fp,agedb_30', +# help='test targets.') +# parser.add_argument('--gpu', default=0, type=int, help='gpu id') +# parser.add_argument('--batch-size', default=32, type=int, help='') +# parser.add_argument('--max', default='', type=str, help='') +# parser.add_argument('--mode', default=0, type=int, help='') +# parser.add_argument('--nfolds', default=10, type=int, help='') +# args = parser.parse_args() +# image_size = [112, 112] +# print('image_size', image_size) +# ctx = mx.gpu(args.gpu) +# nets = [] +# vec = args.model.split(',') +# prefix = args.model.split(',')[0] +# epochs = [] +# if len(vec) == 1: +# pdir = os.path.dirname(prefix) +# for fname in os.listdir(pdir): +# if not fname.endswith('.params'): +# continue +# _file = os.path.join(pdir, fname) +# if _file.startswith(prefix): +# epoch = int(fname.split('.')[0].split('-')[1]) +# epochs.append(epoch) +# epochs = sorted(epochs, reverse=True) +# if len(args.max) > 0: +# _max = [int(x) for x in args.max.split(',')] +# assert len(_max) == 2 +# if len(epochs) > _max[1]: +# epochs = epochs[_max[0]:_max[1]] +# +# else: +# epochs = [int(x) for x in vec[1].split('|')] +# print('model number', len(epochs)) +# time0 = datetime.datetime.now() +# for epoch in epochs: +# print('loading', prefix, epoch) +# sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) +# # arg_params, aux_params = ch_dev(arg_params, aux_params, ctx) +# all_layers = sym.get_internals() +# sym = all_layers['fc1_output'] +# model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) +# # model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) +# model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], +# image_size[1]))]) +# model.set_params(arg_params, aux_params) +# nets.append(model) +# time_now = datetime.datetime.now() +# diff = time_now - time0 +# print('model loading time', diff.total_seconds()) +# +# ver_list = [] +# ver_name_list = [] +# for name in args.target.split(','): +# path = os.path.join(args.data_dir, name + ".bin") +# if os.path.exists(path): +# print('loading.. ', name) +# data_set = load_bin(path, image_size) +# ver_list.append(data_set) +# ver_name_list.append(name) +# +# if args.mode == 0: +# for i in range(len(ver_list)): +# results = [] +# for model in nets: +# acc1, std1, acc2, std2, xnorm, embeddings_list = test( +# ver_list[i], model, args.batch_size, args.nfolds) +# print('[%s]XNorm: %f' % (ver_name_list[i], xnorm)) +# print('[%s]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], acc1, std1)) +# print('[%s]Accuracy-Flip: %1.5f+-%1.5f' % (ver_name_list[i], acc2, std2)) +# results.append(acc2) +# print('Max of [%s] is %1.5f' % (ver_name_list[i], np.max(results))) +# elif args.mode == 1: +# raise ValueError +# else: +# model = nets[0] +# dumpR(ver_list[0], model, args.batch_size, args.target) diff --git a/src/face3d/models/arcface_torch/eval_ijbc.py b/src/face3d/models/arcface_torch/eval_ijbc.py new file mode 100644 index 0000000000000000000000000000000000000000..9c5a650d486d18eb02d6f60d448fc3b315261f5d --- /dev/null +++ b/src/face3d/models/arcface_torch/eval_ijbc.py @@ -0,0 +1,483 @@ +# coding: utf-8 + +import os +import pickle + +import matplotlib +import pandas as pd + +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import timeit +import sklearn +import argparse +import cv2 +import numpy as np +import torch +from skimage import transform as trans +from backbones import get_model +from sklearn.metrics import roc_curve, auc + +from menpo.visualize.viewmatplotlib import sample_colours_from_colourmap +from prettytable import PrettyTable +from pathlib import Path + +import sys +import warnings + +sys.path.insert(0, "../") +warnings.filterwarnings("ignore") + +parser = argparse.ArgumentParser(description='do ijb test') +# general +parser.add_argument('--model-prefix', default='', help='path to load model.') +parser.add_argument('--image-path', default='', type=str, help='') +parser.add_argument('--result-dir', default='.', type=str, help='') +parser.add_argument('--batch-size', default=128, type=int, help='') +parser.add_argument('--network', default='iresnet50', type=str, help='') +parser.add_argument('--job', default='insightface', type=str, help='job name') +parser.add_argument('--target', default='IJBC', type=str, help='target, set to IJBC or IJBB') +args = parser.parse_args() + +target = args.target +model_path = args.model_prefix +image_path = args.image_path +result_dir = args.result_dir +gpu_id = None +use_norm_score = True # if Ture, TestMode(N1) +use_detector_score = True # if Ture, TestMode(D1) +use_flip_test = True # if Ture, TestMode(F1) +job = args.job +batch_size = args.batch_size + + +class Embedding(object): + def __init__(self, prefix, data_shape, batch_size=1): + image_size = (112, 112) + self.image_size = image_size + weight = torch.load(prefix) + resnet = get_model(args.network, dropout=0, fp16=False).cuda() + resnet.load_state_dict(weight) + model = torch.nn.DataParallel(resnet) + self.model = model + self.model.eval() + src = np.array([ + [30.2946, 51.6963], + [65.5318, 51.5014], + [48.0252, 71.7366], + [33.5493, 92.3655], + [62.7299, 92.2041]], dtype=np.float32) + src[:, 0] += 8.0 + self.src = src + self.batch_size = batch_size + self.data_shape = data_shape + + def get(self, rimg, landmark): + + assert landmark.shape[0] == 68 or landmark.shape[0] == 5 + assert landmark.shape[1] == 2 + if landmark.shape[0] == 68: + landmark5 = np.zeros((5, 2), dtype=np.float32) + landmark5[0] = (landmark[36] + landmark[39]) / 2 + landmark5[1] = (landmark[42] + landmark[45]) / 2 + landmark5[2] = landmark[30] + landmark5[3] = landmark[48] + landmark5[4] = landmark[54] + else: + landmark5 = landmark + tform = trans.SimilarityTransform() + tform.estimate(landmark5, self.src) + M = tform.params[0:2, :] + img = cv2.warpAffine(rimg, + M, (self.image_size[1], self.image_size[0]), + borderValue=0.0) + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + img_flip = np.fliplr(img) + img = np.transpose(img, (2, 0, 1)) # 3*112*112, RGB + img_flip = np.transpose(img_flip, (2, 0, 1)) + input_blob = np.zeros((2, 3, self.image_size[1], self.image_size[0]), dtype=np.uint8) + input_blob[0] = img + input_blob[1] = img_flip + return input_blob + + @torch.no_grad() + def forward_db(self, batch_data): + imgs = torch.Tensor(batch_data).cuda() + imgs.div_(255).sub_(0.5).div_(0.5) + feat = self.model(imgs) + feat = feat.reshape([self.batch_size, 2 * feat.shape[1]]) + return feat.cpu().numpy() + + +# 将一个list尽量均分成n份，限制len(list)==n，份数大于原list内元素个数则分配空list[] +def divideIntoNstrand(listTemp, n): + twoList = [[] for i in range(n)] + for i, e in enumerate(listTemp): + twoList[i % n].append(e) + return twoList + + +def read_template_media_list(path): + # ijb_meta = np.loadtxt(path, dtype=str) + ijb_meta = pd.read_csv(path, sep=' ', header=None).values + templates = ijb_meta[:, 1].astype(np.int) + medias = ijb_meta[:, 2].astype(np.int) + return templates, medias + + +# In[ ]: + + +def read_template_pair_list(path): + # pairs = np.loadtxt(path, dtype=str) + pairs = pd.read_csv(path, sep=' ', header=None).values + # print(pairs.shape) + # print(pairs[:, 0].astype(np.int)) + t1 = pairs[:, 0].astype(np.int) + t2 = pairs[:, 1].astype(np.int) + label = pairs[:, 2].astype(np.int) + return t1, t2, label + + +# In[ ]: + + +def read_image_feature(path): + with open(path, 'rb') as fid: + img_feats = pickle.load(fid) + return img_feats + + +# In[ ]: + + +def get_image_feature(img_path, files_list, model_path, epoch, gpu_id): + batch_size = args.batch_size + data_shape = (3, 112, 112) + + files = files_list + print('files:', len(files)) + rare_size = len(files) % batch_size + faceness_scores = [] + batch = 0 + img_feats = np.empty((len(files), 1024), dtype=np.float32) + + batch_data = np.empty((2 * batch_size, 3, 112, 112)) + embedding = Embedding(model_path, data_shape, batch_size) + for img_index, each_line in enumerate(files[:len(files) - rare_size]): + name_lmk_score = each_line.strip().split(' ') + img_name = os.path.join(img_path, name_lmk_score[0]) + img = cv2.imread(img_name) + lmk = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) + lmk = lmk.reshape((5, 2)) + input_blob = embedding.get(img, lmk) + + batch_data[2 * (img_index - batch * batch_size)][:] = input_blob[0] + batch_data[2 * (img_index - batch * batch_size) + 1][:] = input_blob[1] + if (img_index + 1) % batch_size == 0: + print('batch', batch) + img_feats[batch * batch_size:batch * batch_size + + batch_size][:] = embedding.forward_db(batch_data) + batch += 1 + faceness_scores.append(name_lmk_score[-1]) + + batch_data = np.empty((2 * rare_size, 3, 112, 112)) + embedding = Embedding(model_path, data_shape, rare_size) + for img_index, each_line in enumerate(files[len(files) - rare_size:]): + name_lmk_score = each_line.strip().split(' ') + img_name = os.path.join(img_path, name_lmk_score[0]) + img = cv2.imread(img_name) + lmk = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) + lmk = lmk.reshape((5, 2)) + input_blob = embedding.get(img, lmk) + batch_data[2 * img_index][:] = input_blob[0] + batch_data[2 * img_index + 1][:] = input_blob[1] + if (img_index + 1) % rare_size == 0: + print('batch', batch) + img_feats[len(files) - + rare_size:][:] = embedding.forward_db(batch_data) + batch += 1 + faceness_scores.append(name_lmk_score[-1]) + faceness_scores = np.array(faceness_scores).astype(np.float32) + # img_feats = np.ones( (len(files), 1024), dtype=np.float32) * 0.01 + # faceness_scores = np.ones( (len(files), ), dtype=np.float32 ) + return img_feats, faceness_scores + + +# In[ ]: + + +def image2template_feature(img_feats=None, templates=None, medias=None): + # ========================================================== + # 1. face image feature l2 normalization. img_feats:[number_image x feats_dim] + # 2. compute media feature. + # 3. compute template feature. + # ========================================================== + unique_templates = np.unique(templates) + template_feats = np.zeros((len(unique_templates), img_feats.shape[1])) + + for count_template, uqt in enumerate(unique_templates): + + (ind_t,) = np.where(templates == uqt) + face_norm_feats = img_feats[ind_t] + face_medias = medias[ind_t] + unique_medias, unique_media_counts = np.unique(face_medias, + return_counts=True) + media_norm_feats = [] + for u, ct in zip(unique_medias, unique_media_counts): + (ind_m,) = np.where(face_medias == u) + if ct == 1: + media_norm_feats += [face_norm_feats[ind_m]] + else: # image features from the same video will be aggregated into one feature + media_norm_feats += [ + np.mean(face_norm_feats[ind_m], axis=0, keepdims=True) + ] + media_norm_feats = np.array(media_norm_feats) + # media_norm_feats = media_norm_feats / np.sqrt(np.sum(media_norm_feats ** 2, -1, keepdims=True)) + template_feats[count_template] = np.sum(media_norm_feats, axis=0) + if count_template % 2000 == 0: + print('Finish Calculating {} template features.'.format( + count_template)) + # template_norm_feats = template_feats / np.sqrt(np.sum(template_feats ** 2, -1, keepdims=True)) + template_norm_feats = sklearn.preprocessing.normalize(template_feats) + # print(template_norm_feats.shape) + return template_norm_feats, unique_templates + + +# In[ ]: + + +def verification(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): + # ========================================================== + # Compute set-to-set Similarity Score. + # ========================================================== + template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) + for count_template, uqt in enumerate(unique_templates): + template2id[uqt] = count_template + + score = np.zeros((len(p1),)) # save cosine distance between pairs + + total_pairs = np.array(range(len(p1))) + batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] + total_sublists = len(sublists) + for c, s in enumerate(sublists): + feat1 = template_norm_feats[template2id[p1[s]]] + feat2 = template_norm_feats[template2id[p2[s]]] + similarity_score = np.sum(feat1 * feat2, -1) + score[s] = similarity_score.flatten() + if c % 10 == 0: + print('Finish {}/{} pairs.'.format(c, total_sublists)) + return score + + +# In[ ]: +def verification2(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): + template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) + for count_template, uqt in enumerate(unique_templates): + template2id[uqt] = count_template + score = np.zeros((len(p1),)) # save cosine distance between pairs + total_pairs = np.array(range(len(p1))) + batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] + total_sublists = len(sublists) + for c, s in enumerate(sublists): + feat1 = template_norm_feats[template2id[p1[s]]] + feat2 = template_norm_feats[template2id[p2[s]]] + similarity_score = np.sum(feat1 * feat2, -1) + score[s] = similarity_score.flatten() + if c % 10 == 0: + print('Finish {}/{} pairs.'.format(c, total_sublists)) + return score + + +def read_score(path): + with open(path, 'rb') as fid: + img_feats = pickle.load(fid) + return img_feats + + +# # Step1: Load Meta Data + +# In[ ]: + +assert target == 'IJBC' or target == 'IJBB' + +# ============================================================= +# load image and template relationships for template feature embedding +# tid --> template id, mid --> media id +# format: +# image_name tid mid +# ============================================================= +start = timeit.default_timer() +templates, medias = read_template_media_list( + os.path.join('%s/meta' % image_path, + '%s_face_tid_mid.txt' % target.lower())) +stop = timeit.default_timer() +print('Time: %.2f s. ' % (stop - start)) + +# In[ ]: + +# ============================================================= +# load template pairs for template-to-template verification +# tid : template id, label : 1/0 +# format: +# tid_1 tid_2 label +# ============================================================= +start = timeit.default_timer() +p1, p2, label = read_template_pair_list( + os.path.join('%s/meta' % image_path, + '%s_template_pair_label.txt' % target.lower())) +stop = timeit.default_timer() +print('Time: %.2f s. ' % (stop - start)) + +# # Step 2: Get Image Features + +# In[ ]: + +# ============================================================= +# load image features +# format: +# img_feats: [image_num x feats_dim] (227630, 512) +# ============================================================= +start = timeit.default_timer() +img_path = '%s/loose_crop' % image_path +img_list_path = '%s/meta/%s_name_5pts_score.txt' % (image_path, target.lower()) +img_list = open(img_list_path) +files = img_list.readlines() +# files_list = divideIntoNstrand(files, rank_size) +files_list = files + +# img_feats +# for i in range(rank_size): +img_feats, faceness_scores = get_image_feature(img_path, files_list, + model_path, 0, gpu_id) +stop = timeit.default_timer() +print('Time: %.2f s. ' % (stop - start)) +print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], + img_feats.shape[1])) + +# # Step3: Get Template Features + +# In[ ]: + +# ============================================================= +# compute template features from image features. +# ============================================================= +start = timeit.default_timer() +# ========================================================== +# Norm feature before aggregation into template feature? +# Feature norm from embedding network and faceness score are able to decrease weights for noise samples (not face). +# ========================================================== +# 1. FaceScore （Feature Norm） +# 2. FaceScore （Detector） + +if use_flip_test: + # concat --- F1 + # img_input_feats = img_feats + # add --- F2 + img_input_feats = img_feats[:, 0:img_feats.shape[1] // + 2] + img_feats[:, img_feats.shape[1] // 2:] +else: + img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] + +if use_norm_score: + img_input_feats = img_input_feats +else: + # normalise features to remove norm information + img_input_feats = img_input_feats / np.sqrt( + np.sum(img_input_feats ** 2, -1, keepdims=True)) + +if use_detector_score: + print(img_input_feats.shape, faceness_scores.shape) + img_input_feats = img_input_feats * faceness_scores[:, np.newaxis] +else: + img_input_feats = img_input_feats + +template_norm_feats, unique_templates = image2template_feature( + img_input_feats, templates, medias) +stop = timeit.default_timer() +print('Time: %.2f s. ' % (stop - start)) + +# # Step 4: Get Template Similarity Scores + +# In[ ]: + +# ============================================================= +# compute verification scores between template pairs. +# ============================================================= +start = timeit.default_timer() +score = verification(template_norm_feats, unique_templates, p1, p2) +stop = timeit.default_timer() +print('Time: %.2f s. ' % (stop - start)) + +# In[ ]: +save_path = os.path.join(result_dir, args.job) +# save_path = result_dir + '/%s_result' % target + +if not os.path.exists(save_path): + os.makedirs(save_path) + +score_save_file = os.path.join(save_path, "%s.npy" % target.lower()) +np.save(score_save_file, score) + +# # Step 5: Get ROC Curves and TPR@FPR Table + +# In[ ]: + +files = [score_save_file] +methods = [] +scores = [] +for file in files: + methods.append(Path(file).stem) + scores.append(np.load(file)) + +methods = np.array(methods) +scores = dict(zip(methods, scores)) +colours = dict( + zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) +x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] +tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) +fig = plt.figure() +for method in methods: + fpr, tpr, _ = roc_curve(label, scores[method]) + roc_auc = auc(fpr, tpr) + fpr = np.flipud(fpr) + tpr = np.flipud(tpr) # select largest tpr at same fpr + plt.plot(fpr, + tpr, + color=colours[method], + lw=1, + label=('[%s (AUC = %0.4f %%)]' % + (method.split('-')[-1], roc_auc * 100))) + tpr_fpr_row = [] + tpr_fpr_row.append("%s-%s" % (method, target)) + for fpr_iter in np.arange(len(x_labels)): + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) + tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) + tpr_fpr_table.add_row(tpr_fpr_row) +plt.xlim([10 ** -6, 0.1]) +plt.ylim([0.3, 1.0]) +plt.grid(linestyle='--', linewidth=1) +plt.xticks(x_labels) +plt.yticks(np.linspace(0.3, 1.0, 8, endpoint=True)) +plt.xscale('log') +plt.xlabel('False Positive Rate') +plt.ylabel('True Positive Rate') +plt.title('ROC on IJB') +plt.legend(loc="lower right") +fig.savefig(os.path.join(save_path, '%s.pdf' % target.lower())) +print(tpr_fpr_table) diff --git a/src/face3d/models/arcface_torch/inference.py b/src/face3d/models/arcface_torch/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..3e5156e8d649954837e397c2ff15ec29995e7502 --- /dev/null +++ b/src/face3d/models/arcface_torch/inference.py @@ -0,0 +1,35 @@ +import argparse + +import cv2 +import numpy as np +import torch + +from backbones import get_model + + +@torch.no_grad() +def inference(weight, name, img): + if img is None: + img = np.random.randint(0, 255, size=(112, 112, 3), dtype=np.uint8) + else: + img = cv2.imread(img) + img = cv2.resize(img, (112, 112)) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + img = np.transpose(img, (2, 0, 1)) + img = torch.from_numpy(img).unsqueeze(0).float() + img.div_(255).sub_(0.5).div_(0.5) + net = get_model(name, fp16=False) + net.load_state_dict(torch.load(weight)) + net.eval() + feat = net(img).numpy() + print(feat) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='PyTorch ArcFace Training') + parser.add_argument('--network', type=str, default='r50', help='backbone network') + parser.add_argument('--weight', type=str, default='') + parser.add_argument('--img', type=str, default=None) + args = parser.parse_args() + inference(args.weight, args.network, args.img) diff --git a/src/face3d/models/arcface_torch/losses.py b/src/face3d/models/arcface_torch/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..87aeaa107af4d53f5a6132b3739d5cafdcded7fc --- /dev/null +++ b/src/face3d/models/arcface_torch/losses.py @@ -0,0 +1,42 @@ +import torch +from torch import nn + + +def get_loss(name): + if name == "cosface": + return CosFace() + elif name == "arcface": + return ArcFace() + else: + raise ValueError() + + +class CosFace(nn.Module): + def __init__(self, s=64.0, m=0.40): + super(CosFace, self).__init__() + self.s = s + self.m = m + + def forward(self, cosine, label): + index = torch.where(label != -1)[0] + m_hot = torch.zeros(index.size()[0], cosine.size()[1], device=cosine.device) + m_hot.scatter_(1, label[index, None], self.m) + cosine[index] -= m_hot + ret = cosine * self.s + return ret + + +class ArcFace(nn.Module): + def __init__(self, s=64.0, m=0.5): + super(ArcFace, self).__init__() + self.s = s + self.m = m + + def forward(self, cosine: torch.Tensor, label): + index = torch.where(label != -1)[0] + m_hot = torch.zeros(index.size()[0], cosine.size()[1], device=cosine.device) + m_hot.scatter_(1, label[index, None], self.m) + cosine.acos_() + cosine[index] += m_hot + cosine.cos_().mul_(self.s) + return cosine diff --git a/src/face3d/models/arcface_torch/onnx_helper.py b/src/face3d/models/arcface_torch/onnx_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..ca922ca6d410655029e459cf8fd1c323d276c34c --- /dev/null +++ b/src/face3d/models/arcface_torch/onnx_helper.py @@ -0,0 +1,250 @@ +from __future__ import division +import datetime +import os +import os.path as osp +import glob +import numpy as np +import cv2 +import sys +import onnxruntime +import onnx +import argparse +from onnx import numpy_helper +from insightface.data import get_image + +class ArcFaceORT: + def __init__(self, model_path, cpu=False): + self.model_path = model_path + # providers = None will use available provider, for onnxruntime-gpu it will be "CUDAExecutionProvider" + self.providers = ['CPUExecutionProvider'] if cpu else None + + #input_size is (w,h), return error message, return None if success + def check(self, track='cfat', test_img = None): + #default is cfat + max_model_size_mb=1024 + max_feat_dim=512 + max_time_cost=15 + if track.startswith('ms1m'): + max_model_size_mb=1024 + max_feat_dim=512 + max_time_cost=10 + elif track.startswith('glint'): + max_model_size_mb=1024 + max_feat_dim=1024 + max_time_cost=20 + elif track.startswith('cfat'): + max_model_size_mb = 1024 + max_feat_dim = 512 + max_time_cost = 15 + elif track.startswith('unconstrained'): + max_model_size_mb=1024 + max_feat_dim=1024 + max_time_cost=30 + else: + return "track not found" + + if not os.path.exists(self.model_path): + return "model_path not exists" + if not os.path.isdir(self.model_path): + return "model_path should be directory" + onnx_files = [] + for _file in os.listdir(self.model_path): + if _file.endswith('.onnx'): + onnx_files.append(osp.join(self.model_path, _file)) + if len(onnx_files)==0: + return "do not have onnx files" + self.model_file = sorted(onnx_files)[-1] + print('use onnx-model:', self.model_file) + try: + session = onnxruntime.InferenceSession(self.model_file, providers=self.providers) + except: + return "load onnx failed" + input_cfg = session.get_inputs()[0] + input_shape = input_cfg.shape + print('input-shape:', input_shape) + if len(input_shape)!=4: + return "length of input_shape should be 4" + if not isinstance(input_shape[0], str): + #return "input_shape[0] should be str to support batch-inference" + print('reset input-shape[0] to None') + model = onnx.load(self.model_file) + model.graph.input[0].type.tensor_type.shape.dim[0].dim_param = 'None' + new_model_file = osp.join(self.model_path, 'zzzzrefined.onnx') + onnx.save(model, new_model_file) + self.model_file = new_model_file + print('use new onnx-model:', self.model_file) + try: + session = onnxruntime.InferenceSession(self.model_file, providers=self.providers) + except: + return "load onnx failed" + input_cfg = session.get_inputs()[0] + input_shape = input_cfg.shape + print('new-input-shape:', input_shape) + + self.image_size = tuple(input_shape[2:4][::-1]) + #print('image_size:', self.image_size) + input_name = input_cfg.name + outputs = session.get_outputs() + output_names = [] + for o in outputs: + output_names.append(o.name) + #print(o.name, o.shape) + if len(output_names)!=1: + return "number of output nodes should be 1" + self.session = session + self.input_name = input_name + self.output_names = output_names + #print(self.output_names) + model = onnx.load(self.model_file) + graph = model.graph + if len(graph.node)<8: + return "too small onnx graph" + + input_size = (112,112) + self.crop = None + if track=='cfat': + crop_file = osp.join(self.model_path, 'crop.txt') + if osp.exists(crop_file): + lines = open(crop_file,'r').readlines() + if len(lines)!=6: + return "crop.txt should contain 6 lines" + lines = [int(x) for x in lines] + self.crop = lines[:4] + input_size = tuple(lines[4:6]) + if input_size!=self.image_size: + return "input-size is inconsistant with onnx model input, %s vs %s"%(input_size, self.image_size) + + self.model_size_mb = os.path.getsize(self.model_file) / float(1024*1024) + if self.model_size_mb > max_model_size_mb: + return "max model size exceed, given %.3f-MB"%self.model_size_mb + + input_mean = None + input_std = None + if track=='cfat': + pn_file = osp.join(self.model_path, 'pixel_norm.txt') + if osp.exists(pn_file): + lines = open(pn_file,'r').readlines() + if len(lines)!=2: + return "pixel_norm.txt should contain 2 lines" + input_mean = float(lines[0]) + input_std = float(lines[1]) + if input_mean is not None or input_std is not None: + if input_mean is None or input_std is None: + return "please set input_mean and input_std simultaneously" + else: + find_sub = False + find_mul = False + for nid, node in enumerate(graph.node[:8]): + print(nid, node.name) + if node.name.startswith('Sub') or node.name.startswith('_minus'): + find_sub = True + if node.name.startswith('Mul') or node.name.startswith('_mul') or node.name.startswith('Div'): + find_mul = True + if find_sub and find_mul: + print("find sub and mul") + #mxnet arcface model + input_mean = 0.0 + input_std = 1.0 + else: + input_mean = 127.5 + input_std = 127.5 + self.input_mean = input_mean + self.input_std = input_std + for initn in graph.initializer: + weight_array = numpy_helper.to_array(initn) + dt = weight_array.dtype + if dt.itemsize<4: + return 'invalid weight type - (%s:%s)' % (initn.name, dt.name) + if test_img is None: + test_img = get_image('Tom_Hanks_54745') + test_img = cv2.resize(test_img, self.image_size) + else: + test_img = cv2.resize(test_img, self.image_size) + feat, cost = self.benchmark(test_img) + batch_result = self.check_batch(test_img) + batch_result_sum = float(np.sum(batch_result)) + if batch_result_sum in [float('inf'), -float('inf')] or batch_result_sum != batch_result_sum: + print(batch_result) + print(batch_result_sum) + return "batch result output contains NaN!" + + if len(feat.shape) < 2: + return "the shape of the feature must be two, but get {}".format(str(feat.shape)) + + if feat.shape[1] > max_feat_dim: + return "max feat dim exceed, given %d"%feat.shape[1] + self.feat_dim = feat.shape[1] + cost_ms = cost*1000 + if cost_ms>max_time_cost: + return "max time cost exceed, given %.4f"%cost_ms + self.cost_ms = cost_ms + print('check stat:, model-size-mb: %.4f, feat-dim: %d, time-cost-ms: %.4f, input-mean: %.3f, input-std: %.3f'%(self.model_size_mb, self.feat_dim, self.cost_ms, self.input_mean, self.input_std)) + return None + + def check_batch(self, img): + if not isinstance(img, list): + imgs = [img, ] * 32 + if self.crop is not None: + nimgs = [] + for img in imgs: + nimg = img[self.crop[1]:self.crop[3], self.crop[0]:self.crop[2], :] + if nimg.shape[0] != self.image_size[1] or nimg.shape[1] != self.image_size[0]: + nimg = cv2.resize(nimg, self.image_size) + nimgs.append(nimg) + imgs = nimgs + blob = cv2.dnn.blobFromImages( + images=imgs, scalefactor=1.0 / self.input_std, size=self.image_size, + mean=(self.input_mean, self.input_mean, self.input_mean), swapRB=True) + net_out = self.session.run(self.output_names, {self.input_name: blob})[0] + return net_out + + + def meta_info(self): + return {'model-size-mb':self.model_size_mb, 'feature-dim':self.feat_dim, 'infer': self.cost_ms} + + + def forward(self, imgs): + if not isinstance(imgs, list): + imgs = [imgs] + input_size = self.image_size + if self.crop is not None: + nimgs = [] + for img in imgs: + nimg = img[self.crop[1]:self.crop[3],self.crop[0]:self.crop[2],:] + if nimg.shape[0]!=input_size[1] or nimg.shape[1]!=input_size[0]: + nimg = cv2.resize(nimg, input_size) + nimgs.append(nimg) + imgs = nimgs + blob = cv2.dnn.blobFromImages(imgs, 1.0/self.input_std, input_size, (self.input_mean, self.input_mean, self.input_mean), swapRB=True) + net_out = self.session.run(self.output_names, {self.input_name : blob})[0] + return net_out + + def benchmark(self, img): + input_size = self.image_size + if self.crop is not None: + nimg = img[self.crop[1]:self.crop[3],self.crop[0]:self.crop[2],:] + if nimg.shape[0]!=input_size[1] or nimg.shape[1]!=input_size[0]: + nimg = cv2.resize(nimg, input_size) + img = nimg + blob = cv2.dnn.blobFromImage(img, 1.0/self.input_std, input_size, (self.input_mean, self.input_mean, self.input_mean), swapRB=True) + costs = [] + for _ in range(50): + ta = datetime.datetime.now() + net_out = self.session.run(self.output_names, {self.input_name : blob})[0] + tb = datetime.datetime.now() + cost = (tb-ta).total_seconds() + costs.append(cost) + costs = sorted(costs) + cost = costs[5] + return net_out, cost + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='') + # general + parser.add_argument('workdir', help='submitted work dir', type=str) + parser.add_argument('--track', help='track name, for different challenge', type=str, default='cfat') + args = parser.parse_args() + handler = ArcFaceORT(args.workdir) + err = handler.check(args.track) + print('err:', err) diff --git a/src/face3d/models/arcface_torch/onnx_ijbc.py b/src/face3d/models/arcface_torch/onnx_ijbc.py new file mode 100644 index 0000000000000000000000000000000000000000..05b50bfad4b4cf38903b89f596263a8e29a50d3e --- /dev/null +++ b/src/face3d/models/arcface_torch/onnx_ijbc.py @@ -0,0 +1,267 @@ +import argparse +import os +import pickle +import timeit + +import cv2 +import mxnet as mx +import numpy as np +import pandas as pd +import prettytable +import skimage.transform +from sklearn.metrics import roc_curve +from sklearn.preprocessing import normalize + +from onnx_helper import ArcFaceORT + +SRC = np.array( + [ + [30.2946, 51.6963], + [65.5318, 51.5014], + [48.0252, 71.7366], + [33.5493, 92.3655], + [62.7299, 92.2041]] + , dtype=np.float32) +SRC[:, 0] += 8.0 + + +class AlignedDataSet(mx.gluon.data.Dataset): + def __init__(self, root, lines, align=True): + self.lines = lines + self.root = root + self.align = align + + def __len__(self): + return len(self.lines) + + def __getitem__(self, idx): + each_line = self.lines[idx] + name_lmk_score = each_line.strip().split(' ') + name = os.path.join(self.root, name_lmk_score[0]) + img = cv2.cvtColor(cv2.imread(name), cv2.COLOR_BGR2RGB) + landmark5 = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32).reshape((5, 2)) + st = skimage.transform.SimilarityTransform() + st.estimate(landmark5, SRC) + img = cv2.warpAffine(img, st.params[0:2, :], (112, 112), borderValue=0.0) + img_1 = np.expand_dims(img, 0) + img_2 = np.expand_dims(np.fliplr(img), 0) + output = np.concatenate((img_1, img_2), axis=0).astype(np.float32) + output = np.transpose(output, (0, 3, 1, 2)) + output = mx.nd.array(output) + return output + + +def extract(model_root, dataset): + model = ArcFaceORT(model_path=model_root) + model.check() + feat_mat = np.zeros(shape=(len(dataset), 2 * model.feat_dim)) + + def batchify_fn(data): + return mx.nd.concat(*data, dim=0) + + data_loader = mx.gluon.data.DataLoader( + dataset, 128, last_batch='keep', num_workers=4, + thread_pool=True, prefetch=16, batchify_fn=batchify_fn) + num_iter = 0 + for batch in data_loader: + batch = batch.asnumpy() + batch = (batch - model.input_mean) / model.input_std + feat = model.session.run(model.output_names, {model.input_name: batch})[0] + feat = np.reshape(feat, (-1, model.feat_dim * 2)) + feat_mat[128 * num_iter: 128 * num_iter + feat.shape[0], :] = feat + num_iter += 1 + if num_iter % 50 == 0: + print(num_iter) + return feat_mat + + +def read_template_media_list(path): + ijb_meta = pd.read_csv(path, sep=' ', header=None).values + templates = ijb_meta[:, 1].astype(np.int) + medias = ijb_meta[:, 2].astype(np.int) + return templates, medias + + +def read_template_pair_list(path): + pairs = pd.read_csv(path, sep=' ', header=None).values + t1 = pairs[:, 0].astype(np.int) + t2 = pairs[:, 1].astype(np.int) + label = pairs[:, 2].astype(np.int) + return t1, t2, label + + +def read_image_feature(path): + with open(path, 'rb') as fid: + img_feats = pickle.load(fid) + return img_feats + + +def image2template_feature(img_feats=None, + templates=None, + medias=None): + unique_templates = np.unique(templates) + template_feats = np.zeros((len(unique_templates), img_feats.shape[1])) + for count_template, uqt in enumerate(unique_templates): + (ind_t,) = np.where(templates == uqt) + face_norm_feats = img_feats[ind_t] + face_medias = medias[ind_t] + unique_medias, unique_media_counts = np.unique(face_medias, return_counts=True) + media_norm_feats = [] + for u, ct in zip(unique_medias, unique_media_counts): + (ind_m,) = np.where(face_medias == u) + if ct == 1: + media_norm_feats += [face_norm_feats[ind_m]] + else: # image features from the same video will be aggregated into one feature + media_norm_feats += [np.mean(face_norm_feats[ind_m], axis=0, keepdims=True), ] + media_norm_feats = np.array(media_norm_feats) + template_feats[count_template] = np.sum(media_norm_feats, axis=0) + if count_template % 2000 == 0: + print('Finish Calculating {} template features.'.format( + count_template)) + template_norm_feats = normalize(template_feats) + return template_norm_feats, unique_templates + + +def verification(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): + template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) + for count_template, uqt in enumerate(unique_templates): + template2id[uqt] = count_template + score = np.zeros((len(p1),)) + total_pairs = np.array(range(len(p1))) + batchsize = 100000 + sublists = [total_pairs[i: i + batchsize] for i in range(0, len(p1), batchsize)] + total_sublists = len(sublists) + for c, s in enumerate(sublists): + feat1 = template_norm_feats[template2id[p1[s]]] + feat2 = template_norm_feats[template2id[p2[s]]] + similarity_score = np.sum(feat1 * feat2, -1) + score[s] = similarity_score.flatten() + if c % 10 == 0: + print('Finish {}/{} pairs.'.format(c, total_sublists)) + return score + + +def verification2(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): + template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) + for count_template, uqt in enumerate(unique_templates): + template2id[uqt] = count_template + score = np.zeros((len(p1),)) # save cosine distance between pairs + total_pairs = np.array(range(len(p1))) + batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation + sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + total_sublists = len(sublists) + for c, s in enumerate(sublists): + feat1 = template_norm_feats[template2id[p1[s]]] + feat2 = template_norm_feats[template2id[p2[s]]] + similarity_score = np.sum(feat1 * feat2, -1) + score[s] = similarity_score.flatten() + if c % 10 == 0: + print('Finish {}/{} pairs.'.format(c, total_sublists)) + return score + + +def main(args): + use_norm_score = True # if Ture, TestMode(N1) + use_detector_score = True # if Ture, TestMode(D1) + use_flip_test = True # if Ture, TestMode(F1) + assert args.target == 'IJBC' or args.target == 'IJBB' + + start = timeit.default_timer() + templates, medias = read_template_media_list( + os.path.join('%s/meta' % args.image_path, '%s_face_tid_mid.txt' % args.target.lower())) + stop = timeit.default_timer() + print('Time: %.2f s. ' % (stop - start)) + + start = timeit.default_timer() + p1, p2, label = read_template_pair_list( + os.path.join('%s/meta' % args.image_path, + '%s_template_pair_label.txt' % args.target.lower())) + stop = timeit.default_timer() + print('Time: %.2f s. ' % (stop - start)) + + start = timeit.default_timer() + img_path = '%s/loose_crop' % args.image_path + img_list_path = '%s/meta/%s_name_5pts_score.txt' % (args.image_path, args.target.lower()) + img_list = open(img_list_path) + files = img_list.readlines() + dataset = AlignedDataSet(root=img_path, lines=files, align=True) + img_feats = extract(args.model_root, dataset) + + faceness_scores = [] + for each_line in files: + name_lmk_score = each_line.split() + faceness_scores.append(name_lmk_score[-1]) + faceness_scores = np.array(faceness_scores).astype(np.float32) + stop = timeit.default_timer() + print('Time: %.2f s. ' % (stop - start)) + print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], img_feats.shape[1])) + start = timeit.default_timer() + + if use_flip_test: + img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] + img_feats[:, img_feats.shape[1] // 2:] + else: + img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] + + if use_norm_score: + img_input_feats = img_input_feats + else: + img_input_feats = img_input_feats / np.sqrt(np.sum(img_input_feats ** 2, -1, keepdims=True)) + + if use_detector_score: + print(img_input_feats.shape, faceness_scores.shape) + img_input_feats = img_input_feats * faceness_scores[:, np.newaxis] + else: + img_input_feats = img_input_feats + + template_norm_feats, unique_templates = image2template_feature( + img_input_feats, templates, medias) + stop = timeit.default_timer() + print('Time: %.2f s. ' % (stop - start)) + + start = timeit.default_timer() + score = verification(template_norm_feats, unique_templates, p1, p2) + stop = timeit.default_timer() + print('Time: %.2f s. ' % (stop - start)) + save_path = os.path.join(args.result_dir, "{}_result".format(args.target)) + if not os.path.exists(save_path): + os.makedirs(save_path) + score_save_file = os.path.join(save_path, "{}.npy".format(args.model_root)) + np.save(score_save_file, score) + files = [score_save_file] + methods = [] + scores = [] + for file in files: + methods.append(os.path.basename(file)) + scores.append(np.load(file)) + methods = np.array(methods) + scores = dict(zip(methods, scores)) + x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] + tpr_fpr_table = prettytable.PrettyTable(['Methods'] + [str(x) for x in x_labels]) + for method in methods: + fpr, tpr, _ = roc_curve(label, scores[method]) + fpr = np.flipud(fpr) + tpr = np.flipud(tpr) + tpr_fpr_row = [] + tpr_fpr_row.append("%s-%s" % (method, args.target)) + for fpr_iter in np.arange(len(x_labels)): + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) + tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) + tpr_fpr_table.add_row(tpr_fpr_row) + print(tpr_fpr_table) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='do ijb test') + # general + parser.add_argument('--model-root', default='', help='path to load model.') + parser.add_argument('--image-path', default='', type=str, help='') + parser.add_argument('--result-dir', default='.', type=str, help='') + parser.add_argument('--target', default='IJBC', type=str, help='target, set to IJBC or IJBB') + main(parser.parse_args()) diff --git a/src/face3d/models/arcface_torch/partial_fc.py b/src/face3d/models/arcface_torch/partial_fc.py new file mode 100644 index 0000000000000000000000000000000000000000..17e2d25715d10ba446c957e1d2528b0687ed71d5 --- /dev/null +++ b/src/face3d/models/arcface_torch/partial_fc.py @@ -0,0 +1,222 @@ +import logging +import os + +import torch +import torch.distributed as dist +from torch.nn import Module +from torch.nn.functional import normalize, linear +from torch.nn.parameter import Parameter + + +class PartialFC(Module): + """ + Author: {Xiang An, Yang Xiao, XuHan Zhu} in DeepGlint, + Partial FC: Training 10 Million Identities on a Single Machine + See the original paper: + https://arxiv.org/abs/2010.05222 + """ + + @torch.no_grad() + def __init__(self, rank, local_rank, world_size, batch_size, resume, + margin_softmax, num_classes, sample_rate=1.0, embedding_size=512, prefix="./"): + """ + rank: int + Unique process(GPU) ID from 0 to world_size - 1. + local_rank: int + Unique process(GPU) ID within the server from 0 to 7. + world_size: int + Number of GPU. + batch_size: int + Batch size on current rank(GPU). + resume: bool + Select whether to restore the weight of softmax. + margin_softmax: callable + A function of margin softmax, eg: cosface, arcface. + num_classes: int + The number of class center storage in current rank(CPU/GPU), usually is total_classes // world_size, + required. + sample_rate: float + The partial fc sampling rate, when the number of classes increases to more than 2 millions, Sampling + can greatly speed up training, and reduce a lot of GPU memory, default is 1.0. + embedding_size: int + The feature dimension, default is 512. + prefix: str + Path for save checkpoint, default is './'. + """ + super(PartialFC, self).__init__() + # + self.num_classes: int = num_classes + self.rank: int = rank + self.local_rank: int = local_rank + self.device: torch.device = torch.device("cuda:{}".format(self.local_rank)) + self.world_size: int = world_size + self.batch_size: int = batch_size + self.margin_softmax: callable = margin_softmax + self.sample_rate: float = sample_rate + self.embedding_size: int = embedding_size + self.prefix: str = prefix + self.num_local: int = num_classes // world_size + int(rank < num_classes % world_size) + self.class_start: int = num_classes // world_size * rank + min(rank, num_classes % world_size) + self.num_sample: int = int(self.sample_rate * self.num_local) + + self.weight_name = os.path.join(self.prefix, "rank_{}_softmax_weight.pt".format(self.rank)) + self.weight_mom_name = os.path.join(self.prefix, "rank_{}_softmax_weight_mom.pt".format(self.rank)) + + if resume: + try: + self.weight: torch.Tensor = torch.load(self.weight_name) + self.weight_mom: torch.Tensor = torch.load(self.weight_mom_name) + if self.weight.shape[0] != self.num_local or self.weight_mom.shape[0] != self.num_local: + raise IndexError + logging.info("softmax weight resume successfully!") + logging.info("softmax weight mom resume successfully!") + except (FileNotFoundError, KeyError, IndexError): + self.weight = torch.normal(0, 0.01, (self.num_local, self.embedding_size), device=self.device) + self.weight_mom: torch.Tensor = torch.zeros_like(self.weight) + logging.info("softmax weight init!") + logging.info("softmax weight mom init!") + else: + self.weight = torch.normal(0, 0.01, (self.num_local, self.embedding_size), device=self.device) + self.weight_mom: torch.Tensor = torch.zeros_like(self.weight) + logging.info("softmax weight init successfully!") + logging.info("softmax weight mom init successfully!") + self.stream: torch.cuda.Stream = torch.cuda.Stream(local_rank) + + self.index = None + if int(self.sample_rate) == 1: + self.update = lambda: 0 + self.sub_weight = Parameter(self.weight) + self.sub_weight_mom = self.weight_mom + else: + self.sub_weight = Parameter(torch.empty((0, 0)).cuda(local_rank)) + + def save_params(self): + """ Save softmax weight for each rank on prefix + """ + torch.save(self.weight.data, self.weight_name) + torch.save(self.weight_mom, self.weight_mom_name) + + @torch.no_grad() + def sample(self, total_label): + """ + Sample all positive class centers in each rank, and random select neg class centers to filling a fixed + `num_sample`. + + total_label: tensor + Label after all gather, which cross all GPUs. + """ + index_positive = (self.class_start <= total_label) & (total_label < self.class_start + self.num_local) + total_label[~index_positive] = -1 + total_label[index_positive] -= self.class_start + if int(self.sample_rate) != 1: + positive = torch.unique(total_label[index_positive], sorted=True) + if self.num_sample - positive.size(0) >= 0: + perm = torch.rand(size=[self.num_local], device=self.device) + perm[positive] = 2.0 + index = torch.topk(perm, k=self.num_sample)[1] + index = index.sort()[0] + else: + index = positive + self.index = index + total_label[index_positive] = torch.searchsorted(index, total_label[index_positive]) + self.sub_weight = Parameter(self.weight[index]) + self.sub_weight_mom = self.weight_mom[index] + + def forward(self, total_features, norm_weight): + """ Partial fc forward, `logits = X * sample(W)` + """ + torch.cuda.current_stream().wait_stream(self.stream) + logits = linear(total_features, norm_weight) + return logits + + @torch.no_grad() + def update(self): + """ Set updated weight and weight_mom to memory bank. + """ + self.weight_mom[self.index] = self.sub_weight_mom + self.weight[self.index] = self.sub_weight + + def prepare(self, label, optimizer): + """ + get sampled class centers for cal softmax. + + label: tensor + Label tensor on each rank. + optimizer: opt + Optimizer for partial fc, which need to get weight mom. + """ + with torch.cuda.stream(self.stream): + total_label = torch.zeros( + size=[self.batch_size * self.world_size], device=self.device, dtype=torch.long) + dist.all_gather(list(total_label.chunk(self.world_size, dim=0)), label) + self.sample(total_label) + optimizer.state.pop(optimizer.param_groups[-1]['params'][0], None) + optimizer.param_groups[-1]['params'][0] = self.sub_weight + optimizer.state[self.sub_weight]['momentum_buffer'] = self.sub_weight_mom + norm_weight = normalize(self.sub_weight) + return total_label, norm_weight + + def forward_backward(self, label, features, optimizer): + """ + Partial fc forward and backward with model parallel + + label: tensor + Label tensor on each rank(GPU) + features: tensor + Features tensor on each rank(GPU) + optimizer: optimizer + Optimizer for partial fc + + Returns: + -------- + x_grad: tensor + The gradient of features. + loss_v: tensor + Loss value for cross entropy. + """ + total_label, norm_weight = self.prepare(label, optimizer) + total_features = torch.zeros( + size=[self.batch_size * self.world_size, self.embedding_size], device=self.device) + dist.all_gather(list(total_features.chunk(self.world_size, dim=0)), features.data) + total_features.requires_grad = True + + logits = self.forward(total_features, norm_weight) + logits = self.margin_softmax(logits, total_label) + + with torch.no_grad(): + max_fc = torch.max(logits, dim=1, keepdim=True)[0] + dist.all_reduce(max_fc, dist.ReduceOp.MAX) + + # calculate exp(logits) and all-reduce + logits_exp = torch.exp(logits - max_fc) + logits_sum_exp = logits_exp.sum(dim=1, keepdims=True) + dist.all_reduce(logits_sum_exp, dist.ReduceOp.SUM) + + # calculate prob + logits_exp.div_(logits_sum_exp) + + # get one-hot + grad = logits_exp + index = torch.where(total_label != -1)[0] + one_hot = torch.zeros(size=[index.size()[0], grad.size()[1]], device=grad.device) + one_hot.scatter_(1, total_label[index, None], 1) + + # calculate loss + loss = torch.zeros(grad.size()[0], 1, device=grad.device) + loss[index] = grad[index].gather(1, total_label[index, None]) + dist.all_reduce(loss, dist.ReduceOp.SUM) + loss_v = loss.clamp_min_(1e-30).log_().mean() * (-1) + + # calculate grad + grad[index] -= one_hot + grad.div_(self.batch_size * self.world_size) + + logits.backward(grad) + if total_features.grad is not None: + total_features.grad.detach_() + x_grad: torch.Tensor = torch.zeros_like(features, requires_grad=True) + # feature gradient all-reduce + dist.reduce_scatter(x_grad, list(total_features.grad.chunk(self.world_size, dim=0))) + x_grad = x_grad * self.world_size + # backward backbone + return x_grad, loss_v diff --git a/src/face3d/models/arcface_torch/requirement.txt b/src/face3d/models/arcface_torch/requirement.txt new file mode 100644 index 0000000000000000000000000000000000000000..f72c1b3ba814ae1e0bc1c1f56402026978b9e870 --- /dev/null +++ b/src/face3d/models/arcface_torch/requirement.txt @@ -0,0 +1,5 @@ +tensorboard +easydict +mxnet +onnx +sklearn diff --git a/src/face3d/models/arcface_torch/run.sh b/src/face3d/models/arcface_torch/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..61af4b4950eb11334e55362e3e3c5e2796979a01 --- /dev/null +++ b/src/face3d/models/arcface_torch/run.sh @@ -0,0 +1,2 @@ +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python -m torch.distributed.launch --nproc_per_node=8 --nnodes=1 --node_rank=0 --master_addr="127.0.0.1" --master_port=1234 train.py configs/ms1mv3_r50 +ps -ef | grep "train" | grep -v grep | awk '{print "kill -9 "$2}' | sh diff --git a/src/face3d/models/arcface_torch/torch2onnx.py b/src/face3d/models/arcface_torch/torch2onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..fc26ab82e552331bc8d75b34e81000418f4d38ec --- /dev/null +++ b/src/face3d/models/arcface_torch/torch2onnx.py @@ -0,0 +1,59 @@ +import numpy as np +import onnx +import torch + + +def convert_onnx(net, path_module, output, opset=11, simplify=False): + assert isinstance(net, torch.nn.Module) + img = np.random.randint(0, 255, size=(112, 112, 3), dtype=np.int32) + img = img.astype(np.float) + img = (img / 255. - 0.5) / 0.5 # torch style norm + img = img.transpose((2, 0, 1)) + img = torch.from_numpy(img).unsqueeze(0).float() + + weight = torch.load(path_module) + net.load_state_dict(weight) + net.eval() + torch.onnx.export(net, img, output, keep_initializers_as_inputs=False, verbose=False, opset_version=opset) + model = onnx.load(output) + graph = model.graph + graph.input[0].type.tensor_type.shape.dim[0].dim_param = 'None' + if simplify: + from onnxsim import simplify + model, check = simplify(model) + assert check, "Simplified ONNX model could not be validated" + onnx.save(model, output) + + +if __name__ == '__main__': + import os + import argparse + from backbones import get_model + + parser = argparse.ArgumentParser(description='ArcFace PyTorch to onnx') + parser.add_argument('input', type=str, help='input backbone.pth file or path') + parser.add_argument('--output', type=str, default=None, help='output onnx path') + parser.add_argument('--network', type=str, default=None, help='backbone network') + parser.add_argument('--simplify', type=bool, default=False, help='onnx simplify') + args = parser.parse_args() + input_file = args.input + if os.path.isdir(input_file): + input_file = os.path.join(input_file, "backbone.pth") + assert os.path.exists(input_file) + model_name = os.path.basename(os.path.dirname(input_file)).lower() + params = model_name.split("_") + if len(params) >= 3 and params[1] in ('arcface', 'cosface'): + if args.network is None: + args.network = params[2] + assert args.network is not None + print(args) + backbone_onnx = get_model(args.network, dropout=0) + + output_path = args.output + if output_path is None: + output_path = os.path.join(os.path.dirname(__file__), 'onnx') + if not os.path.exists(output_path): + os.makedirs(output_path) + assert os.path.isdir(output_path) + output_file = os.path.join(output_path, "%s.onnx" % model_name) + convert_onnx(backbone_onnx, input_file, output_file, simplify=args.simplify) diff --git a/src/face3d/models/arcface_torch/train.py b/src/face3d/models/arcface_torch/train.py new file mode 100644 index 0000000000000000000000000000000000000000..55eca2d0ad9463415970e09bccab8b722e496704 --- /dev/null +++ b/src/face3d/models/arcface_torch/train.py @@ -0,0 +1,141 @@ +import argparse +import logging +import os + +import torch +import torch.distributed as dist +import torch.nn.functional as F +import torch.utils.data.distributed +from torch.nn.utils import clip_grad_norm_ + +import losses +from backbones import get_model +from dataset import MXFaceDataset, SyntheticDataset, DataLoaderX +from partial_fc import PartialFC +from utils.utils_amp import MaxClipGradScaler +from utils.utils_callbacks import CallBackVerification, CallBackLogging, CallBackModelCheckpoint +from utils.utils_config import get_config +from utils.utils_logging import AverageMeter, init_logging + + +def main(args): + cfg = get_config(args.config) + try: + world_size = int(os.environ['WORLD_SIZE']) + rank = int(os.environ['RANK']) + dist.init_process_group('nccl') + except KeyError: + world_size = 1 + rank = 0 + dist.init_process_group(backend='nccl', init_method="tcp://127.0.0.1:12584", rank=rank, world_size=world_size) + + local_rank = args.local_rank + torch.cuda.set_device(local_rank) + os.makedirs(cfg.output, exist_ok=True) + init_logging(rank, cfg.output) + + if cfg.rec == "synthetic": + train_set = SyntheticDataset(local_rank=local_rank) + else: + train_set = MXFaceDataset(root_dir=cfg.rec, local_rank=local_rank) + + train_sampler = torch.utils.data.distributed.DistributedSampler(train_set, shuffle=True) + train_loader = DataLoaderX( + local_rank=local_rank, dataset=train_set, batch_size=cfg.batch_size, + sampler=train_sampler, num_workers=2, pin_memory=True, drop_last=True) + backbone = get_model(cfg.network, dropout=0.0, fp16=cfg.fp16, num_features=cfg.embedding_size).to(local_rank) + + if cfg.resume: + try: + backbone_pth = os.path.join(cfg.output, "backbone.pth") + backbone.load_state_dict(torch.load(backbone_pth, map_location=torch.device(local_rank))) + if rank == 0: + logging.info("backbone resume successfully!") + except (FileNotFoundError, KeyError, IndexError, RuntimeError): + if rank == 0: + logging.info("resume fail, backbone init successfully!") + + backbone = torch.nn.parallel.DistributedDataParallel( + module=backbone, broadcast_buffers=False, device_ids=[local_rank]) + backbone.train() + margin_softmax = losses.get_loss(cfg.loss) + module_partial_fc = PartialFC( + rank=rank, local_rank=local_rank, world_size=world_size, resume=cfg.resume, + batch_size=cfg.batch_size, margin_softmax=margin_softmax, num_classes=cfg.num_classes, + sample_rate=cfg.sample_rate, embedding_size=cfg.embedding_size, prefix=cfg.output) + + opt_backbone = torch.optim.SGD( + params=[{'params': backbone.parameters()}], + lr=cfg.lr / 512 * cfg.batch_size * world_size, + momentum=0.9, weight_decay=cfg.weight_decay) + opt_pfc = torch.optim.SGD( + params=[{'params': module_partial_fc.parameters()}], + lr=cfg.lr / 512 * cfg.batch_size * world_size, + momentum=0.9, weight_decay=cfg.weight_decay) + + num_image = len(train_set) + total_batch_size = cfg.batch_size * world_size + cfg.warmup_step = num_image // total_batch_size * cfg.warmup_epoch + cfg.total_step = num_image // total_batch_size * cfg.num_epoch + + def lr_step_func(current_step): + cfg.decay_step = [x * num_image // total_batch_size for x in cfg.decay_epoch] + if current_step < cfg.warmup_step: + return current_step / cfg.warmup_step + else: + return 0.1 ** len([m for m in cfg.decay_step if m <= current_step]) + + scheduler_backbone = torch.optim.lr_scheduler.LambdaLR( + optimizer=opt_backbone, lr_lambda=lr_step_func) + scheduler_pfc = torch.optim.lr_scheduler.LambdaLR( + optimizer=opt_pfc, lr_lambda=lr_step_func) + + for key, value in cfg.items(): + num_space = 25 - len(key) + logging.info(": " + key + " " * num_space + str(value)) + + val_target = cfg.val_targets + callback_verification = CallBackVerification(2000, rank, val_target, cfg.rec) + callback_logging = CallBackLogging(50, rank, cfg.total_step, cfg.batch_size, world_size, None) + callback_checkpoint = CallBackModelCheckpoint(rank, cfg.output) + + loss = AverageMeter() + start_epoch = 0 + global_step = 0 + grad_amp = MaxClipGradScaler(cfg.batch_size, 128 * cfg.batch_size, growth_interval=100) if cfg.fp16 else None + for epoch in range(start_epoch, cfg.num_epoch): + train_sampler.set_epoch(epoch) + for step, (img, label) in enumerate(train_loader): + global_step += 1 + features = F.normalize(backbone(img)) + x_grad, loss_v = module_partial_fc.forward_backward(label, features, opt_pfc) + if cfg.fp16: + features.backward(grad_amp.scale(x_grad)) + grad_amp.unscale_(opt_backbone) + clip_grad_norm_(backbone.parameters(), max_norm=5, norm_type=2) + grad_amp.step(opt_backbone) + grad_amp.update() + else: + features.backward(x_grad) + clip_grad_norm_(backbone.parameters(), max_norm=5, norm_type=2) + opt_backbone.step() + + opt_pfc.step() + module_partial_fc.update() + opt_backbone.zero_grad() + opt_pfc.zero_grad() + loss.update(loss_v, 1) + callback_logging(global_step, loss, epoch, cfg.fp16, scheduler_backbone.get_last_lr()[0], grad_amp) + callback_verification(global_step, backbone) + scheduler_backbone.step() + scheduler_pfc.step() + callback_checkpoint(global_step, backbone, module_partial_fc) + dist.destroy_process_group() + + +if __name__ == "__main__": + torch.backends.cudnn.benchmark = True + parser = argparse.ArgumentParser(description='PyTorch ArcFace Training') + parser.add_argument('config', type=str, help='py config file') + parser.add_argument('--local_rank', type=int, default=0, help='local_rank') + main(parser.parse_args()) diff --git a/src/face3d/models/arcface_torch/utils/__init__.py b/src/face3d/models/arcface_torch/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/face3d/models/arcface_torch/utils/plot.py b/src/face3d/models/arcface_torch/utils/plot.py new file mode 100644 index 0000000000000000000000000000000000000000..ccc588e5c01ca550b69c385aeb3fd139c59fb88a --- /dev/null +++ b/src/face3d/models/arcface_torch/utils/plot.py @@ -0,0 +1,72 @@ +# coding: utf-8 + +import os +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from menpo.visualize.viewmatplotlib import sample_colours_from_colourmap +from prettytable import PrettyTable +from sklearn.metrics import roc_curve, auc + +image_path = "/data/anxiang/IJB_release/IJBC" +files = [ + "./ms1mv3_arcface_r100/ms1mv3_arcface_r100/ijbc.npy" +] + + +def read_template_pair_list(path): + pairs = pd.read_csv(path, sep=' ', header=None).values + t1 = pairs[:, 0].astype(np.int) + t2 = pairs[:, 1].astype(np.int) + label = pairs[:, 2].astype(np.int) + return t1, t2, label + + +p1, p2, label = read_template_pair_list( + os.path.join('%s/meta' % image_path, + '%s_template_pair_label.txt' % 'ijbc')) + +methods = [] +scores = [] +for file in files: + methods.append(file.split('/')[-2]) + scores.append(np.load(file)) + +methods = np.array(methods) +scores = dict(zip(methods, scores)) +colours = dict( + zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) +x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] +tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) +fig = plt.figure() +for method in methods: + fpr, tpr, _ = roc_curve(label, scores[method]) + roc_auc = auc(fpr, tpr) + fpr = np.flipud(fpr) + tpr = np.flipud(tpr) # select largest tpr at same fpr + plt.plot(fpr, + tpr, + color=colours[method], + lw=1, + label=('[%s (AUC = %0.4f %%)]' % + (method.split('-')[-1], roc_auc * 100))) + tpr_fpr_row = [] + tpr_fpr_row.append("%s-%s" % (method, "IJBC")) + for fpr_iter in np.arange(len(x_labels)): + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) + tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) + tpr_fpr_table.add_row(tpr_fpr_row) +plt.xlim([10 ** -6, 0.1]) +plt.ylim([0.3, 1.0]) +plt.grid(linestyle='--', linewidth=1) +plt.xticks(x_labels) +plt.yticks(np.linspace(0.3, 1.0, 8, endpoint=True)) +plt.xscale('log') +plt.xlabel('False Positive Rate') +plt.ylabel('True Positive Rate') +plt.title('ROC on IJB') +plt.legend(loc="lower right") +print(tpr_fpr_table) diff --git a/src/face3d/models/arcface_torch/utils/utils_amp.py b/src/face3d/models/arcface_torch/utils/utils_amp.py new file mode 100644 index 0000000000000000000000000000000000000000..9ac2a03f4212faa129faed447a8f4519c0a00a8b --- /dev/null +++ b/src/face3d/models/arcface_torch/utils/utils_amp.py @@ -0,0 +1,88 @@ +from typing import Dict, List + +import torch + +if torch.__version__ < '1.9': + Iterable = torch._six.container_abcs.Iterable +else: + import collections + + Iterable = collections.abc.Iterable +from torch.cuda.amp import GradScaler + + +class _MultiDeviceReplicator(object): + """ + Lazily serves copies of a tensor to requested devices. Copies are cached per-device. + """ + + def __init__(self, master_tensor: torch.Tensor) -> None: + assert master_tensor.is_cuda + self.master = master_tensor + self._per_device_tensors: Dict[torch.device, torch.Tensor] = {} + + def get(self, device) -> torch.Tensor: + retval = self._per_device_tensors.get(device, None) + if retval is None: + retval = self.master.to(device=device, non_blocking=True, copy=True) + self._per_device_tensors[device] = retval + return retval + + +class MaxClipGradScaler(GradScaler): + def __init__(self, init_scale, max_scale: float, growth_interval=100): + GradScaler.__init__(self, init_scale=init_scale, growth_interval=growth_interval) + self.max_scale = max_scale + + def scale_clip(self): + if self.get_scale() == self.max_scale: + self.set_growth_factor(1) + elif self.get_scale() < self.max_scale: + self.set_growth_factor(2) + elif self.get_scale() > self.max_scale: + self._scale.fill_(self.max_scale) + self.set_growth_factor(1) + + def scale(self, outputs): + """ + Multiplies ('scales') a tensor or list of tensors by the scale factor. + + Returns scaled outputs. If this instance of :class:`GradScaler` is not enabled, outputs are returned + unmodified. + + Arguments: + outputs (Tensor or iterable of Tensors): Outputs to scale. + """ + if not self._enabled: + return outputs + self.scale_clip() + # Short-circuit for the common case. + if isinstance(outputs, torch.Tensor): + assert outputs.is_cuda + if self._scale is None: + self._lazy_init_scale_growth_tracker(outputs.device) + assert self._scale is not None + return outputs * self._scale.to(device=outputs.device, non_blocking=True) + + # Invoke the more complex machinery only if we're treating multiple outputs. + stash: List[_MultiDeviceReplicator] = [] # holds a reference that can be overwritten by apply_scale + + def apply_scale(val): + if isinstance(val, torch.Tensor): + assert val.is_cuda + if len(stash) == 0: + if self._scale is None: + self._lazy_init_scale_growth_tracker(val.device) + assert self._scale is not None + stash.append(_MultiDeviceReplicator(self._scale)) + return val * stash[0].get(val.device) + elif isinstance(val, Iterable): + iterable = map(apply_scale, val) + if isinstance(val, list) or isinstance(val, tuple): + return type(val)(iterable) + else: + return iterable + else: + raise ValueError("outputs must be a Tensor or an iterable of Tensors") + + return apply_scale(outputs) diff --git a/src/face3d/models/arcface_torch/utils/utils_callbacks.py b/src/face3d/models/arcface_torch/utils/utils_callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..bd2f56cba47c57de102710ff56eaac591e59f4da --- /dev/null +++ b/src/face3d/models/arcface_torch/utils/utils_callbacks.py @@ -0,0 +1,117 @@ +import logging +import os +import time +from typing import List + +import torch + +from eval import verification +from utils.utils_logging import AverageMeter + + +class CallBackVerification(object): + def __init__(self, frequent, rank, val_targets, rec_prefix, image_size=(112, 112)): + self.frequent: int = frequent + self.rank: int = rank + self.highest_acc: float = 0.0 + self.highest_acc_list: List[float] = [0.0] * len(val_targets) + self.ver_list: List[object] = [] + self.ver_name_list: List[str] = [] + if self.rank is 0: + self.init_dataset(val_targets=val_targets, data_dir=rec_prefix, image_size=image_size) + + def ver_test(self, backbone: torch.nn.Module, global_step: int): + results = [] + for i in range(len(self.ver_list)): + acc1, std1, acc2, std2, xnorm, embeddings_list = verification.test( + self.ver_list[i], backbone, 10, 10) + logging.info('[%s][%d]XNorm: %f' % (self.ver_name_list[i], global_step, xnorm)) + logging.info('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % (self.ver_name_list[i], global_step, acc2, std2)) + if acc2 > self.highest_acc_list[i]: + self.highest_acc_list[i] = acc2 + logging.info( + '[%s][%d]Accuracy-Highest: %1.5f' % (self.ver_name_list[i], global_step, self.highest_acc_list[i])) + results.append(acc2) + + def init_dataset(self, val_targets, data_dir, image_size): + for name in val_targets: + path = os.path.join(data_dir, name + ".bin") + if os.path.exists(path): + data_set = verification.load_bin(path, image_size) + self.ver_list.append(data_set) + self.ver_name_list.append(name) + + def __call__(self, num_update, backbone: torch.nn.Module): + if self.rank is 0 and num_update > 0 and num_update % self.frequent == 0: + backbone.eval() + self.ver_test(backbone, num_update) + backbone.train() + + +class CallBackLogging(object): + def __init__(self, frequent, rank, total_step, batch_size, world_size, writer=None): + self.frequent: int = frequent + self.rank: int = rank + self.time_start = time.time() + self.total_step: int = total_step + self.batch_size: int = batch_size + self.world_size: int = world_size + self.writer = writer + + self.init = False + self.tic = 0 + + def __call__(self, + global_step: int, + loss: AverageMeter, + epoch: int, + fp16: bool, + learning_rate: float, + grad_scaler: torch.cuda.amp.GradScaler): + if self.rank == 0 and global_step > 0 and global_step % self.frequent == 0: + if self.init: + try: + speed: float = self.frequent * self.batch_size / (time.time() - self.tic) + speed_total = speed * self.world_size + except ZeroDivisionError: + speed_total = float('inf') + + time_now = (time.time() - self.time_start) / 3600 + time_total = time_now / ((global_step + 1) / self.total_step) + time_for_end = time_total - time_now + if self.writer is not None: + self.writer.add_scalar('time_for_end', time_for_end, global_step) + self.writer.add_scalar('learning_rate', learning_rate, global_step) + self.writer.add_scalar('loss', loss.avg, global_step) + if fp16: + msg = "Speed %.2f samples/sec Loss %.4f LearningRate %.4f Epoch: %d Global Step: %d " \ + "Fp16 Grad Scale: %2.f Required: %1.f hours" % ( + speed_total, loss.avg, learning_rate, epoch, global_step, + grad_scaler.get_scale(), time_for_end + ) + else: + msg = "Speed %.2f samples/sec Loss %.4f LearningRate %.4f Epoch: %d Global Step: %d " \ + "Required: %1.f hours" % ( + speed_total, loss.avg, learning_rate, epoch, global_step, time_for_end + ) + logging.info(msg) + loss.reset() + self.tic = time.time() + else: + self.init = True + self.tic = time.time() + + +class CallBackModelCheckpoint(object): + def __init__(self, rank, output="./"): + self.rank: int = rank + self.output: str = output + + def __call__(self, global_step, backbone, partial_fc, ): + if global_step > 100 and self.rank == 0: + path_module = os.path.join(self.output, "backbone.pth") + torch.save(backbone.module.state_dict(), path_module) + logging.info("Pytorch Model Saved in '{}'".format(path_module)) + + if global_step > 100 and partial_fc is not None: + partial_fc.save_params() diff --git a/src/face3d/models/arcface_torch/utils/utils_config.py b/src/face3d/models/arcface_torch/utils/utils_config.py new file mode 100644 index 0000000000000000000000000000000000000000..0c02eaf70fc0140aca7925f621c29a496f491cae --- /dev/null +++ b/src/face3d/models/arcface_torch/utils/utils_config.py @@ -0,0 +1,16 @@ +import importlib +import os.path as osp + + +def get_config(config_file): + assert config_file.startswith('configs/'), 'config file setting must start with configs/' + temp_config_name = osp.basename(config_file) + temp_module_name = osp.splitext(temp_config_name)[0] + config = importlib.import_module("configs.base") + cfg = config.config + config = importlib.import_module("configs.%s" % temp_module_name) + job_cfg = config.config + cfg.update(job_cfg) + if cfg.output is None: + cfg.output = osp.join('work_dirs', temp_module_name) + return cfg \ No newline at end of file diff --git a/src/face3d/models/arcface_torch/utils/utils_logging.py b/src/face3d/models/arcface_torch/utils/utils_logging.py new file mode 100644 index 0000000000000000000000000000000000000000..c787b6aae7cd037a4718df44d672b8ffa9e5c249 --- /dev/null +++ b/src/face3d/models/arcface_torch/utils/utils_logging.py @@ -0,0 +1,41 @@ +import logging +import os +import sys + + +class AverageMeter(object): + """Computes and stores the average and current value + """ + + def __init__(self): + self.val = None + self.avg = None + self.sum = None + self.count = None + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def init_logging(rank, models_root): + if rank == 0: + log_root = logging.getLogger() + log_root.setLevel(logging.INFO) + formatter = logging.Formatter("Training: %(asctime)s-%(message)s") + handler_file = logging.FileHandler(os.path.join(models_root, "training.log")) + handler_stream = logging.StreamHandler(sys.stdout) + handler_file.setFormatter(formatter) + handler_stream.setFormatter(formatter) + log_root.addHandler(handler_file) + log_root.addHandler(handler_stream) + log_root.info('rank_id: %d' % rank) diff --git a/src/face3d/models/arcface_torch/utils/utils_os.py b/src/face3d/models/arcface_torch/utils/utils_os.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/face3d/models/base_model.py b/src/face3d/models/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..cfe64a7f739ad8f8cfbf3073a2bf49e1468127fd --- /dev/null +++ b/src/face3d/models/base_model.py @@ -0,0 +1,316 @@ +"""This script defines the base network model for Deep3DFaceRecon_pytorch +""" + +import os +import numpy as np +import torch +from collections import OrderedDict +from abc import ABC, abstractmethod +from . import networks + + +class BaseModel(ABC): + """This class is an abstract base class (ABC) for models. + To create a subclass, you need to implement the following five functions: + -- <__init__>: initialize the class; first call BaseModel.__init__(self, opt). + -- : unpack data from dataset and apply preprocessing. + -- : produce intermediate results. + -- : calculate losses, gradients, and update network weights. + -- : (optionally) add model-specific options and set default options. + """ + + def __init__(self, opt): + """Initialize the BaseModel class. + + Parameters: + opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions + + When creating your custom class, you need to implement your own initialization. + In this fucntion, you should first call + Then, you need to define four lists: + -- self.loss_names (str list): specify the training losses that you want to plot and save. + -- self.model_names (str list): specify the images that you want to display and save. + -- self.visual_names (str list): define networks used in our training. + -- self.optimizers (optimizer list): define and initialize optimizers. You can define one optimizer for each network. If two networks are updated at the same time, you can use itertools.chain to group them. See cycle_gan_model.py for an example. + """ + self.opt = opt + self.isTrain = False + self.device = torch.device('cpu') + self.save_dir = " " # os.path.join(opt.checkpoints_dir, opt.name) # save all the checkpoints to save_dir + self.loss_names = [] + self.model_names = [] + self.visual_names = [] + self.parallel_names = [] + self.optimizers = [] + self.image_paths = [] + self.metric = 0 # used for learning rate policy 'plateau' + + @staticmethod + def dict_grad_hook_factory(add_func=lambda x: x): + saved_dict = dict() + + def hook_gen(name): + def grad_hook(grad): + saved_vals = add_func(grad) + saved_dict[name] = saved_vals + return grad_hook + return hook_gen, saved_dict + + @staticmethod + def modify_commandline_options(parser, is_train): + """Add new model-specific options, and rewrite default values for existing options. + + Parameters: + parser -- original option parser + is_train (bool) -- whether training phase or test phase. You can use this flag to add training-specific or test-specific options. + + Returns: + the modified parser. + """ + return parser + + @abstractmethod + def set_input(self, input): + """Unpack input data from the dataloader and perform necessary pre-processing steps. + + Parameters: + input (dict): includes the data itself and its metadata information. + """ + pass + + @abstractmethod + def forward(self): + """Run forward pass; called by both functions and .""" + pass + + @abstractmethod + def optimize_parameters(self): + """Calculate losses, gradients, and update network weights; called in every training iteration""" + pass + + def setup(self, opt): + """Load and print networks; create schedulers + + Parameters: + opt (Option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions + """ + if self.isTrain: + self.schedulers = [networks.get_scheduler(optimizer, opt) for optimizer in self.optimizers] + + if not self.isTrain or opt.continue_train: + load_suffix = opt.epoch + self.load_networks(load_suffix) + + + # self.print_networks(opt.verbose) + + def parallelize(self, convert_sync_batchnorm=True): + if not self.opt.use_ddp: + for name in self.parallel_names: + if isinstance(name, str): + module = getattr(self, name) + setattr(self, name, module.to(self.device)) + else: + for name in self.model_names: + if isinstance(name, str): + module = getattr(self, name) + if convert_sync_batchnorm: + module = torch.nn.SyncBatchNorm.convert_sync_batchnorm(module) + setattr(self, name, torch.nn.parallel.DistributedDataParallel(module.to(self.device), + device_ids=[self.device.index], + find_unused_parameters=True, broadcast_buffers=True)) + + # DistributedDataParallel is not needed when a module doesn't have any parameter that requires a gradient. + for name in self.parallel_names: + if isinstance(name, str) and name not in self.model_names: + module = getattr(self, name) + setattr(self, name, module.to(self.device)) + + # put state_dict of optimizer to gpu device + if self.opt.phase != 'test': + if self.opt.continue_train: + for optim in self.optimizers: + for state in optim.state.values(): + for k, v in state.items(): + if isinstance(v, torch.Tensor): + state[k] = v.to(self.device) + + def data_dependent_initialize(self, data): + pass + + def train(self): + """Make models train mode""" + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, name) + net.train() + + def eval(self): + """Make models eval mode""" + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, name) + net.eval() + + def test(self): + """Forward function used in test time. + + This function wraps function in no_grad() so we don't save intermediate steps for backprop + It also calls to produce additional visualization results + """ + with torch.no_grad(): + self.forward() + self.compute_visuals() + + def compute_visuals(self): + """Calculate additional output images for visdom and HTML visualization""" + pass + + def get_image_paths(self, name='A'): + """ Return image paths that are used to load current data""" + return self.image_paths if name =='A' else self.image_paths_B + + def update_learning_rate(self): + """Update learning rates for all the networks; called at the end of every epoch""" + for scheduler in self.schedulers: + if self.opt.lr_policy == 'plateau': + scheduler.step(self.metric) + else: + scheduler.step() + + lr = self.optimizers[0].param_groups[0]['lr'] + print('learning rate = %.7f' % lr) + + def get_current_visuals(self): + """Return visualization images. train.py will display these images with visdom, and save the images to a HTML""" + visual_ret = OrderedDict() + for name in self.visual_names: + if isinstance(name, str): + visual_ret[name] = getattr(self, name)[:, :3, ...] + return visual_ret + + def get_current_losses(self): + """Return traning losses / errors. train.py will print out these errors on console, and save them to a file""" + errors_ret = OrderedDict() + for name in self.loss_names: + if isinstance(name, str): + errors_ret[name] = float(getattr(self, 'loss_' + name)) # float(...) works for both scalar tensor and float number + return errors_ret + + def save_networks(self, epoch): + """Save all the networks to the disk. + + Parameters: + epoch (int) -- current epoch; used in the file name '%s_net_%s.pth' % (epoch, name) + """ + if not os.path.isdir(self.save_dir): + os.makedirs(self.save_dir) + + save_filename = 'epoch_%s.pth' % (epoch) + save_path = os.path.join(self.save_dir, save_filename) + + save_dict = {} + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, name) + if isinstance(net, torch.nn.DataParallel) or isinstance(net, + torch.nn.parallel.DistributedDataParallel): + net = net.module + save_dict[name] = net.state_dict() + + + for i, optim in enumerate(self.optimizers): + save_dict['opt_%02d'%i] = optim.state_dict() + + for i, sched in enumerate(self.schedulers): + save_dict['sched_%02d'%i] = sched.state_dict() + + torch.save(save_dict, save_path) + + def __patch_instance_norm_state_dict(self, state_dict, module, keys, i=0): + """Fix InstanceNorm checkpoints incompatibility (prior to 0.4)""" + key = keys[i] + if i + 1 == len(keys): # at the end, pointing to a parameter/buffer + if module.__class__.__name__.startswith('InstanceNorm') and \ + (key == 'running_mean' or key == 'running_var'): + if getattr(module, key) is None: + state_dict.pop('.'.join(keys)) + if module.__class__.__name__.startswith('InstanceNorm') and \ + (key == 'num_batches_tracked'): + state_dict.pop('.'.join(keys)) + else: + self.__patch_instance_norm_state_dict(state_dict, getattr(module, key), keys, i + 1) + + def load_networks(self, epoch): + """Load all the networks from the disk. + + Parameters: + epoch (int) -- current epoch; used in the file name '%s_net_%s.pth' % (epoch, name) + """ + if self.opt.isTrain and self.opt.pretrained_name is not None: + load_dir = os.path.join(self.opt.checkpoints_dir, self.opt.pretrained_name) + else: + load_dir = self.save_dir + load_filename = 'epoch_%s.pth' % (epoch) + load_path = os.path.join(load_dir, load_filename) + state_dict = torch.load(load_path, map_location=self.device) + print('loading the model from %s' % load_path) + + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, name) + if isinstance(net, torch.nn.DataParallel): + net = net.module + net.load_state_dict(state_dict[name]) + + if self.opt.phase != 'test': + if self.opt.continue_train: + print('loading the optim from %s' % load_path) + for i, optim in enumerate(self.optimizers): + optim.load_state_dict(state_dict['opt_%02d'%i]) + + try: + print('loading the sched from %s' % load_path) + for i, sched in enumerate(self.schedulers): + sched.load_state_dict(state_dict['sched_%02d'%i]) + except: + print('Failed to load schedulers, set schedulers according to epoch count manually') + for i, sched in enumerate(self.schedulers): + sched.last_epoch = self.opt.epoch_count - 1 + + + + + def print_networks(self, verbose): + """Print the total number of parameters in the network and (if verbose) network architecture + + Parameters: + verbose (bool) -- if verbose: print the network architecture + """ + print('---------- Networks initialized -------------') + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, name) + num_params = 0 + for param in net.parameters(): + num_params += param.numel() + if verbose: + print(net) + print('[Network %s] Total number of parameters : %.3f M' % (name, num_params / 1e6)) + print('-----------------------------------------------') + + def set_requires_grad(self, nets, requires_grad=False): + """Set requies_grad=Fasle for all the networks to avoid unnecessary computations + Parameters: + nets (network list) -- a list of networks + requires_grad (bool) -- whether the networks require gradients or not + """ + if not isinstance(nets, list): + nets = [nets] + for net in nets: + if net is not None: + for param in net.parameters(): + param.requires_grad = requires_grad + + def generate_visuals_for_evaluation(self, data, mode): + return {} diff --git a/src/face3d/models/bfm.py b/src/face3d/models/bfm.py new file mode 100644 index 0000000000000000000000000000000000000000..a75db682f02dd1979d4a7de1d11dd3aa5cdf5279 --- /dev/null +++ b/src/face3d/models/bfm.py @@ -0,0 +1,331 @@ +"""This script defines the parametric 3d face model for Deep3DFaceRecon_pytorch +""" + +import numpy as np +import torch +import torch.nn.functional as F +from scipy.io import loadmat +from src.face3d.util.load_mats import transferBFM09 +import os + +def perspective_projection(focal, center): + # return p.T (N, 3) @ (3, 3) + return np.array([ + focal, 0, center, + 0, focal, center, + 0, 0, 1 + ]).reshape([3, 3]).astype(np.float32).transpose() + +class SH: + def __init__(self): + self.a = [np.pi, 2 * np.pi / np.sqrt(3.), 2 * np.pi / np.sqrt(8.)] + self.c = [1/np.sqrt(4 * np.pi), np.sqrt(3.) / np.sqrt(4 * np.pi), 3 * np.sqrt(5.) / np.sqrt(12 * np.pi)] + + + +class ParametricFaceModel: + def __init__(self, + bfm_folder='./BFM', + recenter=True, + camera_distance=10., + init_lit=np.array([ + 0.8, 0, 0, 0, 0, 0, 0, 0, 0 + ]), + focal=1015., + center=112., + is_train=True, + default_name='BFM_model_front.mat'): + + if not os.path.isfile(os.path.join(bfm_folder, default_name)): + transferBFM09(bfm_folder) + + model = loadmat(os.path.join(bfm_folder, default_name)) + # mean face shape. [3*N,1] + self.mean_shape = model['meanshape'].astype(np.float32) + # identity basis. [3*N,80] + self.id_base = model['idBase'].astype(np.float32) + # expression basis. [3*N,64] + self.exp_base = model['exBase'].astype(np.float32) + # mean face texture. [3*N,1] (0-255) + self.mean_tex = model['meantex'].astype(np.float32) + # texture basis. [3*N,80] + self.tex_base = model['texBase'].astype(np.float32) + # face indices for each vertex that lies in. starts from 0. [N,8] + self.point_buf = model['point_buf'].astype(np.int64) - 1 + # vertex indices for each face. starts from 0. [F,3] + self.face_buf = model['tri'].astype(np.int64) - 1 + # vertex indices for 68 landmarks. starts from 0. [68,1] + self.keypoints = np.squeeze(model['keypoints']).astype(np.int64) - 1 + + if is_train: + # vertex indices for small face region to compute photometric error. starts from 0. + self.front_mask = np.squeeze(model['frontmask2_idx']).astype(np.int64) - 1 + # vertex indices for each face from small face region. starts from 0. [f,3] + self.front_face_buf = model['tri_mask2'].astype(np.int64) - 1 + # vertex indices for pre-defined skin region to compute reflectance loss + self.skin_mask = np.squeeze(model['skinmask']) + + if recenter: + mean_shape = self.mean_shape.reshape([-1, 3]) + mean_shape = mean_shape - np.mean(mean_shape, axis=0, keepdims=True) + self.mean_shape = mean_shape.reshape([-1, 1]) + + self.persc_proj = perspective_projection(focal, center) + self.device = 'cpu' + self.camera_distance = camera_distance + self.SH = SH() + self.init_lit = init_lit.reshape([1, 1, -1]).astype(np.float32) + + + def to(self, device): + self.device = device + for key, value in self.__dict__.items(): + if type(value).__module__ == np.__name__: + setattr(self, key, torch.tensor(value).to(device)) + + + def compute_shape(self, id_coeff, exp_coeff): + """ + Return: + face_shape -- torch.tensor, size (B, N, 3) + + Parameters: + id_coeff -- torch.tensor, size (B, 80), identity coeffs + exp_coeff -- torch.tensor, size (B, 64), expression coeffs + """ + batch_size = id_coeff.shape[0] + id_part = torch.einsum('ij,aj->ai', self.id_base, id_coeff) + exp_part = torch.einsum('ij,aj->ai', self.exp_base, exp_coeff) + face_shape = id_part + exp_part + self.mean_shape.reshape([1, -1]) + return face_shape.reshape([batch_size, -1, 3]) + + + def compute_texture(self, tex_coeff, normalize=True): + """ + Return: + face_texture -- torch.tensor, size (B, N, 3), in RGB order, range (0, 1.) + + Parameters: + tex_coeff -- torch.tensor, size (B, 80) + """ + batch_size = tex_coeff.shape[0] + face_texture = torch.einsum('ij,aj->ai', self.tex_base, tex_coeff) + self.mean_tex + if normalize: + face_texture = face_texture / 255. + return face_texture.reshape([batch_size, -1, 3]) + + + def compute_norm(self, face_shape): + """ + Return: + vertex_norm -- torch.tensor, size (B, N, 3) + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + """ + + v1 = face_shape[:, self.face_buf[:, 0]] + v2 = face_shape[:, self.face_buf[:, 1]] + v3 = face_shape[:, self.face_buf[:, 2]] + e1 = v1 - v2 + e2 = v2 - v3 + face_norm = torch.cross(e1, e2, dim=-1) + face_norm = F.normalize(face_norm, dim=-1, p=2) + face_norm = torch.cat([face_norm, torch.zeros(face_norm.shape[0], 1, 3).to(self.device)], dim=1) + + vertex_norm = torch.sum(face_norm[:, self.point_buf], dim=2) + vertex_norm = F.normalize(vertex_norm, dim=-1, p=2) + return vertex_norm + + + def compute_color(self, face_texture, face_norm, gamma): + """ + Return: + face_color -- torch.tensor, size (B, N, 3), range (0, 1.) + + Parameters: + face_texture -- torch.tensor, size (B, N, 3), from texture model, range (0, 1.) + face_norm -- torch.tensor, size (B, N, 3), rotated face normal + gamma -- torch.tensor, size (B, 27), SH coeffs + """ + batch_size = gamma.shape[0] + v_num = face_texture.shape[1] + a, c = self.SH.a, self.SH.c + gamma = gamma.reshape([batch_size, 3, 9]) + gamma = gamma + self.init_lit + gamma = gamma.permute(0, 2, 1) + Y = torch.cat([ + a[0] * c[0] * torch.ones_like(face_norm[..., :1]).to(self.device), + -a[1] * c[1] * face_norm[..., 1:2], + a[1] * c[1] * face_norm[..., 2:], + -a[1] * c[1] * face_norm[..., :1], + a[2] * c[2] * face_norm[..., :1] * face_norm[..., 1:2], + -a[2] * c[2] * face_norm[..., 1:2] * face_norm[..., 2:], + 0.5 * a[2] * c[2] / np.sqrt(3.) * (3 * face_norm[..., 2:] ** 2 - 1), + -a[2] * c[2] * face_norm[..., :1] * face_norm[..., 2:], + 0.5 * a[2] * c[2] * (face_norm[..., :1] ** 2 - face_norm[..., 1:2] ** 2) + ], dim=-1) + r = Y @ gamma[..., :1] + g = Y @ gamma[..., 1:2] + b = Y @ gamma[..., 2:] + face_color = torch.cat([r, g, b], dim=-1) * face_texture + return face_color + + + def compute_rotation(self, angles): + """ + Return: + rot -- torch.tensor, size (B, 3, 3) pts @ trans_mat + + Parameters: + angles -- torch.tensor, size (B, 3), radian + """ + + batch_size = angles.shape[0] + ones = torch.ones([batch_size, 1]).to(self.device) + zeros = torch.zeros([batch_size, 1]).to(self.device) + x, y, z = angles[:, :1], angles[:, 1:2], angles[:, 2:], + + rot_x = torch.cat([ + ones, zeros, zeros, + zeros, torch.cos(x), -torch.sin(x), + zeros, torch.sin(x), torch.cos(x) + ], dim=1).reshape([batch_size, 3, 3]) + + rot_y = torch.cat([ + torch.cos(y), zeros, torch.sin(y), + zeros, ones, zeros, + -torch.sin(y), zeros, torch.cos(y) + ], dim=1).reshape([batch_size, 3, 3]) + + rot_z = torch.cat([ + torch.cos(z), -torch.sin(z), zeros, + torch.sin(z), torch.cos(z), zeros, + zeros, zeros, ones + ], dim=1).reshape([batch_size, 3, 3]) + + rot = rot_z @ rot_y @ rot_x + return rot.permute(0, 2, 1) + + + def to_camera(self, face_shape): + face_shape[..., -1] = self.camera_distance - face_shape[..., -1] + return face_shape + + def to_image(self, face_shape): + """ + Return: + face_proj -- torch.tensor, size (B, N, 2), y direction is opposite to v direction + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + """ + # to image_plane + face_proj = face_shape @ self.persc_proj + face_proj = face_proj[..., :2] / face_proj[..., 2:] + + return face_proj + + + def transform(self, face_shape, rot, trans): + """ + Return: + face_shape -- torch.tensor, size (B, N, 3) pts @ rot + trans + + Parameters: + face_shape -- torch.tensor, size (B, N, 3) + rot -- torch.tensor, size (B, 3, 3) + trans -- torch.tensor, size (B, 3) + """ + return face_shape @ rot + trans.unsqueeze(1) + + + def get_landmarks(self, face_proj): + """ + Return: + face_lms -- torch.tensor, size (B, 68, 2) + + Parameters: + face_proj -- torch.tensor, size (B, N, 2) + """ + return face_proj[:, self.keypoints] + + def split_coeff(self, coeffs): + """ + Return: + coeffs_dict -- a dict of torch.tensors + + Parameters: + coeffs -- torch.tensor, size (B, 256) + """ + id_coeffs = coeffs[:, :80] + exp_coeffs = coeffs[:, 80: 144] + tex_coeffs = coeffs[:, 144: 224] + angles = coeffs[:, 224: 227] + gammas = coeffs[:, 227: 254] + translations = coeffs[:, 254:] + return { + 'id': id_coeffs, + 'exp': exp_coeffs, + 'tex': tex_coeffs, + 'angle': angles, + 'gamma': gammas, + 'trans': translations + } + def compute_for_render(self, coeffs): + """ + Return: + face_vertex -- torch.tensor, size (B, N, 3), in camera coordinate + face_color -- torch.tensor, size (B, N, 3), in RGB order + landmark -- torch.tensor, size (B, 68, 2), y direction is opposite to v direction + Parameters: + coeffs -- torch.tensor, size (B, 257) + """ + coef_dict = self.split_coeff(coeffs) + face_shape = self.compute_shape(coef_dict['id'], coef_dict['exp']) + rotation = self.compute_rotation(coef_dict['angle']) + + + face_shape_transformed = self.transform(face_shape, rotation, coef_dict['trans']) + face_vertex = self.to_camera(face_shape_transformed) + + face_proj = self.to_image(face_vertex) + landmark = self.get_landmarks(face_proj) + + face_texture = self.compute_texture(coef_dict['tex']) + face_norm = self.compute_norm(face_shape) + face_norm_roted = face_norm @ rotation + face_color = self.compute_color(face_texture, face_norm_roted, coef_dict['gamma']) + + return face_vertex, face_texture, face_color, landmark + + def compute_for_render_woRotation(self, coeffs): + """ + Return: + face_vertex -- torch.tensor, size (B, N, 3), in camera coordinate + face_color -- torch.tensor, size (B, N, 3), in RGB order + landmark -- torch.tensor, size (B, 68, 2), y direction is opposite to v direction + Parameters: + coeffs -- torch.tensor, size (B, 257) + """ + coef_dict = self.split_coeff(coeffs) + face_shape = self.compute_shape(coef_dict['id'], coef_dict['exp']) + #rotation = self.compute_rotation(coef_dict['angle']) + + + #face_shape_transformed = self.transform(face_shape, rotation, coef_dict['trans']) + face_vertex = self.to_camera(face_shape) + + face_proj = self.to_image(face_vertex) + landmark = self.get_landmarks(face_proj) + + face_texture = self.compute_texture(coef_dict['tex']) + face_norm = self.compute_norm(face_shape) + face_norm_roted = face_norm # @ rotation + face_color = self.compute_color(face_texture, face_norm_roted, coef_dict['gamma']) + + return face_vertex, face_texture, face_color, landmark + + +if __name__ == '__main__': + transferBFM09() \ No newline at end of file diff --git a/src/face3d/models/facerecon_model.py b/src/face3d/models/facerecon_model.py new file mode 100644 index 0000000000000000000000000000000000000000..7de8ca6eebc50ff1ed52c5ba37d31b43f977b5e1 --- /dev/null +++ b/src/face3d/models/facerecon_model.py @@ -0,0 +1,220 @@ +"""This script defines the face reconstruction model for Deep3DFaceRecon_pytorch +""" + +import numpy as np +import torch +from src.face3d.models.base_model import BaseModel +from src.face3d.models import networks +from src.face3d.models.bfm import ParametricFaceModel +from src.face3d.models.losses import perceptual_loss, photo_loss, reg_loss, reflectance_loss, landmark_loss +from src.face3d.util import util +from src.face3d.util.nvdiffrast import MeshRenderer +# from src.face3d.util.preprocess import estimate_norm_torch + +import trimesh +from scipy.io import savemat + +class FaceReconModel(BaseModel): + + @staticmethod + def modify_commandline_options(parser, is_train=False): + """ Configures options specific for CUT model + """ + # net structure and parameters + parser.add_argument('--net_recon', type=str, default='resnet50', choices=['resnet18', 'resnet34', 'resnet50'], help='network structure') + parser.add_argument('--init_path', type=str, default='./checkpoints/init_model/resnet50-0676ba61.pth') + parser.add_argument('--use_last_fc', type=util.str2bool, nargs='?', const=True, default=False, help='zero initialize the last fc') + parser.add_argument('--bfm_folder', type=str, default='./checkpoints/BFM_Fitting/') + parser.add_argument('--bfm_model', type=str, default='BFM_model_front.mat', help='bfm model') + + # renderer parameters + parser.add_argument('--focal', type=float, default=1015.) + parser.add_argument('--center', type=float, default=112.) + parser.add_argument('--camera_d', type=float, default=10.) + parser.add_argument('--z_near', type=float, default=5.) + parser.add_argument('--z_far', type=float, default=15.) + + if is_train: + # training parameters + parser.add_argument('--net_recog', type=str, default='r50', choices=['r18', 'r43', 'r50'], help='face recog network structure') + parser.add_argument('--net_recog_path', type=str, default='checkpoints/recog_model/ms1mv3_arcface_r50_fp16/backbone.pth') + parser.add_argument('--use_crop_face', type=util.str2bool, nargs='?', const=True, default=False, help='use crop mask for photo loss') + parser.add_argument('--use_predef_M', type=util.str2bool, nargs='?', const=True, default=False, help='use predefined M for predicted face') + + + # augmentation parameters + parser.add_argument('--shift_pixs', type=float, default=10., help='shift pixels') + parser.add_argument('--scale_delta', type=float, default=0.1, help='delta scale factor') + parser.add_argument('--rot_angle', type=float, default=10., help='rot angles, degree') + + # loss weights + parser.add_argument('--w_feat', type=float, default=0.2, help='weight for feat loss') + parser.add_argument('--w_color', type=float, default=1.92, help='weight for loss loss') + parser.add_argument('--w_reg', type=float, default=3.0e-4, help='weight for reg loss') + parser.add_argument('--w_id', type=float, default=1.0, help='weight for id_reg loss') + parser.add_argument('--w_exp', type=float, default=0.8, help='weight for exp_reg loss') + parser.add_argument('--w_tex', type=float, default=1.7e-2, help='weight for tex_reg loss') + parser.add_argument('--w_gamma', type=float, default=10.0, help='weight for gamma loss') + parser.add_argument('--w_lm', type=float, default=1.6e-3, help='weight for lm loss') + parser.add_argument('--w_reflc', type=float, default=5.0, help='weight for reflc loss') + + opt, _ = parser.parse_known_args() + parser.set_defaults( + focal=1015., center=112., camera_d=10., use_last_fc=False, z_near=5., z_far=15. + ) + if is_train: + parser.set_defaults( + use_crop_face=True, use_predef_M=False + ) + return parser + + def __init__(self, opt): + """Initialize this model class. + + Parameters: + opt -- training/test options + + A few things can be done here. + - (required) call the initialization function of BaseModel + - define loss function, visualization images, model names, and optimizers + """ + BaseModel.__init__(self, opt) # call the initialization method of BaseModel + + self.visual_names = ['output_vis'] + self.model_names = ['net_recon'] + self.parallel_names = self.model_names + ['renderer'] + + self.facemodel = ParametricFaceModel( + bfm_folder=opt.bfm_folder, camera_distance=opt.camera_d, focal=opt.focal, center=opt.center, + is_train=self.isTrain, default_name=opt.bfm_model + ) + + fov = 2 * np.arctan(opt.center / opt.focal) * 180 / np.pi + self.renderer = MeshRenderer( + rasterize_fov=fov, znear=opt.z_near, zfar=opt.z_far, rasterize_size=int(2 * opt.center) + ) + + if self.isTrain: + self.loss_names = ['all', 'feat', 'color', 'lm', 'reg', 'gamma', 'reflc'] + + self.net_recog = networks.define_net_recog( + net_recog=opt.net_recog, pretrained_path=opt.net_recog_path + ) + # loss func name: (compute_%s_loss) % loss_name + self.compute_feat_loss = perceptual_loss + self.comupte_color_loss = photo_loss + self.compute_lm_loss = landmark_loss + self.compute_reg_loss = reg_loss + self.compute_reflc_loss = reflectance_loss + + self.optimizer = torch.optim.Adam(self.net_recon.parameters(), lr=opt.lr) + self.optimizers = [self.optimizer] + self.parallel_names += ['net_recog'] + # Our program will automatically call to define schedulers, load networks, and print networks + + def set_input(self, input): + """Unpack input data from the dataloader and perform necessary pre-processing steps. + + Parameters: + input: a dictionary that contains the data itself and its metadata information. + """ + self.input_img = input['imgs'].to(self.device) + self.atten_mask = input['msks'].to(self.device) if 'msks' in input else None + self.gt_lm = input['lms'].to(self.device) if 'lms' in input else None + self.trans_m = input['M'].to(self.device) if 'M' in input else None + self.image_paths = input['im_paths'] if 'im_paths' in input else None + + def forward(self, output_coeff, device): + self.facemodel.to(device) + self.pred_vertex, self.pred_tex, self.pred_color, self.pred_lm = \ + self.facemodel.compute_for_render(output_coeff) + self.pred_mask, _, self.pred_face = self.renderer( + self.pred_vertex, self.facemodel.face_buf, feat=self.pred_color) + + self.pred_coeffs_dict = self.facemodel.split_coeff(output_coeff) + + + def compute_losses(self): + """Calculate losses, gradients, and update network weights; called in every training iteration""" + + assert self.net_recog.training == False + trans_m = self.trans_m + if not self.opt.use_predef_M: + trans_m = estimate_norm_torch(self.pred_lm, self.input_img.shape[-2]) + + pred_feat = self.net_recog(self.pred_face, trans_m) + gt_feat = self.net_recog(self.input_img, self.trans_m) + self.loss_feat = self.opt.w_feat * self.compute_feat_loss(pred_feat, gt_feat) + + face_mask = self.pred_mask + if self.opt.use_crop_face: + face_mask, _, _ = self.renderer(self.pred_vertex, self.facemodel.front_face_buf) + + face_mask = face_mask.detach() + self.loss_color = self.opt.w_color * self.comupte_color_loss( + self.pred_face, self.input_img, self.atten_mask * face_mask) + + loss_reg, loss_gamma = self.compute_reg_loss(self.pred_coeffs_dict, self.opt) + self.loss_reg = self.opt.w_reg * loss_reg + self.loss_gamma = self.opt.w_gamma * loss_gamma + + self.loss_lm = self.opt.w_lm * self.compute_lm_loss(self.pred_lm, self.gt_lm) + + self.loss_reflc = self.opt.w_reflc * self.compute_reflc_loss(self.pred_tex, self.facemodel.skin_mask) + + self.loss_all = self.loss_feat + self.loss_color + self.loss_reg + self.loss_gamma \ + + self.loss_lm + self.loss_reflc + + + def optimize_parameters(self, isTrain=True): + self.forward() + self.compute_losses() + """Update network weights; it will be called in every training iteration.""" + if isTrain: + self.optimizer.zero_grad() + self.loss_all.backward() + self.optimizer.step() + + def compute_visuals(self): + with torch.no_grad(): + input_img_numpy = 255. * self.input_img.detach().cpu().permute(0, 2, 3, 1).numpy() + output_vis = self.pred_face * self.pred_mask + (1 - self.pred_mask) * self.input_img + output_vis_numpy_raw = 255. * output_vis.detach().cpu().permute(0, 2, 3, 1).numpy() + + if self.gt_lm is not None: + gt_lm_numpy = self.gt_lm.cpu().numpy() + pred_lm_numpy = self.pred_lm.detach().cpu().numpy() + output_vis_numpy = util.draw_landmarks(output_vis_numpy_raw, gt_lm_numpy, 'b') + output_vis_numpy = util.draw_landmarks(output_vis_numpy, pred_lm_numpy, 'r') + + output_vis_numpy = np.concatenate((input_img_numpy, + output_vis_numpy_raw, output_vis_numpy), axis=-2) + else: + output_vis_numpy = np.concatenate((input_img_numpy, + output_vis_numpy_raw), axis=-2) + + self.output_vis = torch.tensor( + output_vis_numpy / 255., dtype=torch.float32 + ).permute(0, 3, 1, 2).to(self.device) + + def save_mesh(self, name): + + recon_shape = self.pred_vertex # get reconstructed shape + recon_shape[..., -1] = 10 - recon_shape[..., -1] # from camera space to world space + recon_shape = recon_shape.cpu().numpy()[0] + recon_color = self.pred_color + recon_color = recon_color.cpu().numpy()[0] + tri = self.facemodel.face_buf.cpu().numpy() + mesh = trimesh.Trimesh(vertices=recon_shape, faces=tri, vertex_colors=np.clip(255. * recon_color, 0, 255).astype(np.uint8)) + mesh.export(name) + + def save_coeff(self,name): + + pred_coeffs = {key:self.pred_coeffs_dict[key].cpu().numpy() for key in self.pred_coeffs_dict} + pred_lm = self.pred_lm.cpu().numpy() + pred_lm = np.stack([pred_lm[:,:,0],self.input_img.shape[2]-1-pred_lm[:,:,1]],axis=2) # transfer to image coordinate + pred_coeffs['lm68'] = pred_lm + savemat(name,pred_coeffs) + + + diff --git a/src/face3d/models/losses.py b/src/face3d/models/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..09d6a85870af1ef2b857e4a3fdd4b2f7fc991317 --- /dev/null +++ b/src/face3d/models/losses.py @@ -0,0 +1,113 @@ +import numpy as np +import torch +import torch.nn as nn +from kornia.geometry import warp_affine +import torch.nn.functional as F + +def resize_n_crop(image, M, dsize=112): + # image: (b, c, h, w) + # M : (b, 2, 3) + return warp_affine(image, M, dsize=(dsize, dsize), align_corners=True) + +### perceptual level loss +class PerceptualLoss(nn.Module): + def __init__(self, recog_net, input_size=112): + super(PerceptualLoss, self).__init__() + self.recog_net = recog_net + self.preprocess = lambda x: 2 * x - 1 + self.input_size=input_size + def forward(imageA, imageB, M): + """ + 1 - cosine distance + Parameters: + imageA --torch.tensor (B, 3, H, W), range (0, 1) , RGB order + imageB --same as imageA + """ + + imageA = self.preprocess(resize_n_crop(imageA, M, self.input_size)) + imageB = self.preprocess(resize_n_crop(imageB, M, self.input_size)) + + # freeze bn + self.recog_net.eval() + + id_featureA = F.normalize(self.recog_net(imageA), dim=-1, p=2) + id_featureB = F.normalize(self.recog_net(imageB), dim=-1, p=2) + cosine_d = torch.sum(id_featureA * id_featureB, dim=-1) + # assert torch.sum((cosine_d > 1).float()) == 0 + return torch.sum(1 - cosine_d) / cosine_d.shape[0] + +def perceptual_loss(id_featureA, id_featureB): + cosine_d = torch.sum(id_featureA * id_featureB, dim=-1) + # assert torch.sum((cosine_d > 1).float()) == 0 + return torch.sum(1 - cosine_d) / cosine_d.shape[0] + +### image level loss +def photo_loss(imageA, imageB, mask, eps=1e-6): + """ + l2 norm (with sqrt, to ensure backward stabililty, use eps, otherwise Nan may occur) + Parameters: + imageA --torch.tensor (B, 3, H, W), range (0, 1), RGB order + imageB --same as imageA + """ + loss = torch.sqrt(eps + torch.sum((imageA - imageB) ** 2, dim=1, keepdims=True)) * mask + loss = torch.sum(loss) / torch.max(torch.sum(mask), torch.tensor(1.0).to(mask.device)) + return loss + +def landmark_loss(predict_lm, gt_lm, weight=None): + """ + weighted mse loss + Parameters: + predict_lm --torch.tensor (B, 68, 2) + gt_lm --torch.tensor (B, 68, 2) + weight --numpy.array (1, 68) + """ + if not weight: + weight = np.ones([68]) + weight[28:31] = 20 + weight[-8:] = 20 + weight = np.expand_dims(weight, 0) + weight = torch.tensor(weight).to(predict_lm.device) + loss = torch.sum((predict_lm - gt_lm)**2, dim=-1) * weight + loss = torch.sum(loss) / (predict_lm.shape[0] * predict_lm.shape[1]) + return loss + + +### regulization +def reg_loss(coeffs_dict, opt=None): + """ + l2 norm without the sqrt, from yu's implementation (mse) + tf.nn.l2_loss https://www.tensorflow.org/api_docs/python/tf/nn/l2_loss + Parameters: + coeffs_dict -- a dict of torch.tensors , keys: id, exp, tex, angle, gamma, trans + + """ + # coefficient regularization to ensure plausible 3d faces + if opt: + w_id, w_exp, w_tex = opt.w_id, opt.w_exp, opt.w_tex + else: + w_id, w_exp, w_tex = 1, 1, 1, 1 + creg_loss = w_id * torch.sum(coeffs_dict['id'] ** 2) + \ + w_exp * torch.sum(coeffs_dict['exp'] ** 2) + \ + w_tex * torch.sum(coeffs_dict['tex'] ** 2) + creg_loss = creg_loss / coeffs_dict['id'].shape[0] + + # gamma regularization to ensure a nearly-monochromatic light + gamma = coeffs_dict['gamma'].reshape([-1, 3, 9]) + gamma_mean = torch.mean(gamma, dim=1, keepdims=True) + gamma_loss = torch.mean((gamma - gamma_mean) ** 2) + + return creg_loss, gamma_loss + +def reflectance_loss(texture, mask): + """ + minimize texture variance (mse), albedo regularization to ensure an uniform skin albedo + Parameters: + texture --torch.tensor, (B, N, 3) + mask --torch.tensor, (N), 1 or 0 + + """ + mask = mask.reshape([1, mask.shape[0], 1]) + texture_mean = torch.sum(mask * texture, dim=1, keepdims=True) / torch.sum(mask) + loss = torch.sum(((texture - texture_mean) * mask)**2) / (texture.shape[0] * torch.sum(mask)) + return loss + diff --git a/src/face3d/models/networks.py b/src/face3d/models/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..ead9cdcb8720b845c233de79dc8a8d1668492108 --- /dev/null +++ b/src/face3d/models/networks.py @@ -0,0 +1,521 @@ +"""This script defines deep neural networks for Deep3DFaceRecon_pytorch +""" + +import os +import numpy as np +import torch.nn.functional as F +from torch.nn import init +import functools +from torch.optim import lr_scheduler +import torch +from torch import Tensor +import torch.nn as nn +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url +from typing import Type, Any, Callable, Union, List, Optional +from .arcface_torch.backbones import get_model +from kornia.geometry import warp_affine + +def resize_n_crop(image, M, dsize=112): + # image: (b, c, h, w) + # M : (b, 2, 3) + return warp_affine(image, M, dsize=(dsize, dsize), align_corners=True) + +def filter_state_dict(state_dict, remove_name='fc'): + new_state_dict = {} + for key in state_dict: + if remove_name in key: + continue + new_state_dict[key] = state_dict[key] + return new_state_dict + +def get_scheduler(optimizer, opt): + """Return a learning rate scheduler + + Parameters: + optimizer -- the optimizer of the network + opt (option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions．　 + opt.lr_policy is the name of learning rate policy: linear | step | plateau | cosine + + For other schedulers (step, plateau, and cosine), we use the default PyTorch schedulers. + See https://pytorch.org/docs/stable/optim.html for more details. + """ + if opt.lr_policy == 'linear': + def lambda_rule(epoch): + lr_l = 1.0 - max(0, epoch + opt.epoch_count - opt.n_epochs) / float(opt.n_epochs + 1) + return lr_l + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule) + elif opt.lr_policy == 'step': + scheduler = lr_scheduler.StepLR(optimizer, step_size=opt.lr_decay_epochs, gamma=0.2) + elif opt.lr_policy == 'plateau': + scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.2, threshold=0.01, patience=5) + elif opt.lr_policy == 'cosine': + scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=opt.n_epochs, eta_min=0) + else: + return NotImplementedError('learning rate policy [%s] is not implemented', opt.lr_policy) + return scheduler + + +def define_net_recon(net_recon, use_last_fc=False, init_path=None): + return ReconNetWrapper(net_recon, use_last_fc=use_last_fc, init_path=init_path) + +def define_net_recog(net_recog, pretrained_path=None): + net = RecogNetWrapper(net_recog=net_recog, pretrained_path=pretrained_path) + net.eval() + return net + +class ReconNetWrapper(nn.Module): + fc_dim=257 + def __init__(self, net_recon, use_last_fc=False, init_path=None): + super(ReconNetWrapper, self).__init__() + self.use_last_fc = use_last_fc + if net_recon not in func_dict: + return NotImplementedError('network [%s] is not implemented', net_recon) + func, last_dim = func_dict[net_recon] + backbone = func(use_last_fc=use_last_fc, num_classes=self.fc_dim) + if init_path and os.path.isfile(init_path): + state_dict = filter_state_dict(torch.load(init_path, map_location='cpu')) + backbone.load_state_dict(state_dict) + print("loading init net_recon %s from %s" %(net_recon, init_path)) + self.backbone = backbone + if not use_last_fc: + self.final_layers = nn.ModuleList([ + conv1x1(last_dim, 80, bias=True), # id layer + conv1x1(last_dim, 64, bias=True), # exp layer + conv1x1(last_dim, 80, bias=True), # tex layer + conv1x1(last_dim, 3, bias=True), # angle layer + conv1x1(last_dim, 27, bias=True), # gamma layer + conv1x1(last_dim, 2, bias=True), # tx, ty + conv1x1(last_dim, 1, bias=True) # tz + ]) + for m in self.final_layers: + nn.init.constant_(m.weight, 0.) + nn.init.constant_(m.bias, 0.) + + def forward(self, x): + x = self.backbone(x) + if not self.use_last_fc: + output = [] + for layer in self.final_layers: + output.append(layer(x)) + x = torch.flatten(torch.cat(output, dim=1), 1) + return x + + +class RecogNetWrapper(nn.Module): + def __init__(self, net_recog, pretrained_path=None, input_size=112): + super(RecogNetWrapper, self).__init__() + net = get_model(name=net_recog, fp16=False) + if pretrained_path: + state_dict = torch.load(pretrained_path, map_location='cpu') + net.load_state_dict(state_dict) + print("loading pretrained net_recog %s from %s" %(net_recog, pretrained_path)) + for param in net.parameters(): + param.requires_grad = False + self.net = net + self.preprocess = lambda x: 2 * x - 1 + self.input_size=input_size + + def forward(self, image, M): + image = self.preprocess(resize_n_crop(image, M, self.input_size)) + id_feature = F.normalize(self.net(image), dim=-1, p=2) + return id_feature + + +# adapted from https://github.com/pytorch/vision/edit/master/torchvision/models/resnet.py +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', + 'wide_resnet50_2', 'wide_resnet101_2'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-f37072fd.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-b627a593.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-0676ba61.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-63fe2227.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-394f9c45.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d: + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + + +def conv1x1(in_planes: int, out_planes: int, stride: int = 1, bias: bool = False) -> nn.Conv2d: + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=bias) + + +class BasicBlock(nn.Module): + expansion: int = 1 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None + ) -> None: + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion: int = 4 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None + ) -> None: + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__( + self, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + num_classes: int = 1000, + zero_init_residual: bool = False, + use_last_fc: bool = False, + groups: int = 1, + width_per_group: int = 64, + replace_stride_with_dilation: Optional[List[bool]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None + ) -> None: + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.use_last_fc = use_last_fc + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + if self.use_last_fc: + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) # type: ignore[arg-type] + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) # type: ignore[arg-type] + + def _make_layer(self, block: Type[Union[BasicBlock, Bottleneck]], planes: int, blocks: int, + stride: int = 1, dilate: bool = False) -> nn.Sequential: + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x: Tensor) -> Tensor: + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + if self.use_last_fc: + x = torch.flatten(x, 1) + x = self.fc(x) + return x + + def forward(self, x: Tensor) -> Tensor: + return self._forward_impl(x) + + +def _resnet( + arch: str, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + pretrained: bool, + progress: bool, + **kwargs: Any +) -> ResNet: + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + model.load_state_dict(state_dict) + return model + + +def resnet18(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress, + **kwargs) + + +def resnet152(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress, + **kwargs) + + +def resnext50_32x4d(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def resnext101_32x8d(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) + + +def wide_resnet50_2(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def wide_resnet101_2(pretrained: bool = False, progress: bool = True, **kwargs: Any) -> ResNet: + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) + + +func_dict = { + 'resnet18': (resnet18, 512), + 'resnet50': (resnet50, 2048) +} diff --git a/src/face3d/models/template_model.py b/src/face3d/models/template_model.py new file mode 100644 index 0000000000000000000000000000000000000000..dac7b33d5889777eb63c9882a3b9fa094dcab293 --- /dev/null +++ b/src/face3d/models/template_model.py @@ -0,0 +1,100 @@ +"""Model class template + +This module provides a template for users to implement custom models. +You can specify '--model template' to use this model. +The class name should be consistent with both the filename and its model option. +The filename should be _dataset.py +The class name should be Dataset.py +It implements a simple image-to-image translation baseline based on regression loss. +Given input-output pairs (data_A, data_B), it learns a network netG that can minimize the following L1 loss: + min_ ||netG(data_A) - data_B||_1 +You need to implement the following functions: + :　Add model-specific options and rewrite default values for existing options. + <__init__>: Initialize this model class. + : Unpack input data and perform data pre-processing. + : Run forward pass. This will be called by both and . + : Update network weights; it will be called in every training iteration. +""" +import numpy as np +import torch +from .base_model import BaseModel +from . import networks + + +class TemplateModel(BaseModel): + @staticmethod + def modify_commandline_options(parser, is_train=True): + """Add new model-specific options and rewrite default values for existing options. + + Parameters: + parser -- the option parser + is_train -- if it is training phase or test phase. You can use this flag to add training-specific or test-specific options. + + Returns: + the modified parser. + """ + parser.set_defaults(dataset_mode='aligned') # You can rewrite default values for this model. For example, this model usually uses aligned dataset as its dataset. + if is_train: + parser.add_argument('--lambda_regression', type=float, default=1.0, help='weight for the regression loss') # You can define new arguments for this model. + + return parser + + def __init__(self, opt): + """Initialize this model class. + + Parameters: + opt -- training/test options + + A few things can be done here. + - (required) call the initialization function of BaseModel + - define loss function, visualization images, model names, and optimizers + """ + BaseModel.__init__(self, opt) # call the initialization method of BaseModel + # specify the training losses you want to print out. The program will call base_model.get_current_losses to plot the losses to the console and save them to the disk. + self.loss_names = ['loss_G'] + # specify the images you want to save and display. The program will call base_model.get_current_visuals to save and display these images. + self.visual_names = ['data_A', 'data_B', 'output'] + # specify the models you want to save to the disk. The program will call base_model.save_networks and base_model.load_networks to save and load networks. + # you can use opt.isTrain to specify different behaviors for training and test. For example, some networks will not be used during test, and you don't need to load them. + self.model_names = ['G'] + # define networks; you can use opt.isTrain to specify different behaviors for training and test. + self.netG = networks.define_G(opt.input_nc, opt.output_nc, opt.ngf, opt.netG, gpu_ids=self.gpu_ids) + if self.isTrain: # only defined during training time + # define your loss functions. You can use losses provided by torch.nn such as torch.nn.L1Loss. + # We also provide a GANLoss class "networks.GANLoss". self.criterionGAN = networks.GANLoss().to(self.device) + self.criterionLoss = torch.nn.L1Loss() + # define and initialize optimizers. You can define one optimizer for each network. + # If two networks are updated at the same time, you can use itertools.chain to group them. See cycle_gan_model.py for an example. + self.optimizer = torch.optim.Adam(self.netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + self.optimizers = [self.optimizer] + + # Our program will automatically call to define schedulers, load networks, and print networks + + def set_input(self, input): + """Unpack input data from the dataloader and perform necessary pre-processing steps. + + Parameters: + input: a dictionary that contains the data itself and its metadata information. + """ + AtoB = self.opt.direction == 'AtoB' # use to swap data_A and data_B + self.data_A = input['A' if AtoB else 'B'].to(self.device) # get image data A + self.data_B = input['B' if AtoB else 'A'].to(self.device) # get image data B + self.image_paths = input['A_paths' if AtoB else 'B_paths'] # get image paths + + def forward(self): + """Run forward pass. This will be called by both functions and .""" + self.output = self.netG(self.data_A) # generate output image given the input data_A + + def backward(self): + """Calculate losses, gradients, and update network weights; called in every training iteration""" + # caculate the intermediate results if necessary; here self.output has been computed during function + # calculate loss given the input and intermediate results + self.loss_G = self.criterionLoss(self.output, self.data_B) * self.opt.lambda_regression + self.loss_G.backward() # calculate gradients of network G w.r.t. loss_G + + def optimize_parameters(self): + """Update network weights; it will be called in every training iteration.""" + self.forward() # first call forward to calculate intermediate results + self.optimizer.zero_grad() # clear network G's existing gradients + self.backward() # calculate gradients for network G + self.optimizer.step() # update gradients for network G diff --git a/src/face3d/options/__init__.py b/src/face3d/options/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e7eedebe54aa70169fd25951b3034d819e396c90 --- /dev/null +++ b/src/face3d/options/__init__.py @@ -0,0 +1 @@ +"""This package options includes option modules: training options, test options, and basic options (used in both training and test).""" diff --git a/src/face3d/options/base_options.py b/src/face3d/options/base_options.py new file mode 100644 index 0000000000000000000000000000000000000000..d8f921d5a43434ae802a55a0fa3889c4b7ab9f6d --- /dev/null +++ b/src/face3d/options/base_options.py @@ -0,0 +1,169 @@ +"""This script contains base options for Deep3DFaceRecon_pytorch +""" + +import argparse +import os +from util import util +import numpy as np +import torch +import face3d.models as models +import face3d.data as data + + +class BaseOptions(): + """This class defines options used during both training and test time. + + It also implements several helper functions such as parsing, printing, and saving the options. + It also gathers additional options defined in functions in both dataset class and model class. + """ + + def __init__(self, cmd_line=None): + """Reset the class; indicates the class hasn't been initailized""" + self.initialized = False + self.cmd_line = None + if cmd_line is not None: + self.cmd_line = cmd_line.split() + + def initialize(self, parser): + """Define the common options that are used in both training and test.""" + # basic parameters + parser.add_argument('--name', type=str, default='face_recon', help='name of the experiment. It decides where to store samples and models') + parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0 0,1,2, 0,2. use -1 for CPU') + parser.add_argument('--checkpoints_dir', type=str, default='./checkpoints', help='models are saved here') + parser.add_argument('--vis_batch_nums', type=float, default=1, help='batch nums of images for visulization') + parser.add_argument('--eval_batch_nums', type=float, default=float('inf'), help='batch nums of images for evaluation') + parser.add_argument('--use_ddp', type=util.str2bool, nargs='?', const=True, default=True, help='whether use distributed data parallel') + parser.add_argument('--ddp_port', type=str, default='12355', help='ddp port') + parser.add_argument('--display_per_batch', type=util.str2bool, nargs='?', const=True, default=True, help='whether use batch to show losses') + parser.add_argument('--add_image', type=util.str2bool, nargs='?', const=True, default=True, help='whether add image to tensorboard') + parser.add_argument('--world_size', type=int, default=1, help='batch nums of images for evaluation') + + # model parameters + parser.add_argument('--model', type=str, default='facerecon', help='chooses which model to use.') + + # additional parameters + parser.add_argument('--epoch', type=str, default='latest', help='which epoch to load? set to latest to use latest cached model') + parser.add_argument('--verbose', action='store_true', help='if specified, print more debugging information') + parser.add_argument('--suffix', default='', type=str, help='customized suffix: opt.name = opt.name + suffix: e.g., {model}_{netG}_size{load_size}') + + self.initialized = True + return parser + + def gather_options(self): + """Initialize our parser with basic options(only once). + Add additional model-specific and dataset-specific options. + These options are defined in the function + in model and dataset classes. + """ + if not self.initialized: # check if it has been initialized + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = self.initialize(parser) + + # get the basic options + if self.cmd_line is None: + opt, _ = parser.parse_known_args() + else: + opt, _ = parser.parse_known_args(self.cmd_line) + + # set cuda visible devices + os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu_ids + + # modify model-related parser options + model_name = opt.model + model_option_setter = models.get_option_setter(model_name) + parser = model_option_setter(parser, self.isTrain) + if self.cmd_line is None: + opt, _ = parser.parse_known_args() # parse again with new defaults + else: + opt, _ = parser.parse_known_args(self.cmd_line) # parse again with new defaults + + # modify dataset-related parser options + if opt.dataset_mode: + dataset_name = opt.dataset_mode + dataset_option_setter = data.get_option_setter(dataset_name) + parser = dataset_option_setter(parser, self.isTrain) + + # save and return the parser + self.parser = parser + if self.cmd_line is None: + return parser.parse_args() + else: + return parser.parse_args(self.cmd_line) + + def print_options(self, opt): + """Print and save options + + It will print both current options and default values(if different). + It will save options into a text file / [checkpoints_dir] / opt.txt + """ + message = '' + message += '----------------- Options ---------------\n' + for k, v in sorted(vars(opt).items()): + comment = '' + default = self.parser.get_default(k) + if v != default: + comment = '\t[default: %s]' % str(default) + message += '{:>25}: {:<30}{}\n'.format(str(k), str(v), comment) + message += '----------------- End -------------------' + print(message) + + # save to the disk + expr_dir = os.path.join(opt.checkpoints_dir, opt.name) + util.mkdirs(expr_dir) + file_name = os.path.join(expr_dir, '{}_opt.txt'.format(opt.phase)) + try: + with open(file_name, 'wt') as opt_file: + opt_file.write(message) + opt_file.write('\n') + except PermissionError as error: + print("permission error {}".format(error)) + pass + + def parse(self): + """Parse our options, create checkpoints directory suffix, and set up gpu device.""" + opt = self.gather_options() + opt.isTrain = self.isTrain # train or test + + # process opt.suffix + if opt.suffix: + suffix = ('_' + opt.suffix.format(**vars(opt))) if opt.suffix != '' else '' + opt.name = opt.name + suffix + + + # set gpu ids + str_ids = opt.gpu_ids.split(',') + gpu_ids = [] + for str_id in str_ids: + id = int(str_id) + if id >= 0: + gpu_ids.append(id) + opt.world_size = len(gpu_ids) + # if len(opt.gpu_ids) > 0: + # torch.cuda.set_device(gpu_ids[0]) + if opt.world_size == 1: + opt.use_ddp = False + + if opt.phase != 'test': + # set continue_train automatically + if opt.pretrained_name is None: + model_dir = os.path.join(opt.checkpoints_dir, opt.name) + else: + model_dir = os.path.join(opt.checkpoints_dir, opt.pretrained_name) + if os.path.isdir(model_dir): + model_pths = [i for i in os.listdir(model_dir) if i.endswith('pth')] + if os.path.isdir(model_dir) and len(model_pths) != 0: + opt.continue_train= True + + # update the latest epoch count + if opt.continue_train: + if opt.epoch == 'latest': + epoch_counts = [int(i.split('.')[0].split('_')[-1]) for i in model_pths if 'latest' not in i] + if len(epoch_counts) != 0: + opt.epoch_count = max(epoch_counts) + 1 + else: + opt.epoch_count = int(opt.epoch) + 1 + + + self.print_options(opt) + self.opt = opt + return self.opt diff --git a/src/face3d/options/inference_options.py b/src/face3d/options/inference_options.py new file mode 100644 index 0000000000000000000000000000000000000000..c453965959ab4cfb31acbc424f994db68c3d4df5 --- /dev/null +++ b/src/face3d/options/inference_options.py @@ -0,0 +1,23 @@ +from face3d.options.base_options import BaseOptions + + +class InferenceOptions(BaseOptions): + """This class includes test options. + + It also includes shared options defined in BaseOptions. + """ + + def initialize(self, parser): + parser = BaseOptions.initialize(self, parser) # define shared options + parser.add_argument('--phase', type=str, default='test', help='train, val, test, etc') + parser.add_argument('--dataset_mode', type=str, default=None, help='chooses how datasets are loaded. [None | flist]') + + parser.add_argument('--input_dir', type=str, help='the folder of the input files') + parser.add_argument('--keypoint_dir', type=str, help='the folder of the keypoint files') + parser.add_argument('--output_dir', type=str, default='mp4', help='the output dir to save the extracted coefficients') + parser.add_argument('--save_split_files', action='store_true', help='save split files or not') + parser.add_argument('--inference_batch_size', type=int, default=8) + + # Dropout and Batchnorm has different behavior during training and test. + self.isTrain = False + return parser diff --git a/src/face3d/options/test_options.py b/src/face3d/options/test_options.py new file mode 100644 index 0000000000000000000000000000000000000000..4ff3ad142779850d1d5a1640bc00f70d34d4a862 --- /dev/null +++ b/src/face3d/options/test_options.py @@ -0,0 +1,21 @@ +"""This script contains the test options for Deep3DFaceRecon_pytorch +""" + +from .base_options import BaseOptions + + +class TestOptions(BaseOptions): + """This class includes test options. + + It also includes shared options defined in BaseOptions. + """ + + def initialize(self, parser): + parser = BaseOptions.initialize(self, parser) # define shared options + parser.add_argument('--phase', type=str, default='test', help='train, val, test, etc') + parser.add_argument('--dataset_mode', type=str, default=None, help='chooses how datasets are loaded. [None | flist]') + parser.add_argument('--img_folder', type=str, default='examples', help='folder for test images.') + + # Dropout and Batchnorm has different behavior during training and test. + self.isTrain = False + return parser diff --git a/src/face3d/options/train_options.py b/src/face3d/options/train_options.py new file mode 100644 index 0000000000000000000000000000000000000000..1337bfdd5f372b5c686a91b394a2aadbe5741f44 --- /dev/null +++ b/src/face3d/options/train_options.py @@ -0,0 +1,53 @@ +"""This script contains the training options for Deep3DFaceRecon_pytorch +""" + +from .base_options import BaseOptions +from util import util + +class TrainOptions(BaseOptions): + """This class includes training options. + + It also includes shared options defined in BaseOptions. + """ + + def initialize(self, parser): + parser = BaseOptions.initialize(self, parser) + # dataset parameters + # for train + parser.add_argument('--data_root', type=str, default='./', help='dataset root') + parser.add_argument('--flist', type=str, default='datalist/train/masks.txt', help='list of mask names of training set') + parser.add_argument('--batch_size', type=int, default=32) + parser.add_argument('--dataset_mode', type=str, default='flist', help='chooses how datasets are loaded. [None | flist]') + parser.add_argument('--serial_batches', action='store_true', help='if true, takes images in order to make batches, otherwise takes them randomly') + parser.add_argument('--num_threads', default=4, type=int, help='# threads for loading data') + parser.add_argument('--max_dataset_size', type=int, default=float("inf"), help='Maximum number of samples allowed per dataset. If the dataset directory contains more than max_dataset_size, only a subset is loaded.') + parser.add_argument('--preprocess', type=str, default='shift_scale_rot_flip', help='scaling and cropping of images at load time [shift_scale_rot_flip | shift_scale | shift | shift_rot_flip ]') + parser.add_argument('--use_aug', type=util.str2bool, nargs='?', const=True, default=True, help='whether use data augmentation') + + # for val + parser.add_argument('--flist_val', type=str, default='datalist/val/masks.txt', help='list of mask names of val set') + parser.add_argument('--batch_size_val', type=int, default=32) + + + # visualization parameters + parser.add_argument('--display_freq', type=int, default=1000, help='frequency of showing training results on screen') + parser.add_argument('--print_freq', type=int, default=100, help='frequency of showing training results on console') + + # network saving and loading parameters + parser.add_argument('--save_latest_freq', type=int, default=5000, help='frequency of saving the latest results') + parser.add_argument('--save_epoch_freq', type=int, default=1, help='frequency of saving checkpoints at the end of epochs') + parser.add_argument('--evaluation_freq', type=int, default=5000, help='evaluation freq') + parser.add_argument('--save_by_iter', action='store_true', help='whether saves model by iteration') + parser.add_argument('--continue_train', action='store_true', help='continue training: load the latest model') + parser.add_argument('--epoch_count', type=int, default=1, help='the starting epoch count, we save the model by , +, ...') + parser.add_argument('--phase', type=str, default='train', help='train, val, test, etc') + parser.add_argument('--pretrained_name', type=str, default=None, help='resume training from another checkpoint') + + # training parameters + parser.add_argument('--n_epochs', type=int, default=20, help='number of epochs with the initial learning rate') + parser.add_argument('--lr', type=float, default=0.0001, help='initial learning rate for adam') + parser.add_argument('--lr_policy', type=str, default='step', help='learning rate policy. [linear | step | plateau | cosine]') + parser.add_argument('--lr_decay_epochs', type=int, default=10, help='multiply by a gamma every lr_decay_epochs epoches') + + self.isTrain = True + return parser diff --git a/src/face3d/util/BBRegressorParam_r.mat b/src/face3d/util/BBRegressorParam_r.mat new file mode 100644 index 0000000000000000000000000000000000000000..1430a94ed2ab570a09f9d980d3585e8aaa933084 Binary files /dev/null and b/src/face3d/util/BBRegressorParam_r.mat differ diff --git a/src/face3d/util/__init__.py b/src/face3d/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..04eecb58b62f8c9d11d17606c6241d278a48b9b9 --- /dev/null +++ b/src/face3d/util/__init__.py @@ -0,0 +1,3 @@ +"""This package includes a miscellaneous collection of useful helper functions.""" +from src.face3d.util import * + diff --git a/src/face3d/util/__pycache__/__init__.cpython-310.pyc b/src/face3d/util/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b054f9520a1c5b527793dc27e05c9da340c1a9d Binary files /dev/null and b/src/face3d/util/__pycache__/__init__.cpython-310.pyc differ diff --git a/src/face3d/util/__pycache__/load_mats.cpython-310.pyc b/src/face3d/util/__pycache__/load_mats.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c4f40f83c82a564e0938ba03c75324d7977ec423 Binary files /dev/null and b/src/face3d/util/__pycache__/load_mats.cpython-310.pyc differ diff --git a/src/face3d/util/__pycache__/my_awing_arch.cpython-310.pyc b/src/face3d/util/__pycache__/my_awing_arch.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9e17bf11a5885cd2748bdd604d748c9d03306fb5 Binary files /dev/null and b/src/face3d/util/__pycache__/my_awing_arch.cpython-310.pyc differ diff --git a/src/face3d/util/__pycache__/preprocess.cpython-310.pyc b/src/face3d/util/__pycache__/preprocess.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1f1e0a4c6ba8b6bb85d1dc46ee437d522b3f547b Binary files /dev/null and b/src/face3d/util/__pycache__/preprocess.cpython-310.pyc differ diff --git a/src/face3d/util/detect_lm68.py b/src/face3d/util/detect_lm68.py new file mode 100644 index 0000000000000000000000000000000000000000..b7e40997289e17405e1fb6c408d21adce7b626ce --- /dev/null +++ b/src/face3d/util/detect_lm68.py @@ -0,0 +1,106 @@ +import os +import cv2 +import numpy as np +from scipy.io import loadmat +import tensorflow as tf +from util.preprocess import align_for_lm +from shutil import move + +mean_face = np.loadtxt('util/test_mean_face.txt') +mean_face = mean_face.reshape([68, 2]) + +def save_label(labels, save_path): + np.savetxt(save_path, labels) + +def draw_landmarks(img, landmark, save_name): + landmark = landmark + lm_img = np.zeros([img.shape[0], img.shape[1], 3]) + lm_img[:] = img.astype(np.float32) + landmark = np.round(landmark).astype(np.int32) + + for i in range(len(landmark)): + for j in range(-1, 1): + for k in range(-1, 1): + if img.shape[0] - 1 - landmark[i, 1]+j > 0 and \ + img.shape[0] - 1 - landmark[i, 1]+j < img.shape[0] and \ + landmark[i, 0]+k > 0 and \ + landmark[i, 0]+k < img.shape[1]: + lm_img[img.shape[0] - 1 - landmark[i, 1]+j, landmark[i, 0]+k, + :] = np.array([0, 0, 255]) + lm_img = lm_img.astype(np.uint8) + + cv2.imwrite(save_name, lm_img) + + +def load_data(img_name, txt_name): + return cv2.imread(img_name), np.loadtxt(txt_name) + +# create tensorflow graph for landmark detector +def load_lm_graph(graph_filename): + with tf.gfile.GFile(graph_filename, 'rb') as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + + with tf.Graph().as_default() as graph: + tf.import_graph_def(graph_def, name='net') + img_224 = graph.get_tensor_by_name('net/input_imgs:0') + output_lm = graph.get_tensor_by_name('net/lm:0') + lm_sess = tf.Session(graph=graph) + + return lm_sess,img_224,output_lm + +# landmark detection +def detect_68p(img_path,sess,input_op,output_op): + print('detecting landmarks......') + names = [i for i in sorted(os.listdir( + img_path)) if 'jpg' in i or 'png' in i or 'jpeg' in i or 'PNG' in i] + vis_path = os.path.join(img_path, 'vis') + remove_path = os.path.join(img_path, 'remove') + save_path = os.path.join(img_path, 'landmarks') + if not os.path.isdir(vis_path): + os.makedirs(vis_path) + if not os.path.isdir(remove_path): + os.makedirs(remove_path) + if not os.path.isdir(save_path): + os.makedirs(save_path) + + for i in range(0, len(names)): + name = names[i] + print('%05d' % (i), ' ', name) + full_image_name = os.path.join(img_path, name) + txt_name = '.'.join(name.split('.')[:-1]) + '.txt' + full_txt_name = os.path.join(img_path, 'detections', txt_name) # 5 facial landmark path for each image + + # if an image does not have detected 5 facial landmarks, remove it from the training list + if not os.path.isfile(full_txt_name): + move(full_image_name, os.path.join(remove_path, name)) + continue + + # load data + img, five_points = load_data(full_image_name, full_txt_name) + input_img, scale, bbox = align_for_lm(img, five_points) # align for 68 landmark detection + + # if the alignment fails, remove corresponding image from the training list + if scale == 0: + move(full_txt_name, os.path.join( + remove_path, txt_name)) + move(full_image_name, os.path.join(remove_path, name)) + continue + + # detect landmarks + input_img = np.reshape( + input_img, [1, 224, 224, 3]).astype(np.float32) + landmark = sess.run( + output_op, feed_dict={input_op: input_img}) + + # transform back to original image coordinate + landmark = landmark.reshape([68, 2]) + mean_face + landmark[:, 1] = 223 - landmark[:, 1] + landmark = landmark / scale + landmark[:, 0] = landmark[:, 0] + bbox[0] + landmark[:, 1] = landmark[:, 1] + bbox[1] + landmark[:, 1] = img.shape[0] - 1 - landmark[:, 1] + + if i % 100 == 0: + draw_landmarks(img, landmark, os.path.join(vis_path, name)) + save_label(landmark, os.path.join(save_path, txt_name)) diff --git a/src/face3d/util/generate_list.py b/src/face3d/util/generate_list.py new file mode 100644 index 0000000000000000000000000000000000000000..943d906781063c3584a7e5b5c784f8aac0694985 --- /dev/null +++ b/src/face3d/util/generate_list.py @@ -0,0 +1,34 @@ +"""This script is to generate training list files for Deep3DFaceRecon_pytorch +""" + +import os + +# save path to training data +def write_list(lms_list, imgs_list, msks_list, mode='train',save_folder='datalist', save_name=''): + save_path = os.path.join(save_folder, mode) + if not os.path.isdir(save_path): + os.makedirs(save_path) + with open(os.path.join(save_path, save_name + 'landmarks.txt'), 'w') as fd: + fd.writelines([i + '\n' for i in lms_list]) + + with open(os.path.join(save_path, save_name + 'images.txt'), 'w') as fd: + fd.writelines([i + '\n' for i in imgs_list]) + + with open(os.path.join(save_path, save_name + 'masks.txt'), 'w') as fd: + fd.writelines([i + '\n' for i in msks_list]) + +# check if the path is valid +def check_list(rlms_list, rimgs_list, rmsks_list): + lms_list, imgs_list, msks_list = [], [], [] + for i in range(len(rlms_list)): + flag = 'false' + lm_path = rlms_list[i] + im_path = rimgs_list[i] + msk_path = rmsks_list[i] + if os.path.isfile(lm_path) and os.path.isfile(im_path) and os.path.isfile(msk_path): + flag = 'true' + lms_list.append(rlms_list[i]) + imgs_list.append(rimgs_list[i]) + msks_list.append(rmsks_list[i]) + print(i, rlms_list[i], flag) + return lms_list, imgs_list, msks_list diff --git a/src/face3d/util/html.py b/src/face3d/util/html.py new file mode 100644 index 0000000000000000000000000000000000000000..cc3262a1eafda34842e4dbad47bb6ba72f0c5a68 --- /dev/null +++ b/src/face3d/util/html.py @@ -0,0 +1,86 @@ +import dominate +from dominate.tags import meta, h3, table, tr, td, p, a, img, br +import os + + +class HTML: + """This HTML class allows us to save images and write texts into a single HTML file. + + It consists of functions such as (add a text header to the HTML file), + (add a row of images to the HTML file), and (save the HTML to the disk). + It is based on Python library 'dominate', a Python library for creating and manipulating HTML documents using a DOM API. + """ + + def __init__(self, web_dir, title, refresh=0): + """Initialize the HTML classes + + Parameters: + web_dir (str) -- a directory that stores the webpage. HTML file will be created at /index.html; images will be saved at 0: + with self.doc.head: + meta(http_equiv="refresh", content=str(refresh)) + + def get_image_dir(self): + """Return the directory that stores images""" + return self.img_dir + + def add_header(self, text): + """Insert a header to the HTML file + + Parameters: + text (str) -- the header text + """ + with self.doc: + h3(text) + + def add_images(self, ims, txts, links, width=400): + """add images to the HTML file + + Parameters: + ims (str list) -- a list of image paths + txts (str list) -- a list of image names shown on the website + links (str list) -- a list of hyperref links; when you click an image, it will redirect you to a new page + """ + self.t = table(border=1, style="table-layout: fixed;") # Insert a table + self.doc.add(self.t) + with self.t: + with tr(): + for im, txt, link in zip(ims, txts, links): + with td(style="word-wrap: break-word;", halign="center", valign="top"): + with p(): + with a(href=os.path.join('images', link)): + img(style="width:%dpx" % width, src=os.path.join('images', im)) + br() + p(txt) + + def save(self): + """save the current content to the HMTL file""" + html_file = '%s/index.html' % self.web_dir + f = open(html_file, 'wt') + f.write(self.doc.render()) + f.close() + + +if __name__ == '__main__': # we show an example usage here. + html = HTML('web/', 'test_html') + html.add_header('hello world') + + ims, txts, links = [], [], [] + for n in range(4): + ims.append('image_%d.png' % n) + txts.append('text_%d' % n) + links.append('image_%d.png' % n) + html.add_images(ims, txts, links) + html.save() diff --git a/src/face3d/util/load_mats.py b/src/face3d/util/load_mats.py new file mode 100644 index 0000000000000000000000000000000000000000..f9a6fcc71de1d7dad8b0f81c67dc1c213764ff0b --- /dev/null +++ b/src/face3d/util/load_mats.py @@ -0,0 +1,120 @@ +"""This script is to load 3D face model for Deep3DFaceRecon_pytorch +""" + +import numpy as np +from PIL import Image +from scipy.io import loadmat, savemat +from array import array +import os.path as osp + +# load expression basis +def LoadExpBasis(bfm_folder='BFM'): + n_vertex = 53215 + Expbin = open(osp.join(bfm_folder, 'Exp_Pca.bin'), 'rb') + exp_dim = array('i') + exp_dim.fromfile(Expbin, 1) + expMU = array('f') + expPC = array('f') + expMU.fromfile(Expbin, 3*n_vertex) + expPC.fromfile(Expbin, 3*exp_dim[0]*n_vertex) + Expbin.close() + + expPC = np.array(expPC) + expPC = np.reshape(expPC, [exp_dim[0], -1]) + expPC = np.transpose(expPC) + + expEV = np.loadtxt(osp.join(bfm_folder, 'std_exp.txt')) + + return expPC, expEV + + +# transfer original BFM09 to our face model +def transferBFM09(bfm_folder='BFM'): + print('Transfer BFM09 to BFM_model_front......') + original_BFM = loadmat(osp.join(bfm_folder, '01_MorphableModel.mat')) + shapePC = original_BFM['shapePC'] # shape basis + shapeEV = original_BFM['shapeEV'] # corresponding eigen value + shapeMU = original_BFM['shapeMU'] # mean face + texPC = original_BFM['texPC'] # texture basis + texEV = original_BFM['texEV'] # eigen value + texMU = original_BFM['texMU'] # mean texture + + expPC, expEV = LoadExpBasis(bfm_folder) + + # transfer BFM09 to our face model + + idBase = shapePC*np.reshape(shapeEV, [-1, 199]) + idBase = idBase/1e5 # unify the scale to decimeter + idBase = idBase[:, :80] # use only first 80 basis + + exBase = expPC*np.reshape(expEV, [-1, 79]) + exBase = exBase/1e5 # unify the scale to decimeter + exBase = exBase[:, :64] # use only first 64 basis + + texBase = texPC*np.reshape(texEV, [-1, 199]) + texBase = texBase[:, :80] # use only first 80 basis + + # our face model is cropped along face landmarks and contains only 35709 vertex. + # original BFM09 contains 53490 vertex, and expression basis provided by Guo et al. contains 53215 vertex. + # thus we select corresponding vertex to get our face model. + + index_exp = loadmat(osp.join(bfm_folder, 'BFM_front_idx.mat')) + index_exp = index_exp['idx'].astype(np.int32) - 1 # starts from 0 (to 53215) + + index_shape = loadmat(osp.join(bfm_folder, 'BFM_exp_idx.mat')) + index_shape = index_shape['trimIndex'].astype( + np.int32) - 1 # starts from 0 (to 53490) + index_shape = index_shape[index_exp] + + idBase = np.reshape(idBase, [-1, 3, 80]) + idBase = idBase[index_shape, :, :] + idBase = np.reshape(idBase, [-1, 80]) + + texBase = np.reshape(texBase, [-1, 3, 80]) + texBase = texBase[index_shape, :, :] + texBase = np.reshape(texBase, [-1, 80]) + + exBase = np.reshape(exBase, [-1, 3, 64]) + exBase = exBase[index_exp, :, :] + exBase = np.reshape(exBase, [-1, 64]) + + meanshape = np.reshape(shapeMU, [-1, 3])/1e5 + meanshape = meanshape[index_shape, :] + meanshape = np.reshape(meanshape, [1, -1]) + + meantex = np.reshape(texMU, [-1, 3]) + meantex = meantex[index_shape, :] + meantex = np.reshape(meantex, [1, -1]) + + # other info contains triangles, region used for computing photometric loss, + # region used for skin texture regularization, and 68 landmarks index etc. + other_info = loadmat(osp.join(bfm_folder, 'facemodel_info.mat')) + frontmask2_idx = other_info['frontmask2_idx'] + skinmask = other_info['skinmask'] + keypoints = other_info['keypoints'] + point_buf = other_info['point_buf'] + tri = other_info['tri'] + tri_mask2 = other_info['tri_mask2'] + + # save our face model + savemat(osp.join(bfm_folder, 'BFM_model_front.mat'), {'meanshape': meanshape, 'meantex': meantex, 'idBase': idBase, 'exBase': exBase, 'texBase': texBase, + 'tri': tri, 'point_buf': point_buf, 'tri_mask2': tri_mask2, 'keypoints': keypoints, 'frontmask2_idx': frontmask2_idx, 'skinmask': skinmask}) + + +# load landmarks for standard face, which is used for image preprocessing +def load_lm3d(bfm_folder): + + Lm3D = loadmat(osp.join(bfm_folder, 'similarity_Lm3D_all.mat')) + Lm3D = Lm3D['lm'] + + # calculate 5 facial landmarks using 68 landmarks + lm_idx = np.array([31, 37, 40, 43, 46, 49, 55]) - 1 + Lm3D = np.stack([Lm3D[lm_idx[0], :], np.mean(Lm3D[lm_idx[[1, 2]], :], 0), np.mean( + Lm3D[lm_idx[[3, 4]], :], 0), Lm3D[lm_idx[5], :], Lm3D[lm_idx[6], :]], axis=0) + Lm3D = Lm3D[[1, 2, 0, 3, 4], :] + + return Lm3D + + +if __name__ == '__main__': + transferBFM09() \ No newline at end of file diff --git a/src/face3d/util/my_awing_arch.py b/src/face3d/util/my_awing_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..cd5656177dc5a1dde82ffee5d43434bc5e69c88e --- /dev/null +++ b/src/face3d/util/my_awing_arch.py @@ -0,0 +1,378 @@ +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def calculate_points(heatmaps): + # change heatmaps to landmarks + B, N, H, W = heatmaps.shape + HW = H * W + BN_range = np.arange(B * N) + + heatline = heatmaps.reshape(B, N, HW) + indexes = np.argmax(heatline, axis=2) + + preds = np.stack((indexes % W, indexes // W), axis=2) + preds = preds.astype(np.float, copy=False) + + inr = indexes.ravel() + + heatline = heatline.reshape(B * N, HW) + x_up = heatline[BN_range, inr + 1] + x_down = heatline[BN_range, inr - 1] + # y_up = heatline[BN_range, inr + W] + + if any((inr + W) >= 4096): + y_up = heatline[BN_range, 4095] + else: + y_up = heatline[BN_range, inr + W] + if any((inr - W) <= 0): + y_down = heatline[BN_range, 0] + else: + y_down = heatline[BN_range, inr - W] + + think_diff = np.sign(np.stack((x_up - x_down, y_up - y_down), axis=1)) + think_diff *= .25 + + preds += think_diff.reshape(B, N, 2) + preds += .5 + return preds + + +class AddCoordsTh(nn.Module): + + def __init__(self, x_dim=64, y_dim=64, with_r=False, with_boundary=False): + super(AddCoordsTh, self).__init__() + self.x_dim = x_dim + self.y_dim = y_dim + self.with_r = with_r + self.with_boundary = with_boundary + + def forward(self, input_tensor, heatmap=None): + """ + input_tensor: (batch, c, x_dim, y_dim) + """ + batch_size_tensor = input_tensor.shape[0] + + xx_ones = torch.ones([1, self.y_dim], dtype=torch.int32, device=input_tensor.device) + xx_ones = xx_ones.unsqueeze(-1) + + xx_range = torch.arange(self.x_dim, dtype=torch.int32, device=input_tensor.device).unsqueeze(0) + xx_range = xx_range.unsqueeze(1) + + xx_channel = torch.matmul(xx_ones.float(), xx_range.float()) + xx_channel = xx_channel.unsqueeze(-1) + + yy_ones = torch.ones([1, self.x_dim], dtype=torch.int32, device=input_tensor.device) + yy_ones = yy_ones.unsqueeze(1) + + yy_range = torch.arange(self.y_dim, dtype=torch.int32, device=input_tensor.device).unsqueeze(0) + yy_range = yy_range.unsqueeze(-1) + + yy_channel = torch.matmul(yy_range.float(), yy_ones.float()) + yy_channel = yy_channel.unsqueeze(-1) + + xx_channel = xx_channel.permute(0, 3, 2, 1) + yy_channel = yy_channel.permute(0, 3, 2, 1) + + xx_channel = xx_channel / (self.x_dim - 1) + yy_channel = yy_channel / (self.y_dim - 1) + + xx_channel = xx_channel * 2 - 1 + yy_channel = yy_channel * 2 - 1 + + xx_channel = xx_channel.repeat(batch_size_tensor, 1, 1, 1) + yy_channel = yy_channel.repeat(batch_size_tensor, 1, 1, 1) + + if self.with_boundary and heatmap is not None: + boundary_channel = torch.clamp(heatmap[:, -1:, :, :], 0.0, 1.0) + + zero_tensor = torch.zeros_like(xx_channel) + xx_boundary_channel = torch.where(boundary_channel > 0.05, xx_channel, zero_tensor) + yy_boundary_channel = torch.where(boundary_channel > 0.05, yy_channel, zero_tensor) + if self.with_boundary and heatmap is not None: + xx_boundary_channel = xx_boundary_channel.to(input_tensor.device) + yy_boundary_channel = yy_boundary_channel.to(input_tensor.device) + ret = torch.cat([input_tensor, xx_channel, yy_channel], dim=1) + + if self.with_r: + rr = torch.sqrt(torch.pow(xx_channel, 2) + torch.pow(yy_channel, 2)) + rr = rr / torch.max(rr) + ret = torch.cat([ret, rr], dim=1) + + if self.with_boundary and heatmap is not None: + ret = torch.cat([ret, xx_boundary_channel, yy_boundary_channel], dim=1) + return ret + + +class CoordConvTh(nn.Module): + """CoordConv layer as in the paper.""" + + def __init__(self, x_dim, y_dim, with_r, with_boundary, in_channels, first_one=False, *args, **kwargs): + super(CoordConvTh, self).__init__() + self.addcoords = AddCoordsTh(x_dim=x_dim, y_dim=y_dim, with_r=with_r, with_boundary=with_boundary) + in_channels += 2 + if with_r: + in_channels += 1 + if with_boundary and not first_one: + in_channels += 2 + self.conv = nn.Conv2d(in_channels=in_channels, *args, **kwargs) + + def forward(self, input_tensor, heatmap=None): + ret = self.addcoords(input_tensor, heatmap) + last_channel = ret[:, -2:, :, :] + ret = self.conv(ret) + return ret, last_channel + + +def conv3x3(in_planes, out_planes, strd=1, padding=1, bias=False, dilation=1): + '3x3 convolution with padding' + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=strd, padding=padding, bias=bias, dilation=dilation) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + # self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + # self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.relu(out) + + out = self.conv2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ConvBlock(nn.Module): + + def __init__(self, in_planes, out_planes): + super(ConvBlock, self).__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.conv1 = conv3x3(in_planes, int(out_planes / 2)) + self.bn2 = nn.BatchNorm2d(int(out_planes / 2)) + self.conv2 = conv3x3(int(out_planes / 2), int(out_planes / 4), padding=1, dilation=1) + self.bn3 = nn.BatchNorm2d(int(out_planes / 4)) + self.conv3 = conv3x3(int(out_planes / 4), int(out_planes / 4), padding=1, dilation=1) + + if in_planes != out_planes: + self.downsample = nn.Sequential( + nn.BatchNorm2d(in_planes), + nn.ReLU(True), + nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, bias=False), + ) + else: + self.downsample = None + + def forward(self, x): + residual = x + + out1 = self.bn1(x) + out1 = F.relu(out1, True) + out1 = self.conv1(out1) + + out2 = self.bn2(out1) + out2 = F.relu(out2, True) + out2 = self.conv2(out2) + + out3 = self.bn3(out2) + out3 = F.relu(out3, True) + out3 = self.conv3(out3) + + out3 = torch.cat((out1, out2, out3), 1) + + if self.downsample is not None: + residual = self.downsample(residual) + + out3 += residual + + return out3 + + +class HourGlass(nn.Module): + + def __init__(self, num_modules, depth, num_features, first_one=False): + super(HourGlass, self).__init__() + self.num_modules = num_modules + self.depth = depth + self.features = num_features + self.coordconv = CoordConvTh( + x_dim=64, + y_dim=64, + with_r=True, + with_boundary=True, + in_channels=256, + first_one=first_one, + out_channels=256, + kernel_size=1, + stride=1, + padding=0) + self._generate_network(self.depth) + + def _generate_network(self, level): + self.add_module('b1_' + str(level), ConvBlock(256, 256)) + + self.add_module('b2_' + str(level), ConvBlock(256, 256)) + + if level > 1: + self._generate_network(level - 1) + else: + self.add_module('b2_plus_' + str(level), ConvBlock(256, 256)) + + self.add_module('b3_' + str(level), ConvBlock(256, 256)) + + def _forward(self, level, inp): + # Upper branch + up1 = inp + up1 = self._modules['b1_' + str(level)](up1) + + # Lower branch + low1 = F.avg_pool2d(inp, 2, stride=2) + low1 = self._modules['b2_' + str(level)](low1) + + if level > 1: + low2 = self._forward(level - 1, low1) + else: + low2 = low1 + low2 = self._modules['b2_plus_' + str(level)](low2) + + low3 = low2 + low3 = self._modules['b3_' + str(level)](low3) + + up2 = F.interpolate(low3, scale_factor=2, mode='nearest') + + return up1 + up2 + + def forward(self, x, heatmap): + x, last_channel = self.coordconv(x, heatmap) + return self._forward(self.depth, x), last_channel + + +class FAN(nn.Module): + + def __init__(self, num_modules=1, end_relu=False, gray_scale=False, num_landmarks=68, device='cuda'): + super(FAN, self).__init__() + self.device = device + self.num_modules = num_modules + self.gray_scale = gray_scale + self.end_relu = end_relu + self.num_landmarks = num_landmarks + + # Base part + if self.gray_scale: + self.conv1 = CoordConvTh( + x_dim=256, + y_dim=256, + with_r=True, + with_boundary=False, + in_channels=3, + out_channels=64, + kernel_size=7, + stride=2, + padding=3) + else: + self.conv1 = CoordConvTh( + x_dim=256, + y_dim=256, + with_r=True, + with_boundary=False, + in_channels=3, + out_channels=64, + kernel_size=7, + stride=2, + padding=3) + self.bn1 = nn.BatchNorm2d(64) + self.conv2 = ConvBlock(64, 128) + self.conv3 = ConvBlock(128, 128) + self.conv4 = ConvBlock(128, 256) + + # Stacking part + for hg_module in range(self.num_modules): + if hg_module == 0: + first_one = True + else: + first_one = False + self.add_module('m' + str(hg_module), HourGlass(1, 4, 256, first_one)) + self.add_module('top_m_' + str(hg_module), ConvBlock(256, 256)) + self.add_module('conv_last' + str(hg_module), nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0)) + self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256)) + self.add_module('l' + str(hg_module), nn.Conv2d(256, num_landmarks + 1, kernel_size=1, stride=1, padding=0)) + + if hg_module < self.num_modules - 1: + self.add_module('bl' + str(hg_module), nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0)) + self.add_module('al' + str(hg_module), + nn.Conv2d(num_landmarks + 1, 256, kernel_size=1, stride=1, padding=0)) + + def forward(self, x): + x, _ = self.conv1(x) + x = F.relu(self.bn1(x), True) + # x = F.relu(self.bn1(self.conv1(x)), True) + x = F.avg_pool2d(self.conv2(x), 2, stride=2) + x = self.conv3(x) + x = self.conv4(x) + + previous = x + + outputs = [] + boundary_channels = [] + tmp_out = None + for i in range(self.num_modules): + hg, boundary_channel = self._modules['m' + str(i)](previous, tmp_out) + + ll = hg + ll = self._modules['top_m_' + str(i)](ll) + + ll = F.relu(self._modules['bn_end' + str(i)](self._modules['conv_last' + str(i)](ll)), True) + + # Predict heatmaps + tmp_out = self._modules['l' + str(i)](ll) + if self.end_relu: + tmp_out = F.relu(tmp_out) # HACK: Added relu + outputs.append(tmp_out) + boundary_channels.append(boundary_channel) + + if i < self.num_modules - 1: + ll = self._modules['bl' + str(i)](ll) + tmp_out_ = self._modules['al' + str(i)](tmp_out) + previous = previous + ll + tmp_out_ + + return outputs, boundary_channels + + def get_landmarks(self, img): + H, W, _ = img.shape + offset = W / 64, H / 64, 0, 0 + + img = cv2.resize(img, (256, 256)) + inp = img[..., ::-1] + inp = torch.from_numpy(np.ascontiguousarray(inp.transpose((2, 0, 1)))).float() + inp = inp.to(self.device) + inp.div_(255.0).unsqueeze_(0) + + outputs, _ = self.forward(inp) + out = outputs[-1][:, :-1, :, :] + heatmaps = out.detach().cpu().numpy() + + pred = calculate_points(heatmaps).reshape(-1, 2) + + pred *= offset[:2] + pred += offset[-2:] + + return pred diff --git a/src/face3d/util/nvdiffrast.py b/src/face3d/util/nvdiffrast.py new file mode 100644 index 0000000000000000000000000000000000000000..f3245859c650afbfe841a66b74cddefaf28820d9 --- /dev/null +++ b/src/face3d/util/nvdiffrast.py @@ -0,0 +1,126 @@ +"""This script is the differentiable renderer for Deep3DFaceRecon_pytorch + Attention, antialiasing step is missing in current version. +""" +import pytorch3d.ops +import torch +import torch.nn.functional as F +import kornia +from kornia.geometry.camera import pixel2cam +import numpy as np +from typing import List +from scipy.io import loadmat +from torch import nn + +from pytorch3d.structures import Meshes +from pytorch3d.renderer import ( + look_at_view_transform, + FoVPerspectiveCameras, + DirectionalLights, + RasterizationSettings, + MeshRenderer, + MeshRasterizer, + SoftPhongShader, + TexturesUV, +) + +# def ndc_projection(x=0.1, n=1.0, f=50.0): +# return np.array([[n/x, 0, 0, 0], +# [ 0, n/-x, 0, 0], +# [ 0, 0, -(f+n)/(f-n), -(2*f*n)/(f-n)], +# [ 0, 0, -1, 0]]).astype(np.float32) + +class MeshRenderer(nn.Module): + def __init__(self, + rasterize_fov, + znear=0.1, + zfar=10, + rasterize_size=224): + super(MeshRenderer, self).__init__() + + # x = np.tan(np.deg2rad(rasterize_fov * 0.5)) * znear + # self.ndc_proj = torch.tensor(ndc_projection(x=x, n=znear, f=zfar)).matmul( + # torch.diag(torch.tensor([1., -1, -1, 1]))) + self.rasterize_size = rasterize_size + self.fov = rasterize_fov + self.znear = znear + self.zfar = zfar + + self.rasterizer = None + + def forward(self, vertex, tri, feat=None): + """ + Return: + mask -- torch.tensor, size (B, 1, H, W) + depth -- torch.tensor, size (B, 1, H, W) + features(optional) -- torch.tensor, size (B, C, H, W) if feat is not None + + Parameters: + vertex -- torch.tensor, size (B, N, 3) + tri -- torch.tensor, size (B, M, 3) or (M, 3), triangles + feat(optional) -- torch.tensor, size (B, N ,C), features + """ + device = vertex.device + rsize = int(self.rasterize_size) + # ndc_proj = self.ndc_proj.to(device) + # trans to homogeneous coordinates of 3d vertices, the direction of y is the same as v + if vertex.shape[-1] == 3: + vertex = torch.cat([vertex, torch.ones([*vertex.shape[:2], 1]).to(device)], dim=-1) + vertex[..., 0] = -vertex[..., 0] + + + # vertex_ndc = vertex @ ndc_proj.t() + if self.rasterizer is None: + self.rasterizer = MeshRasterizer() + print("create rasterizer on device cuda:%d"%device.index) + + # ranges = None + # if isinstance(tri, List) or len(tri.shape) == 3: + # vum = vertex_ndc.shape[1] + # fnum = torch.tensor([f.shape[0] for f in tri]).unsqueeze(1).to(device) + # fstartidx = torch.cumsum(fnum, dim=0) - fnum + # ranges = torch.cat([fstartidx, fnum], axis=1).type(torch.int32).cpu() + # for i in range(tri.shape[0]): + # tri[i] = tri[i] + i*vum + # vertex_ndc = torch.cat(vertex_ndc, dim=0) + # tri = torch.cat(tri, dim=0) + + # for range_mode vetex: [B*N, 4], tri: [B*M, 3], for instance_mode vetex: [B, N, 4], tri: [M, 3] + tri = tri.type(torch.int32).contiguous() + + # rasterize + cameras = FoVPerspectiveCameras( + device=device, + fov=self.fov, + znear=self.znear, + zfar=self.zfar, + ) + + raster_settings = RasterizationSettings( + image_size=rsize + ) + + # print(vertex.shape, tri.shape) + mesh = Meshes(vertex.contiguous()[...,:3], tri.unsqueeze(0).repeat((vertex.shape[0],1,1))) + + fragments = self.rasterizer(mesh, cameras = cameras, raster_settings = raster_settings) + rast_out = fragments.pix_to_face.squeeze(-1) + depth = fragments.zbuf + + # render depth + depth = depth.permute(0, 3, 1, 2) + mask = (rast_out > 0).float().unsqueeze(1) + depth = mask * depth + + + image = None + if feat is not None: + attributes = feat.reshape(-1,3)[mesh.faces_packed()] + image = pytorch3d.ops.interpolate_face_attributes(fragments.pix_to_face, + fragments.bary_coords, + attributes) + # print(image.shape) + image = image.squeeze(-2).permute(0, 3, 1, 2) + image = mask * image + + return mask, depth, image + diff --git a/src/face3d/util/preprocess.py b/src/face3d/util/preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..b77a3a4058c208e5ba8cb1cfbb563954a5f7a3e2 --- /dev/null +++ b/src/face3d/util/preprocess.py @@ -0,0 +1,103 @@ +"""This script contains the image preprocessing code for Deep3DFaceRecon_pytorch +""" + +import numpy as np +from scipy.io import loadmat +from PIL import Image +import cv2 +import os +from skimage import transform as trans +import torch +import warnings +warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) +warnings.filterwarnings("ignore", category=FutureWarning) + + +# calculating least square problem for image alignment +def POS(xp, x): + npts = xp.shape[1] + + A = np.zeros([2*npts, 8]) + + A[0:2*npts-1:2, 0:3] = x.transpose() + A[0:2*npts-1:2, 3] = 1 + + A[1:2*npts:2, 4:7] = x.transpose() + A[1:2*npts:2, 7] = 1 + + b = np.reshape(xp.transpose(), [2*npts, 1]) + + k, _, _, _ = np.linalg.lstsq(A, b) + + R1 = k[0:3] + R2 = k[4:7] + sTx = k[3] + sTy = k[7] + s = (np.linalg.norm(R1) + np.linalg.norm(R2))/2 + t = np.stack([sTx, sTy], axis=0) + + return t, s + +# resize and crop images for face reconstruction +def resize_n_crop_img(img, lm, t, s, target_size=224., mask=None): + w0, h0 = img.size + w = (w0*s).astype(np.int32) + h = (h0*s).astype(np.int32) + left = (w/2 - target_size/2 + float((t[0] - w0/2)*s)).astype(np.int32) + right = left + target_size + up = (h/2 - target_size/2 + float((h0/2 - t[1])*s)).astype(np.int32) + below = up + target_size + + img = img.resize((w, h), resample=Image.BICUBIC) + img = img.crop((left, up, right, below)) + + if mask is not None: + mask = mask.resize((w, h), resample=Image.BICUBIC) + mask = mask.crop((left, up, right, below)) + + lm = np.stack([lm[:, 0] - t[0] + w0/2, lm[:, 1] - + t[1] + h0/2], axis=1)*s + lm = lm - np.reshape( + np.array([(w/2 - target_size/2), (h/2-target_size/2)]), [1, 2]) + + return img, lm, mask + +# utils for face reconstruction +def extract_5p(lm): + lm_idx = np.array([31, 37, 40, 43, 46, 49, 55]) - 1 + lm5p = np.stack([lm[lm_idx[0], :], np.mean(lm[lm_idx[[1, 2]], :], 0), np.mean( + lm[lm_idx[[3, 4]], :], 0), lm[lm_idx[5], :], lm[lm_idx[6], :]], axis=0) + lm5p = lm5p[[1, 2, 0, 3, 4], :] + return lm5p + +# utils for face reconstruction +def align_img(img, lm, lm3D, mask=None, target_size=224., rescale_factor=102.): + """ + Return: + transparams --numpy.array (raw_W, raw_H, scale, tx, ty) + img_new --PIL.Image (target_size, target_size, 3) + lm_new --numpy.array (68, 2), y direction is opposite to v direction + mask_new --PIL.Image (target_size, target_size) + + Parameters: + img --PIL.Image (raw_H, raw_W, 3) + lm --numpy.array (68, 2), y direction is opposite to v direction + lm3D --numpy.array (5, 3) + mask --PIL.Image (raw_H, raw_W, 3) + """ + + w0, h0 = img.size + if lm.shape[0] != 5: + lm5p = extract_5p(lm) + else: + lm5p = lm + + # calculate translation and scale factors using 5 facial landmarks and standard landmarks of a 3D face + t, s = POS(lm5p.transpose(), lm3D.transpose()) + s = rescale_factor/s + + # processing the image + img_new, lm_new, mask_new = resize_n_crop_img(img, lm, t, s, target_size=target_size, mask=mask) + trans_params = np.array([w0, h0, s, t[0], t[1]]) + + return trans_params, img_new, lm_new, mask_new diff --git a/src/face3d/util/skin_mask.py b/src/face3d/util/skin_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..a8a74e4c3b40d13b0258b83a12f56321a85bb179 --- /dev/null +++ b/src/face3d/util/skin_mask.py @@ -0,0 +1,125 @@ +"""This script is to generate skin attention mask for Deep3DFaceRecon_pytorch +""" + +import math +import numpy as np +import os +import cv2 + +class GMM: + def __init__(self, dim, num, w, mu, cov, cov_det, cov_inv): + self.dim = dim # feature dimension + self.num = num # number of Gaussian components + self.w = w # weights of Gaussian components (a list of scalars) + self.mu= mu # mean of Gaussian components (a list of 1xdim vectors) + self.cov = cov # covariance matrix of Gaussian components (a list of dimxdim matrices) + self.cov_det = cov_det # pre-computed determinet of covariance matrices (a list of scalars) + self.cov_inv = cov_inv # pre-computed inverse covariance matrices (a list of dimxdim matrices) + + self.factor = [0]*num + for i in range(self.num): + self.factor[i] = (2*math.pi)**(self.dim/2) * self.cov_det[i]**0.5 + + def likelihood(self, data): + assert(data.shape[1] == self.dim) + N = data.shape[0] + lh = np.zeros(N) + + for i in range(self.num): + data_ = data - self.mu[i] + + tmp = np.matmul(data_,self.cov_inv[i]) * data_ + tmp = np.sum(tmp,axis=1) + power = -0.5 * tmp + + p = np.array([math.exp(power[j]) for j in range(N)]) + p = p/self.factor[i] + lh += p*self.w[i] + + return lh + + +def _rgb2ycbcr(rgb): + m = np.array([[65.481, 128.553, 24.966], + [-37.797, -74.203, 112], + [112, -93.786, -18.214]]) + shape = rgb.shape + rgb = rgb.reshape((shape[0] * shape[1], 3)) + ycbcr = np.dot(rgb, m.transpose() / 255.) + ycbcr[:, 0] += 16. + ycbcr[:, 1:] += 128. + return ycbcr.reshape(shape) + + +def _bgr2ycbcr(bgr): + rgb = bgr[..., ::-1] + return _rgb2ycbcr(rgb) + + +gmm_skin_w = [0.24063933, 0.16365987, 0.26034665, 0.33535415] +gmm_skin_mu = [np.array([113.71862, 103.39613, 164.08226]), + np.array([150.19858, 105.18467, 155.51428]), + np.array([183.92976, 107.62468, 152.71820]), + np.array([114.90524, 113.59782, 151.38217])] +gmm_skin_cov_det = [5692842.5, 5851930.5, 2329131., 1585971.] +gmm_skin_cov_inv = [np.array([[0.0019472069, 0.0020450759, -0.00060243998],[0.0020450759, 0.017700525, 0.0051420014],[-0.00060243998, 0.0051420014, 0.0081308950]]), + np.array([[0.0027110141, 0.0011036990, 0.0023122299],[0.0011036990, 0.010707724, 0.010742856],[0.0023122299, 0.010742856, 0.017481629]]), + np.array([[0.0048026871, 0.00022935172, 0.0077668377],[0.00022935172, 0.011729696, 0.0081661865],[0.0077668377, 0.0081661865, 0.025374353]]), + np.array([[0.0011989699, 0.0022453172, -0.0010748957],[0.0022453172, 0.047758564, 0.020332102],[-0.0010748957, 0.020332102, 0.024502251]])] + +gmm_skin = GMM(3, 4, gmm_skin_w, gmm_skin_mu, [], gmm_skin_cov_det, gmm_skin_cov_inv) + +gmm_nonskin_w = [0.12791070, 0.31130761, 0.34245777, 0.21832393] +gmm_nonskin_mu = [np.array([99.200851, 112.07533, 140.20602]), + np.array([110.91392, 125.52969, 130.19237]), + np.array([129.75864, 129.96107, 126.96808]), + np.array([112.29587, 128.85121, 129.05431])] +gmm_nonskin_cov_det = [458703648., 6466488., 90611376., 133097.63] +gmm_nonskin_cov_inv = [np.array([[0.00085371657, 0.00071197288, 0.00023958916],[0.00071197288, 0.0025935620, 0.00076557708],[0.00023958916, 0.00076557708, 0.0015042332]]), + np.array([[0.00024650150, 0.00045542428, 0.00015019422],[0.00045542428, 0.026412144, 0.018419769],[0.00015019422, 0.018419769, 0.037497383]]), + np.array([[0.00037054974, 0.00038146760, 0.00040408765],[0.00038146760, 0.0085505722, 0.0079136286],[0.00040408765, 0.0079136286, 0.010982352]]), + np.array([[0.00013709733, 0.00051228428, 0.00012777430],[0.00051228428, 0.28237113, 0.10528370],[0.00012777430, 0.10528370, 0.23468947]])] + +gmm_nonskin = GMM(3, 4, gmm_nonskin_w, gmm_nonskin_mu, [], gmm_nonskin_cov_det, gmm_nonskin_cov_inv) + +prior_skin = 0.8 +prior_nonskin = 1 - prior_skin + + +# calculate skin attention mask +def skinmask(imbgr): + im = _bgr2ycbcr(imbgr) + + data = im.reshape((-1,3)) + + lh_skin = gmm_skin.likelihood(data) + lh_nonskin = gmm_nonskin.likelihood(data) + + tmp1 = prior_skin * lh_skin + tmp2 = prior_nonskin * lh_nonskin + post_skin = tmp1 / (tmp1+tmp2) # posterior probability + + post_skin = post_skin.reshape((im.shape[0],im.shape[1])) + + post_skin = np.round(post_skin*255) + post_skin = post_skin.astype(np.uint8) + post_skin = np.tile(np.expand_dims(post_skin,2),[1,1,3]) # reshape to H*W*3 + + return post_skin + + +def get_skin_mask(img_path): + print('generating skin masks......') + names = [i for i in sorted(os.listdir( + img_path)) if 'jpg' in i or 'png' in i or 'jpeg' in i or 'PNG' in i] + save_path = os.path.join(img_path, 'mask') + if not os.path.isdir(save_path): + os.makedirs(save_path) + + for i in range(0, len(names)): + name = names[i] + print('%05d' % (i), ' ', name) + full_image_name = os.path.join(img_path, name) + img = cv2.imread(full_image_name).astype(np.float32) + skin_img = skinmask(img) + cv2.imwrite(os.path.join(save_path, name), skin_img.astype(np.uint8)) diff --git a/src/face3d/util/test_mean_face.txt b/src/face3d/util/test_mean_face.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a46d4db7699ffed8f898fcee64099631509946d --- /dev/null +++ b/src/face3d/util/test_mean_face.txt @@ -0,0 +1,136 @@ +-5.228591537475585938e+01 +2.078247070312500000e-01 +-5.064269638061523438e+01 +-1.315765380859375000e+01 +-4.952939224243164062e+01 +-2.592591094970703125e+01 +-4.793047332763671875e+01 +-3.832135772705078125e+01 +-4.512159729003906250e+01 +-5.059623336791992188e+01 +-3.917720794677734375e+01 +-6.043736648559570312e+01 +-2.929953765869140625e+01 +-6.861183166503906250e+01 +-1.719801330566406250e+01 +-7.572736358642578125e+01 +-1.961936950683593750e+00 +-7.862001037597656250e+01 +1.467941284179687500e+01 +-7.607844543457031250e+01 +2.744073486328125000e+01 +-6.915261840820312500e+01 +3.855677795410156250e+01 +-5.950350570678710938e+01 +4.478240966796875000e+01 +-4.867547225952148438e+01 +4.714337158203125000e+01 +-3.800830078125000000e+01 +4.940315246582031250e+01 +-2.496297454833984375e+01 +5.117234802246093750e+01 +-1.241538238525390625e+01 +5.190507507324218750e+01 +8.244247436523437500e-01 +-4.150688934326171875e+01 +2.386329650878906250e+01 +-3.570307159423828125e+01 +3.017010498046875000e+01 +-2.790358734130859375e+01 +3.212951660156250000e+01 +-1.941773223876953125e+01 +3.156523132324218750e+01 +-1.138106536865234375e+01 +2.841992187500000000e+01 +5.993263244628906250e+00 +2.895182800292968750e+01 +1.343590545654296875e+01 +3.189880371093750000e+01 +2.203153991699218750e+01 +3.302221679687500000e+01 +2.992478942871093750e+01 +3.099150085449218750e+01 +3.628388977050781250e+01 +2.765748596191406250e+01 +-1.933914184570312500e+00 +1.405374145507812500e+01 +-2.153038024902343750e+00 +5.772636413574218750e+00 +-2.270050048828125000e+00 +-2.121643066406250000e+00 +-2.218330383300781250e+00 +-1.068978118896484375e+01 +-1.187252044677734375e+01 +-1.997912597656250000e+01 +-6.879402160644531250e+00 +-2.143579864501953125e+01 +-1.227821350097656250e+00 +-2.193494415283203125e+01 +4.623237609863281250e+00 +-2.152721405029296875e+01 +9.721397399902343750e+00 +-1.953671264648437500e+01 +-3.648714447021484375e+01 +9.811126708984375000e+00 +-3.130242919921875000e+01 +1.422447967529296875e+01 +-2.212834930419921875e+01 +1.493019866943359375e+01 +-1.500880432128906250e+01 +1.073588562011718750e+01 +-2.095037078857421875e+01 +9.054298400878906250e+00 +-3.050099182128906250e+01 +8.704177856445312500e+00 +1.173237609863281250e+01 +1.054329681396484375e+01 +1.856353759765625000e+01 +1.535009765625000000e+01 +2.893331909179687500e+01 +1.451992797851562500e+01 +3.452944946289062500e+01 +1.065280151367187500e+01 +2.875990295410156250e+01 +8.654792785644531250e+00 +1.942100524902343750e+01 +9.422447204589843750e+00 +-2.204488372802734375e+01 +-3.983994293212890625e+01 +-1.324458312988281250e+01 +-3.467377471923828125e+01 +-6.749649047851562500e+00 +-3.092894744873046875e+01 +-9.183349609375000000e-01 +-3.196458435058593750e+01 +4.220649719238281250e+00 +-3.090406036376953125e+01 +1.089889526367187500e+01 +-3.497008514404296875e+01 +1.874589538574218750e+01 +-4.065438079833984375e+01 +1.124106597900390625e+01 +-4.438417816162109375e+01 +5.181709289550781250e+00 +-4.649170684814453125e+01 +-1.158607482910156250e+00 +-4.680406951904296875e+01 +-7.918922424316406250e+00 +-4.671575164794921875e+01 +-1.452505493164062500e+01 +-4.416526031494140625e+01 +-2.005007171630859375e+01 +-3.997841644287109375e+01 +-1.054919433593750000e+01 +-3.849683380126953125e+01 +-1.051826477050781250e+00 +-3.794863128662109375e+01 +6.412681579589843750e+00 +-3.804645538330078125e+01 +1.627674865722656250e+01 +-4.039697265625000000e+01 +6.373878479003906250e+00 +-4.087213897705078125e+01 +-8.551712036132812500e-01 +-4.157129669189453125e+01 +-1.014953613281250000e+01 +-4.128469085693359375e+01 diff --git a/src/face3d/util/util.py b/src/face3d/util/util.py new file mode 100644 index 0000000000000000000000000000000000000000..0d689ca138fc0fbf5bec794511ea0f9e638f9ea9 --- /dev/null +++ b/src/face3d/util/util.py @@ -0,0 +1,208 @@ +"""This script contains basic utilities for Deep3DFaceRecon_pytorch +""" +from __future__ import print_function +import numpy as np +import torch +from PIL import Image +import os +import importlib +import argparse +from argparse import Namespace +import torchvision + + +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + +def copyconf(default_opt, **kwargs): + conf = Namespace(**vars(default_opt)) + for key in kwargs: + setattr(conf, key, kwargs[key]) + return conf + +def genvalconf(train_opt, **kwargs): + conf = Namespace(**vars(train_opt)) + attr_dict = train_opt.__dict__ + for key, value in attr_dict.items(): + if 'val' in key and key.split('_')[0] in attr_dict: + setattr(conf, key.split('_')[0], value) + + for key in kwargs: + setattr(conf, key, kwargs[key]) + + return conf + +def find_class_in_module(target_cls_name, module): + target_cls_name = target_cls_name.replace('_', '').lower() + clslib = importlib.import_module(module) + cls = None + for name, clsobj in clslib.__dict__.items(): + if name.lower() == target_cls_name: + cls = clsobj + + assert cls is not None, "In %s, there should be a class whose name matches %s in lowercase without underscore(_)" % (module, target_cls_name) + + return cls + + +def tensor2im(input_image, imtype=np.uint8): + """"Converts a Tensor array into a numpy image array. + + Parameters: + input_image (tensor) -- the input image tensor array, range(0, 1) + imtype (type) -- the desired type of the converted numpy array + """ + if not isinstance(input_image, np.ndarray): + if isinstance(input_image, torch.Tensor): # get the data from a variable + image_tensor = input_image.data + else: + return input_image + image_numpy = image_tensor.clamp(0.0, 1.0).cpu().float().numpy() # convert it into a numpy array + if image_numpy.shape[0] == 1: # grayscale to RGB + image_numpy = np.tile(image_numpy, (3, 1, 1)) + image_numpy = np.transpose(image_numpy, (1, 2, 0)) * 255.0 # post-processing: tranpose and scaling + else: # if it is a numpy array, do nothing + image_numpy = input_image + return image_numpy.astype(imtype) + + +def diagnose_network(net, name='network'): + """Calculate and print the mean of average absolute(gradients) + + Parameters: + net (torch network) -- Torch network + name (str) -- the name of the network + """ + mean = 0.0 + count = 0 + for param in net.parameters(): + if param.grad is not None: + mean += torch.mean(torch.abs(param.grad.data)) + count += 1 + if count > 0: + mean = mean / count + print(name) + print(mean) + + +def save_image(image_numpy, image_path, aspect_ratio=1.0): + """Save a numpy image to the disk + + Parameters: + image_numpy (numpy array) -- input numpy array + image_path (str) -- the path of the image + """ + + image_pil = Image.fromarray(image_numpy) + h, w, _ = image_numpy.shape + + if aspect_ratio is None: + pass + elif aspect_ratio > 1.0: + image_pil = image_pil.resize((h, int(w * aspect_ratio)), Image.BICUBIC) + elif aspect_ratio < 1.0: + image_pil = image_pil.resize((int(h / aspect_ratio), w), Image.BICUBIC) + image_pil.save(image_path) + + +def print_numpy(x, val=True, shp=False): + """Print the mean, min, max, median, std, and size of a numpy array + + Parameters: + val (bool) -- if print the values of the numpy array + shp (bool) -- if print the shape of the numpy array + """ + x = x.astype(np.float64) + if shp: + print('shape,', x.shape) + if val: + x = x.flatten() + print('mean = %3.3f, min = %3.3f, max = %3.3f, median = %3.3f, std=%3.3f' % ( + np.mean(x), np.min(x), np.max(x), np.median(x), np.std(x))) + + +def mkdirs(paths): + """create empty directories if they don't exist + + Parameters: + paths (str list) -- a list of directory paths + """ + if isinstance(paths, list) and not isinstance(paths, str): + for path in paths: + mkdir(path) + else: + mkdir(paths) + + +def mkdir(path): + """create a single empty directory if it didn't exist + + Parameters: + path (str) -- a single directory path + """ + if not os.path.exists(path): + os.makedirs(path) + + +def correct_resize_label(t, size): + device = t.device + t = t.detach().cpu() + resized = [] + for i in range(t.size(0)): + one_t = t[i, :1] + one_np = np.transpose(one_t.numpy().astype(np.uint8), (1, 2, 0)) + one_np = one_np[:, :, 0] + one_image = Image.fromarray(one_np).resize(size, Image.NEAREST) + resized_t = torch.from_numpy(np.array(one_image)).long() + resized.append(resized_t) + return torch.stack(resized, dim=0).to(device) + + +def correct_resize(t, size, mode=Image.BICUBIC): + device = t.device + t = t.detach().cpu() + resized = [] + for i in range(t.size(0)): + one_t = t[i:i + 1] + one_image = Image.fromarray(tensor2im(one_t)).resize(size, Image.BICUBIC) + resized_t = torchvision.transforms.functional.to_tensor(one_image) * 2 - 1.0 + resized.append(resized_t) + return torch.stack(resized, dim=0).to(device) + +def draw_landmarks(img, landmark, color='r', step=2): + """ + Return: + img -- numpy.array, (B, H, W, 3) img with landmark, RGB order, range (0, 255) + + + Parameters: + img -- numpy.array, (B, H, W, 3), RGB order, range (0, 255) + landmark -- numpy.array, (B, 68, 2), y direction is opposite to v direction + color -- str, 'r' or 'b' (red or blue) + """ + if color =='r': + c = np.array([255., 0, 0]) + else: + c = np.array([0, 0, 255.]) + + _, H, W, _ = img.shape + img, landmark = img.copy(), landmark.copy() + landmark[..., 1] = H - 1 - landmark[..., 1] + landmark = np.round(landmark).astype(np.int32) + for i in range(landmark.shape[1]): + x, y = landmark[:, i, 0], landmark[:, i, 1] + for j in range(-step, step): + for k in range(-step, step): + u = np.clip(x + j, 0, W - 1) + v = np.clip(y + k, 0, H - 1) + for m in range(landmark.shape[0]): + img[m, v[m], u[m]] = c + return img diff --git a/src/face3d/util/visualizer.py b/src/face3d/util/visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..4023a6d4086acba9bc88e079f625194d324d7c9e --- /dev/null +++ b/src/face3d/util/visualizer.py @@ -0,0 +1,227 @@ +"""This script defines the visualizer for Deep3DFaceRecon_pytorch +""" + +import numpy as np +import os +import sys +import ntpath +import time +from . import util, html +from subprocess import Popen, PIPE +from torch.utils.tensorboard import SummaryWriter + +def save_images(webpage, visuals, image_path, aspect_ratio=1.0, width=256): + """Save images to the disk. + + Parameters: + webpage (the HTML class) -- the HTML webpage class that stores these imaegs (see html.py for more details) + visuals (OrderedDict) -- an ordered dictionary that stores (name, images (either tensor or numpy) ) pairs + image_path (str) -- the string is used to create image paths + aspect_ratio (float) -- the aspect ratio of saved images + width (int) -- the images will be resized to width x width + + This function will save images stored in 'visuals' to the HTML file specified by 'webpage'. + """ + image_dir = webpage.get_image_dir() + short_path = ntpath.basename(image_path[0]) + name = os.path.splitext(short_path)[0] + + webpage.add_header(name) + ims, txts, links = [], [], [] + + for label, im_data in visuals.items(): + im = util.tensor2im(im_data) + image_name = '%s/%s.png' % (label, name) + os.makedirs(os.path.join(image_dir, label), exist_ok=True) + save_path = os.path.join(image_dir, image_name) + util.save_image(im, save_path, aspect_ratio=aspect_ratio) + ims.append(image_name) + txts.append(label) + links.append(image_name) + webpage.add_images(ims, txts, links, width=width) + + +class Visualizer(): + """This class includes several functions that can display/save images and print/save logging information. + + It uses a Python library tensprboardX for display, and a Python library 'dominate' (wrapped in 'HTML') for creating HTML files with images. + """ + + def __init__(self, opt): + """Initialize the Visualizer class + + Parameters: + opt -- stores all the experiment flags; needs to be a subclass of BaseOptions + Step 1: Cache the training/test options + Step 2: create a tensorboard writer + Step 3: create an HTML object for saveing HTML filters + Step 4: create a logging file to store training losses + """ + self.opt = opt # cache the option + self.use_html = opt.isTrain and not opt.no_html + self.writer = SummaryWriter(os.path.join(opt.checkpoints_dir, 'logs', opt.name)) + self.win_size = opt.display_winsize + self.name = opt.name + self.saved = False + if self.use_html: # create an HTML object at /web/; images will be saved under /web/images/ + self.web_dir = os.path.join(opt.checkpoints_dir, opt.name, 'web') + self.img_dir = os.path.join(self.web_dir, 'images') + print('create web directory %s...' % self.web_dir) + util.mkdirs([self.web_dir, self.img_dir]) + # create a logging file to store training losses + self.log_name = os.path.join(opt.checkpoints_dir, opt.name, 'loss_log.txt') + with open(self.log_name, "a") as log_file: + now = time.strftime("%c") + log_file.write('================ Training Loss (%s) ================\n' % now) + + def reset(self): + """Reset the self.saved status""" + self.saved = False + + + def display_current_results(self, visuals, total_iters, epoch, save_result): + """Display current results on tensorboad; save current results to an HTML file. + + Parameters: + visuals (OrderedDict) - - dictionary of images to display or save + total_iters (int) -- total iterations + epoch (int) - - the current epoch + save_result (bool) - - if save the current results to an HTML file + """ + for label, image in visuals.items(): + self.writer.add_image(label, util.tensor2im(image), total_iters, dataformats='HWC') + + if self.use_html and (save_result or not self.saved): # save images to an HTML file if they haven't been saved. + self.saved = True + # save images to the disk + for label, image in visuals.items(): + image_numpy = util.tensor2im(image) + img_path = os.path.join(self.img_dir, 'epoch%.3d_%s.png' % (epoch, label)) + util.save_image(image_numpy, img_path) + + # update website + webpage = html.HTML(self.web_dir, 'Experiment name = %s' % self.name, refresh=0) + for n in range(epoch, 0, -1): + webpage.add_header('epoch [%d]' % n) + ims, txts, links = [], [], [] + + for label, image_numpy in visuals.items(): + image_numpy = util.tensor2im(image) + img_path = 'epoch%.3d_%s.png' % (n, label) + ims.append(img_path) + txts.append(label) + links.append(img_path) + webpage.add_images(ims, txts, links, width=self.win_size) + webpage.save() + + def plot_current_losses(self, total_iters, losses): + # G_loss_collection = {} + # D_loss_collection = {} + # for name, value in losses.items(): + # if 'G' in name or 'NCE' in name or 'idt' in name: + # G_loss_collection[name] = value + # else: + # D_loss_collection[name] = value + # self.writer.add_scalars('G_collec', G_loss_collection, total_iters) + # self.writer.add_scalars('D_collec', D_loss_collection, total_iters) + for name, value in losses.items(): + self.writer.add_scalar(name, value, total_iters) + + # losses: same format as |losses| of plot_current_losses + def print_current_losses(self, epoch, iters, losses, t_comp, t_data): + """print current losses on console; also save the losses to the disk + + Parameters: + epoch (int) -- current epoch + iters (int) -- current training iteration during this epoch (reset to 0 at the end of every epoch) + losses (OrderedDict) -- training losses stored in the format of (name, float) pairs + t_comp (float) -- computational time per data point (normalized by batch_size) + t_data (float) -- data loading time per data point (normalized by batch_size) + """ + message = '(epoch: %d, iters: %d, time: %.3f, data: %.3f) ' % (epoch, iters, t_comp, t_data) + for k, v in losses.items(): + message += '%s: %.3f ' % (k, v) + + print(message) # print the message + with open(self.log_name, "a") as log_file: + log_file.write('%s\n' % message) # save the message + + +class MyVisualizer: + def __init__(self, opt): + """Initialize the Visualizer class + + Parameters: + opt -- stores all the experiment flags; needs to be a subclass of BaseOptions + Step 1: Cache the training/test options + Step 2: create a tensorboard writer + Step 3: create an HTML object for saveing HTML filters + Step 4: create a logging file to store training losses + """ + self.opt = opt # cache the optio + self.name = opt.name + self.img_dir = os.path.join(opt.checkpoints_dir, opt.name, 'results') + + if opt.phase != 'test': + self.writer = SummaryWriter(os.path.join(opt.checkpoints_dir, opt.name, 'logs')) + # create a logging file to store training losses + self.log_name = os.path.join(opt.checkpoints_dir, opt.name, 'loss_log.txt') + with open(self.log_name, "a") as log_file: + now = time.strftime("%c") + log_file.write('================ Training Loss (%s) ================\n' % now) + + + def display_current_results(self, visuals, total_iters, epoch, dataset='train', save_results=False, count=0, name=None, + add_image=True): + """Display current results on tensorboad; save current results to an HTML file. + + Parameters: + visuals (OrderedDict) - - dictionary of images to display or save + total_iters (int) -- total iterations + epoch (int) - - the current epoch + dataset (str) - - 'train' or 'val' or 'test' + """ + # if (not add_image) and (not save_results): return + + for label, image in visuals.items(): + for i in range(image.shape[0]): + image_numpy = util.tensor2im(image[i]) + if add_image: + self.writer.add_image(label + '%s_%02d'%(dataset, i + count), + image_numpy, total_iters, dataformats='HWC') + + if save_results: + save_path = os.path.join(self.img_dir, dataset, 'epoch_%s_%06d'%(epoch, total_iters)) + if not os.path.isdir(save_path): + os.makedirs(save_path) + + if name is not None: + img_path = os.path.join(save_path, '%s.png' % name) + else: + img_path = os.path.join(save_path, '%s_%03d.png' % (label, i + count)) + util.save_image(image_numpy, img_path) + + + def plot_current_losses(self, total_iters, losses, dataset='train'): + for name, value in losses.items(): + self.writer.add_scalar(name + '/%s'%dataset, value, total_iters) + + # losses: same format as |losses| of plot_current_losses + def print_current_losses(self, epoch, iters, losses, t_comp, t_data, dataset='train'): + """print current losses on console; also save the losses to the disk + + Parameters: + epoch (int) -- current epoch + iters (int) -- current training iteration during this epoch (reset to 0 at the end of every epoch) + losses (OrderedDict) -- training losses stored in the format of (name, float) pairs + t_comp (float) -- computational time per data point (normalized by batch_size) + t_data (float) -- data loading time per data point (normalized by batch_size) + """ + message = '(dataset: %s, epoch: %d, iters: %d, time: %.3f, data: %.3f) ' % ( + dataset, epoch, iters, t_comp, t_data) + for k, v in losses.items(): + message += '%s: %.3f ' % (k, v) + + print(message) # print the message + with open(self.log_name, "a") as log_file: + log_file.write('%s\n' % message) # save the message diff --git a/src/face3d/visualize.py b/src/face3d/visualize.py new file mode 100644 index 0000000000000000000000000000000000000000..23a1110806a0ddf37d4aa549c023d1c3f7114e3e --- /dev/null +++ b/src/face3d/visualize.py @@ -0,0 +1,48 @@ +# check the sync of 3dmm feature and the audio +import cv2 +import numpy as np +from src.face3d.models.bfm import ParametricFaceModel +from src.face3d.models.facerecon_model import FaceReconModel +import torch +import subprocess, platform +import scipy.io as scio +from tqdm import tqdm + +# draft +def gen_composed_video(args, device, first_frame_coeff, coeff_path, audio_path, save_path, exp_dim=64): + + coeff_first = scio.loadmat(first_frame_coeff)['full_3dmm'] + + coeff_pred = scio.loadmat(coeff_path)['coeff_3dmm'] + + coeff_full = np.repeat(coeff_first, coeff_pred.shape[0], axis=0) # 257 + + coeff_full[:, 80:144] = coeff_pred[:, 0:64] + coeff_full[:, 224:227] = coeff_pred[:, 64:67] # 3 dim translation + coeff_full[:, 254:] = coeff_pred[:, 67:] # 3 dim translation + + tmp_video_path = '/tmp/face3dtmp.mp4' + + facemodel = FaceReconModel(args) + + video = cv2.VideoWriter(tmp_video_path, cv2.VideoWriter_fourcc(*'mp4v'), 25, (224, 224)) + + for k in tqdm(range(coeff_pred.shape[0]), 'face3d rendering:'): + cur_coeff_full = torch.tensor(coeff_full[k:k+1], device=device) + + facemodel.forward(cur_coeff_full, device) + + predicted_landmark = facemodel.pred_lm # TODO. + predicted_landmark = predicted_landmark.cpu().numpy().squeeze() + + rendered_img = facemodel.pred_face + rendered_img = 255. * rendered_img.cpu().numpy().squeeze().transpose(1,2,0) + out_img = rendered_img[:, :, :3].astype(np.uint8) + + video.write(np.uint8(out_img[:,:,::-1])) + + video.release() + + command = 'ffmpeg -v quiet -y -i {} -i {} -strict -2 -q:v 1 {}'.format(audio_path, tmp_video_path, save_path) + subprocess.call(command, shell=platform.system() != 'Windows') + diff --git a/src/facerender/__pycache__/animate.cpython-310.pyc b/src/facerender/__pycache__/animate.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab3c9b2b31ea2aff4bf87553078a3f1ecaeb4c9e Binary files /dev/null and b/src/facerender/__pycache__/animate.cpython-310.pyc differ diff --git a/src/facerender/animate.py b/src/facerender/animate.py new file mode 100644 index 0000000000000000000000000000000000000000..781f5a3318a086049cc6b74393073ddda7001d5e --- /dev/null +++ b/src/facerender/animate.py @@ -0,0 +1,257 @@ +import os +import cv2 +import yaml +import numpy as np +import warnings +from skimage import img_as_ubyte +import safetensors +import safetensors.torch +warnings.filterwarnings('ignore') + + +import imageio +import torch +import torchvision + + +from src.facerender.modules.keypoint_detector import HEEstimator, KPDetector +from src.facerender.modules.mapping import MappingNet +from src.facerender.modules.generator import OcclusionAwareGenerator, OcclusionAwareSPADEGenerator +from src.facerender.modules.make_animation import make_animation + +from pydub import AudioSegment +from src.utils.face_enhancer import enhancer_generator_with_len, enhancer_list +from src.utils.paste_pic import paste_pic +from src.utils.videoio import save_video_with_watermark + +try: + import webui # in webui + in_webui = True +except: + in_webui = False + +class AnimateFromCoeff(): + + def __init__(self, sadtalker_path, device): + + with open(sadtalker_path['facerender_yaml']) as f: + config = yaml.safe_load(f) + + generator = OcclusionAwareSPADEGenerator(**config['model_params']['generator_params'], + **config['model_params']['common_params']) + kp_extractor = KPDetector(**config['model_params']['kp_detector_params'], + **config['model_params']['common_params']) + he_estimator = HEEstimator(**config['model_params']['he_estimator_params'], + **config['model_params']['common_params']) + mapping = MappingNet(**config['model_params']['mapping_params']) + + generator.to(device) + kp_extractor.to(device) + he_estimator.to(device) + mapping.to(device) + for param in generator.parameters(): + param.requires_grad = False + for param in kp_extractor.parameters(): + param.requires_grad = False + for param in he_estimator.parameters(): + param.requires_grad = False + for param in mapping.parameters(): + param.requires_grad = False + + if sadtalker_path is not None: + if 'checkpoint' in sadtalker_path: # use safe tensor + self.load_cpk_facevid2vid_safetensor(sadtalker_path['checkpoint'], kp_detector=kp_extractor, generator=generator, he_estimator=None) + else: + self.load_cpk_facevid2vid(sadtalker_path['free_view_checkpoint'], kp_detector=kp_extractor, generator=generator, he_estimator=he_estimator) + else: + raise AttributeError("Checkpoint should be specified for video head pose estimator.") + + if sadtalker_path['mappingnet_checkpoint'] is not None: + self.load_cpk_mapping(sadtalker_path['mappingnet_checkpoint'], mapping=mapping) + else: + raise AttributeError("Checkpoint should be specified for video head pose estimator.") + + self.kp_extractor = kp_extractor + self.generator = generator + self.he_estimator = he_estimator + self.mapping = mapping + + self.kp_extractor.eval() + self.generator.eval() + self.he_estimator.eval() + self.mapping.eval() + + self.device = device + + def load_cpk_facevid2vid_safetensor(self, checkpoint_path, generator=None, + kp_detector=None, he_estimator=None, + device="cpu"): + + checkpoint = safetensors.torch.load_file(checkpoint_path) + + if generator is not None: + x_generator = {} + for k,v in checkpoint.items(): + if 'generator' in k: + x_generator[k.replace('generator.', '')] = v + generator.load_state_dict(x_generator) + if kp_detector is not None: + x_generator = {} + for k,v in checkpoint.items(): + if 'kp_extractor' in k: + x_generator[k.replace('kp_extractor.', '')] = v + kp_detector.load_state_dict(x_generator) + if he_estimator is not None: + x_generator = {} + for k,v in checkpoint.items(): + if 'he_estimator' in k: + x_generator[k.replace('he_estimator.', '')] = v + he_estimator.load_state_dict(x_generator) + + return None + + def load_cpk_facevid2vid(self, checkpoint_path, generator=None, discriminator=None, + kp_detector=None, he_estimator=None, optimizer_generator=None, + optimizer_discriminator=None, optimizer_kp_detector=None, + optimizer_he_estimator=None, device="cpu"): + checkpoint = torch.load(checkpoint_path, map_location=torch.device(device)) + if generator is not None: + generator.load_state_dict(checkpoint['generator']) + if kp_detector is not None: + kp_detector.load_state_dict(checkpoint['kp_detector']) + if he_estimator is not None: + he_estimator.load_state_dict(checkpoint['he_estimator']) + if discriminator is not None: + try: + discriminator.load_state_dict(checkpoint['discriminator']) + except: + print ('No discriminator in the state-dict. Dicriminator will be randomly initialized') + if optimizer_generator is not None: + optimizer_generator.load_state_dict(checkpoint['optimizer_generator']) + if optimizer_discriminator is not None: + try: + optimizer_discriminator.load_state_dict(checkpoint['optimizer_discriminator']) + except RuntimeError as e: + print ('No discriminator optimizer in the state-dict. Optimizer will be not initialized') + if optimizer_kp_detector is not None: + optimizer_kp_detector.load_state_dict(checkpoint['optimizer_kp_detector']) + if optimizer_he_estimator is not None: + optimizer_he_estimator.load_state_dict(checkpoint['optimizer_he_estimator']) + + return checkpoint['epoch'] + + def load_cpk_mapping(self, checkpoint_path, mapping=None, discriminator=None, + optimizer_mapping=None, optimizer_discriminator=None, device='cpu'): + checkpoint = torch.load(checkpoint_path, map_location=torch.device(device)) + if mapping is not None: + mapping.load_state_dict(checkpoint['mapping']) + if discriminator is not None: + discriminator.load_state_dict(checkpoint['discriminator']) + if optimizer_mapping is not None: + optimizer_mapping.load_state_dict(checkpoint['optimizer_mapping']) + if optimizer_discriminator is not None: + optimizer_discriminator.load_state_dict(checkpoint['optimizer_discriminator']) + + return checkpoint['epoch'] + + def generate(self, x, video_save_dir, pic_path, crop_info, enhancer=None, background_enhancer=None, preprocess='crop', img_size=256): + + source_image=x['source_image'].type(torch.FloatTensor) + source_semantics=x['source_semantics'].type(torch.FloatTensor) + target_semantics=x['target_semantics_list'].type(torch.FloatTensor) + source_image=source_image.to(self.device) + source_semantics=source_semantics.to(self.device) + target_semantics=target_semantics.to(self.device) + if 'yaw_c_seq' in x: + yaw_c_seq = x['yaw_c_seq'].type(torch.FloatTensor) + yaw_c_seq = x['yaw_c_seq'].to(self.device) + else: + yaw_c_seq = None + if 'pitch_c_seq' in x: + pitch_c_seq = x['pitch_c_seq'].type(torch.FloatTensor) + pitch_c_seq = x['pitch_c_seq'].to(self.device) + else: + pitch_c_seq = None + if 'roll_c_seq' in x: + roll_c_seq = x['roll_c_seq'].type(torch.FloatTensor) + roll_c_seq = x['roll_c_seq'].to(self.device) + else: + roll_c_seq = None + + frame_num = x['frame_num'] + + predictions_video = make_animation(source_image, source_semantics, target_semantics, + self.generator, self.kp_extractor, self.he_estimator, self.mapping, + yaw_c_seq, pitch_c_seq, roll_c_seq, use_exp = True) + + predictions_video = predictions_video.reshape((-1,)+predictions_video.shape[2:]) + predictions_video = predictions_video[:frame_num] + + video = [] + for idx in range(predictions_video.shape[0]): + image = predictions_video[idx] + image = np.transpose(image.data.cpu().numpy(), [1, 2, 0]).astype(np.float32) + video.append(image) + result = img_as_ubyte(video) + + ### the generated video is 256x256, so we keep the aspect ratio, + original_size = crop_info[0] + if original_size: + result = [ cv2.resize(result_i,(img_size, int(img_size * original_size[1]/original_size[0]) )) for result_i in result ] + + video_name = x['video_name'] + '.mp4' + path = os.path.join(video_save_dir, 'temp_'+video_name) + + imageio.mimsave(path, result, fps=float(25)) + + av_path = os.path.join(video_save_dir, video_name) + return_path = av_path + + audio_path = x['audio_path'] + audio_name = os.path.splitext(os.path.split(audio_path)[-1])[0] + new_audio_path = os.path.join(video_save_dir, audio_name+'.wav') + start_time = 0 + # cog will not keep the .mp3 filename + sound = AudioSegment.from_file(audio_path) + frames = frame_num + end_time = start_time + frames*1/25*1000 + word1=sound.set_frame_rate(16000) + word = word1[start_time:end_time] + word.export(new_audio_path, format="wav") + + save_video_with_watermark(path, new_audio_path, av_path, watermark= False) + print(f'The generated video is named {video_save_dir}/{video_name}') + + if 'full' in preprocess.lower(): + # only add watermark to the full image. + video_name_full = x['video_name'] + '_full.mp4' + full_video_path = os.path.join(video_save_dir, video_name_full) + return_path = full_video_path + paste_pic(path, pic_path, crop_info, new_audio_path, full_video_path, extended_crop= True if 'ext' in preprocess.lower() else False) + print(f'The generated video is named {video_save_dir}/{video_name_full}') + else: + full_video_path = av_path + + #### paste back then enhancers + if enhancer: + video_name_enhancer = x['video_name'] + '_enhanced.mp4' + enhanced_path = os.path.join(video_save_dir, 'temp_'+video_name_enhancer) + av_path_enhancer = os.path.join(video_save_dir, video_name_enhancer) + return_path = av_path_enhancer + + try: + enhanced_images_gen_with_len = enhancer_generator_with_len(full_video_path, method=enhancer, bg_upsampler=background_enhancer) + imageio.mimsave(enhanced_path, enhanced_images_gen_with_len, fps=float(25)) + except: + enhanced_images_gen_with_len = enhancer_list(full_video_path, method=enhancer, bg_upsampler=background_enhancer) + imageio.mimsave(enhanced_path, enhanced_images_gen_with_len, fps=float(25)) + + save_video_with_watermark(enhanced_path, new_audio_path, av_path_enhancer, watermark= False) + print(f'The generated video is named {video_save_dir}/{video_name_enhancer}') + os.remove(enhanced_path) + + os.remove(path) + os.remove(new_audio_path) + + return return_path + diff --git a/src/facerender/modules/__pycache__/dense_motion.cpython-310.pyc b/src/facerender/modules/__pycache__/dense_motion.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a5f5685d58e48cb8a56713dba2fa7c71af6f05a4 Binary files /dev/null and b/src/facerender/modules/__pycache__/dense_motion.cpython-310.pyc differ diff --git a/src/facerender/modules/__pycache__/generator.cpython-310.pyc b/src/facerender/modules/__pycache__/generator.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c7003154027d22f875d848a115eade553a588fd Binary files /dev/null and b/src/facerender/modules/__pycache__/generator.cpython-310.pyc differ diff --git a/src/facerender/modules/__pycache__/keypoint_detector.cpython-310.pyc b/src/facerender/modules/__pycache__/keypoint_detector.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a534a9e025b06be8a25e92a13cc7a2b29205f93 Binary files /dev/null and b/src/facerender/modules/__pycache__/keypoint_detector.cpython-310.pyc differ diff --git a/src/facerender/modules/__pycache__/make_animation.cpython-310.pyc b/src/facerender/modules/__pycache__/make_animation.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d2818b08fc660b50b9c8b1aa649525f4f925e11 Binary files /dev/null and b/src/facerender/modules/__pycache__/make_animation.cpython-310.pyc differ diff --git a/src/facerender/modules/__pycache__/mapping.cpython-310.pyc b/src/facerender/modules/__pycache__/mapping.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b96fb4855cfeb0b391775173aec6958aa815e17 Binary files /dev/null and b/src/facerender/modules/__pycache__/mapping.cpython-310.pyc differ diff --git a/src/facerender/modules/__pycache__/util.cpython-310.pyc b/src/facerender/modules/__pycache__/util.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d6bb19f4884726ff9f2ee41d2c8dc967d17ce1c Binary files /dev/null and b/src/facerender/modules/__pycache__/util.cpython-310.pyc differ diff --git a/src/facerender/modules/dense_motion.py b/src/facerender/modules/dense_motion.py new file mode 100644 index 0000000000000000000000000000000000000000..a286ead2e84ed1961335d34a3b50ab38f25e4495 --- /dev/null +++ b/src/facerender/modules/dense_motion.py @@ -0,0 +1,121 @@ +from torch import nn +import torch.nn.functional as F +import torch +from src.facerender.modules.util import Hourglass, make_coordinate_grid, kp2gaussian + +from src.facerender.sync_batchnorm import SynchronizedBatchNorm3d as BatchNorm3d + + +class DenseMotionNetwork(nn.Module): + """ + Module that predicting a dense motion from sparse motion representation given by kp_source and kp_driving + """ + + def __init__(self, block_expansion, num_blocks, max_features, num_kp, feature_channel, reshape_depth, compress, + estimate_occlusion_map=False): + super(DenseMotionNetwork, self).__init__() + # self.hourglass = Hourglass(block_expansion=block_expansion, in_features=(num_kp+1)*(feature_channel+1), max_features=max_features, num_blocks=num_blocks) + self.hourglass = Hourglass(block_expansion=block_expansion, in_features=(num_kp+1)*(compress+1), max_features=max_features, num_blocks=num_blocks) + + self.mask = nn.Conv3d(self.hourglass.out_filters, num_kp + 1, kernel_size=7, padding=3) + + self.compress = nn.Conv3d(feature_channel, compress, kernel_size=1) + self.norm = BatchNorm3d(compress, affine=True) + + if estimate_occlusion_map: + # self.occlusion = nn.Conv2d(reshape_channel*reshape_depth, 1, kernel_size=7, padding=3) + self.occlusion = nn.Conv2d(self.hourglass.out_filters*reshape_depth, 1, kernel_size=7, padding=3) + else: + self.occlusion = None + + self.num_kp = num_kp + + + def create_sparse_motions(self, feature, kp_driving, kp_source): + bs, _, d, h, w = feature.shape + identity_grid = make_coordinate_grid((d, h, w), type=kp_source['value'].type()) + identity_grid = identity_grid.view(1, 1, d, h, w, 3) + coordinate_grid = identity_grid - kp_driving['value'].view(bs, self.num_kp, 1, 1, 1, 3) + + # if 'jacobian' in kp_driving: + if 'jacobian' in kp_driving and kp_driving['jacobian'] is not None: + jacobian = torch.matmul(kp_source['jacobian'], torch.inverse(kp_driving['jacobian'])) + jacobian = jacobian.unsqueeze(-3).unsqueeze(-3).unsqueeze(-3) + jacobian = jacobian.repeat(1, 1, d, h, w, 1, 1) + coordinate_grid = torch.matmul(jacobian, coordinate_grid.unsqueeze(-1)) + coordinate_grid = coordinate_grid.squeeze(-1) + + + driving_to_source = coordinate_grid + kp_source['value'].view(bs, self.num_kp, 1, 1, 1, 3) # (bs, num_kp, d, h, w, 3) + + #adding background feature + identity_grid = identity_grid.repeat(bs, 1, 1, 1, 1, 1) + sparse_motions = torch.cat([identity_grid, driving_to_source], dim=1) #bs num_kp+1 d h w 3 + + # sparse_motions = driving_to_source + + return sparse_motions + + def create_deformed_feature(self, feature, sparse_motions): + bs, _, d, h, w = feature.shape + feature_repeat = feature.unsqueeze(1).unsqueeze(1).repeat(1, self.num_kp+1, 1, 1, 1, 1, 1) # (bs, num_kp+1, 1, c, d, h, w) + feature_repeat = feature_repeat.view(bs * (self.num_kp+1), -1, d, h, w) # (bs*(num_kp+1), c, d, h, w) + sparse_motions = sparse_motions.view((bs * (self.num_kp+1), d, h, w, -1)) # (bs*(num_kp+1), d, h, w, 3) !!!! + sparse_deformed = F.grid_sample(feature_repeat, sparse_motions) + sparse_deformed = sparse_deformed.view((bs, self.num_kp+1, -1, d, h, w)) # (bs, num_kp+1, c, d, h, w) + return sparse_deformed + + def create_heatmap_representations(self, feature, kp_driving, kp_source): + spatial_size = feature.shape[3:] + gaussian_driving = kp2gaussian(kp_driving, spatial_size=spatial_size, kp_variance=0.01) + gaussian_source = kp2gaussian(kp_source, spatial_size=spatial_size, kp_variance=0.01) + heatmap = gaussian_driving - gaussian_source + + # adding background feature + zeros = torch.zeros(heatmap.shape[0], 1, spatial_size[0], spatial_size[1], spatial_size[2]).type(heatmap.type()) + heatmap = torch.cat([zeros, heatmap], dim=1) + heatmap = heatmap.unsqueeze(2) # (bs, num_kp+1, 1, d, h, w) + return heatmap + + def forward(self, feature, kp_driving, kp_source): + bs, _, d, h, w = feature.shape + + feature = self.compress(feature) + feature = self.norm(feature) + feature = F.relu(feature) + + out_dict = dict() + sparse_motion = self.create_sparse_motions(feature, kp_driving, kp_source) + deformed_feature = self.create_deformed_feature(feature, sparse_motion) + + heatmap = self.create_heatmap_representations(deformed_feature, kp_driving, kp_source) + + input_ = torch.cat([heatmap, deformed_feature], dim=2) + input_ = input_.view(bs, -1, d, h, w) + + # input = deformed_feature.view(bs, -1, d, h, w) # (bs, num_kp+1 * c, d, h, w) + + prediction = self.hourglass(input_) + + + mask = self.mask(prediction) + mask = F.softmax(mask, dim=1) + out_dict['mask'] = mask + mask = mask.unsqueeze(2) # (bs, num_kp+1, 1, d, h, w) + + zeros_mask = torch.zeros_like(mask) + mask = torch.where(mask < 1e-3, zeros_mask, mask) + + sparse_motion = sparse_motion.permute(0, 1, 5, 2, 3, 4) # (bs, num_kp+1, 3, d, h, w) + deformation = (sparse_motion * mask).sum(dim=1) # (bs, 3, d, h, w) + deformation = deformation.permute(0, 2, 3, 4, 1) # (bs, d, h, w, 3) + + out_dict['deformation'] = deformation + + if self.occlusion: + bs, c, d, h, w = prediction.shape + prediction = prediction.view(bs, -1, h, w) + occlusion_map = torch.sigmoid(self.occlusion(prediction)) + out_dict['occlusion_map'] = occlusion_map + + return out_dict diff --git a/src/facerender/modules/discriminator.py b/src/facerender/modules/discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..d4459b07cb075c9f9d345f9b3dffc02cd859313b --- /dev/null +++ b/src/facerender/modules/discriminator.py @@ -0,0 +1,90 @@ +from torch import nn +import torch.nn.functional as F +from facerender.modules.util import kp2gaussian +import torch + + +class DownBlock2d(nn.Module): + """ + Simple block for processing video (encoder). + """ + + def __init__(self, in_features, out_features, norm=False, kernel_size=4, pool=False, sn=False): + super(DownBlock2d, self).__init__() + self.conv = nn.Conv2d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size) + + if sn: + self.conv = nn.utils.spectral_norm(self.conv) + + if norm: + self.norm = nn.InstanceNorm2d(out_features, affine=True) + else: + self.norm = None + self.pool = pool + + def forward(self, x): + out = x + out = self.conv(out) + if self.norm: + out = self.norm(out) + out = F.leaky_relu(out, 0.2) + if self.pool: + out = F.avg_pool2d(out, (2, 2)) + return out + + +class Discriminator(nn.Module): + """ + Discriminator similar to Pix2Pix + """ + + def __init__(self, num_channels=3, block_expansion=64, num_blocks=4, max_features=512, + sn=False, **kwargs): + super(Discriminator, self).__init__() + + down_blocks = [] + for i in range(num_blocks): + down_blocks.append( + DownBlock2d(num_channels if i == 0 else min(max_features, block_expansion * (2 ** i)), + min(max_features, block_expansion * (2 ** (i + 1))), + norm=(i != 0), kernel_size=4, pool=(i != num_blocks - 1), sn=sn)) + + self.down_blocks = nn.ModuleList(down_blocks) + self.conv = nn.Conv2d(self.down_blocks[-1].conv.out_channels, out_channels=1, kernel_size=1) + if sn: + self.conv = nn.utils.spectral_norm(self.conv) + + def forward(self, x): + feature_maps = [] + out = x + + for down_block in self.down_blocks: + feature_maps.append(down_block(out)) + out = feature_maps[-1] + prediction_map = self.conv(out) + + return feature_maps, prediction_map + + +class MultiScaleDiscriminator(nn.Module): + """ + Multi-scale (scale) discriminator + """ + + def __init__(self, scales=(), **kwargs): + super(MultiScaleDiscriminator, self).__init__() + self.scales = scales + discs = {} + for scale in scales: + discs[str(scale).replace('.', '-')] = Discriminator(**kwargs) + self.discs = nn.ModuleDict(discs) + + def forward(self, x): + out_dict = {} + for scale, disc in self.discs.items(): + scale = str(scale).replace('-', '.') + key = 'prediction_' + scale + feature_maps, prediction_map = disc(x[key]) + out_dict['feature_maps_' + scale] = feature_maps + out_dict['prediction_map_' + scale] = prediction_map + return out_dict diff --git a/src/facerender/modules/generator.py b/src/facerender/modules/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..5a9edcb3b328d3afc99072b2461d7ca69919f813 --- /dev/null +++ b/src/facerender/modules/generator.py @@ -0,0 +1,255 @@ +import torch +from torch import nn +import torch.nn.functional as F +from src.facerender.modules.util import ResBlock2d, SameBlock2d, UpBlock2d, DownBlock2d, ResBlock3d, SPADEResnetBlock +from src.facerender.modules.dense_motion import DenseMotionNetwork + + +class OcclusionAwareGenerator(nn.Module): + """ + Generator follows NVIDIA architecture. + """ + + def __init__(self, image_channel, feature_channel, num_kp, block_expansion, max_features, num_down_blocks, reshape_channel, reshape_depth, + num_resblocks, estimate_occlusion_map=False, dense_motion_params=None, estimate_jacobian=False): + super(OcclusionAwareGenerator, self).__init__() + + if dense_motion_params is not None: + self.dense_motion_network = DenseMotionNetwork(num_kp=num_kp, feature_channel=feature_channel, + estimate_occlusion_map=estimate_occlusion_map, + **dense_motion_params) + else: + self.dense_motion_network = None + + self.first = SameBlock2d(image_channel, block_expansion, kernel_size=(7, 7), padding=(3, 3)) + + down_blocks = [] + for i in range(num_down_blocks): + in_features = min(max_features, block_expansion * (2 ** i)) + out_features = min(max_features, block_expansion * (2 ** (i + 1))) + down_blocks.append(DownBlock2d(in_features, out_features, kernel_size=(3, 3), padding=(1, 1))) + self.down_blocks = nn.ModuleList(down_blocks) + + self.second = nn.Conv2d(in_channels=out_features, out_channels=max_features, kernel_size=1, stride=1) + + self.reshape_channel = reshape_channel + self.reshape_depth = reshape_depth + + self.resblocks_3d = torch.nn.Sequential() + for i in range(num_resblocks): + self.resblocks_3d.add_module('3dr' + str(i), ResBlock3d(reshape_channel, kernel_size=3, padding=1)) + + out_features = block_expansion * (2 ** (num_down_blocks)) + self.third = SameBlock2d(max_features, out_features, kernel_size=(3, 3), padding=(1, 1), lrelu=True) + self.fourth = nn.Conv2d(in_channels=out_features, out_channels=out_features, kernel_size=1, stride=1) + + self.resblocks_2d = torch.nn.Sequential() + for i in range(num_resblocks): + self.resblocks_2d.add_module('2dr' + str(i), ResBlock2d(out_features, kernel_size=3, padding=1)) + + up_blocks = [] + for i in range(num_down_blocks): + in_features = max(block_expansion, block_expansion * (2 ** (num_down_blocks - i))) + out_features = max(block_expansion, block_expansion * (2 ** (num_down_blocks - i - 1))) + up_blocks.append(UpBlock2d(in_features, out_features, kernel_size=(3, 3), padding=(1, 1))) + self.up_blocks = nn.ModuleList(up_blocks) + + self.final = nn.Conv2d(block_expansion, image_channel, kernel_size=(7, 7), padding=(3, 3)) + self.estimate_occlusion_map = estimate_occlusion_map + self.image_channel = image_channel + + def deform_input(self, inp, deformation): + _, d_old, h_old, w_old, _ = deformation.shape + _, _, d, h, w = inp.shape + if d_old != d or h_old != h or w_old != w: + deformation = deformation.permute(0, 4, 1, 2, 3) + deformation = F.interpolate(deformation, size=(d, h, w), mode='trilinear') + deformation = deformation.permute(0, 2, 3, 4, 1) + return F.grid_sample(inp, deformation) + + def forward(self, source_image, kp_driving, kp_source): + # Encoding (downsampling) part + out = self.first(source_image) + for i in range(len(self.down_blocks)): + out = self.down_blocks[i](out) + out = self.second(out) + bs, c, h, w = out.shape + # print(out.shape) + feature_3d = out.view(bs, self.reshape_channel, self.reshape_depth, h ,w) + feature_3d = self.resblocks_3d(feature_3d) + + # Transforming feature representation according to deformation and occlusion + output_dict = {} + if self.dense_motion_network is not None: + dense_motion = self.dense_motion_network(feature=feature_3d, kp_driving=kp_driving, + kp_source=kp_source) + output_dict['mask'] = dense_motion['mask'] + + if 'occlusion_map' in dense_motion: + occlusion_map = dense_motion['occlusion_map'] + output_dict['occlusion_map'] = occlusion_map + else: + occlusion_map = None + deformation = dense_motion['deformation'] + out = self.deform_input(feature_3d, deformation) + + bs, c, d, h, w = out.shape + out = out.view(bs, c*d, h, w) + out = self.third(out) + out = self.fourth(out) + + if occlusion_map is not None: + if out.shape[2] != occlusion_map.shape[2] or out.shape[3] != occlusion_map.shape[3]: + occlusion_map = F.interpolate(occlusion_map, size=out.shape[2:], mode='bilinear') + out = out * occlusion_map + + # output_dict["deformed"] = self.deform_input(source_image, deformation) # 3d deformation cannot deform 2d image + + # Decoding part + out = self.resblocks_2d(out) + for i in range(len(self.up_blocks)): + out = self.up_blocks[i](out) + out = self.final(out) + out = F.sigmoid(out) + + output_dict["prediction"] = out + + return output_dict + + +class SPADEDecoder(nn.Module): + def __init__(self): + super().__init__() + ic = 256 + oc = 64 + norm_G = 'spadespectralinstance' + label_nc = 256 + + self.fc = nn.Conv2d(ic, 2 * ic, 3, padding=1) + self.G_middle_0 = SPADEResnetBlock(2 * ic, 2 * ic, norm_G, label_nc) + self.G_middle_1 = SPADEResnetBlock(2 * ic, 2 * ic, norm_G, label_nc) + self.G_middle_2 = SPADEResnetBlock(2 * ic, 2 * ic, norm_G, label_nc) + self.G_middle_3 = SPADEResnetBlock(2 * ic, 2 * ic, norm_G, label_nc) + self.G_middle_4 = SPADEResnetBlock(2 * ic, 2 * ic, norm_G, label_nc) + self.G_middle_5 = SPADEResnetBlock(2 * ic, 2 * ic, norm_G, label_nc) + self.up_0 = SPADEResnetBlock(2 * ic, ic, norm_G, label_nc) + self.up_1 = SPADEResnetBlock(ic, oc, norm_G, label_nc) + self.conv_img = nn.Conv2d(oc, 3, 3, padding=1) + self.up = nn.Upsample(scale_factor=2) + + def forward(self, feature): + seg = feature + x = self.fc(feature) + x = self.G_middle_0(x, seg) + x = self.G_middle_1(x, seg) + x = self.G_middle_2(x, seg) + x = self.G_middle_3(x, seg) + x = self.G_middle_4(x, seg) + x = self.G_middle_5(x, seg) + x = self.up(x) + x = self.up_0(x, seg) # 256, 128, 128 + x = self.up(x) + x = self.up_1(x, seg) # 64, 256, 256 + + x = self.conv_img(F.leaky_relu(x, 2e-1)) + # x = torch.tanh(x) + x = F.sigmoid(x) + + return x + + +class OcclusionAwareSPADEGenerator(nn.Module): + + def __init__(self, image_channel, feature_channel, num_kp, block_expansion, max_features, num_down_blocks, reshape_channel, reshape_depth, + num_resblocks, estimate_occlusion_map=False, dense_motion_params=None, estimate_jacobian=False): + super(OcclusionAwareSPADEGenerator, self).__init__() + + if dense_motion_params is not None: + self.dense_motion_network = DenseMotionNetwork(num_kp=num_kp, feature_channel=feature_channel, + estimate_occlusion_map=estimate_occlusion_map, + **dense_motion_params) + else: + self.dense_motion_network = None + + self.first = SameBlock2d(image_channel, block_expansion, kernel_size=(3, 3), padding=(1, 1)) + + down_blocks = [] + for i in range(num_down_blocks): + in_features = min(max_features, block_expansion * (2 ** i)) + out_features = min(max_features, block_expansion * (2 ** (i + 1))) + down_blocks.append(DownBlock2d(in_features, out_features, kernel_size=(3, 3), padding=(1, 1))) + self.down_blocks = nn.ModuleList(down_blocks) + + self.second = nn.Conv2d(in_channels=out_features, out_channels=max_features, kernel_size=1, stride=1) + + self.reshape_channel = reshape_channel + self.reshape_depth = reshape_depth + + self.resblocks_3d = torch.nn.Sequential() + for i in range(num_resblocks): + self.resblocks_3d.add_module('3dr' + str(i), ResBlock3d(reshape_channel, kernel_size=3, padding=1)) + + out_features = block_expansion * (2 ** (num_down_blocks)) + self.third = SameBlock2d(max_features, out_features, kernel_size=(3, 3), padding=(1, 1), lrelu=True) + self.fourth = nn.Conv2d(in_channels=out_features, out_channels=out_features, kernel_size=1, stride=1) + + self.estimate_occlusion_map = estimate_occlusion_map + self.image_channel = image_channel + + self.decoder = SPADEDecoder() + + def deform_input(self, inp, deformation): + _, d_old, h_old, w_old, _ = deformation.shape + _, _, d, h, w = inp.shape + if d_old != d or h_old != h or w_old != w: + deformation = deformation.permute(0, 4, 1, 2, 3) + deformation = F.interpolate(deformation, size=(d, h, w), mode='trilinear') + deformation = deformation.permute(0, 2, 3, 4, 1) + return F.grid_sample(inp, deformation) + + def forward(self, source_image, kp_driving, kp_source): + # Encoding (downsampling) part + out = self.first(source_image) + for i in range(len(self.down_blocks)): + out = self.down_blocks[i](out) + out = self.second(out) + bs, c, h, w = out.shape + # print(out.shape) + feature_3d = out.view(bs, self.reshape_channel, self.reshape_depth, h ,w) + feature_3d = self.resblocks_3d(feature_3d) + + # Transforming feature representation according to deformation and occlusion + output_dict = {} + if self.dense_motion_network is not None: + dense_motion = self.dense_motion_network(feature=feature_3d, kp_driving=kp_driving, + kp_source=kp_source) + output_dict['mask'] = dense_motion['mask'] + + # import pdb; pdb.set_trace() + + if 'occlusion_map' in dense_motion: + occlusion_map = dense_motion['occlusion_map'] + output_dict['occlusion_map'] = occlusion_map + else: + occlusion_map = None + deformation = dense_motion['deformation'] + out = self.deform_input(feature_3d, deformation) + + bs, c, d, h, w = out.shape + out = out.view(bs, c*d, h, w) + out = self.third(out) + out = self.fourth(out) + + # occlusion_map = torch.where(occlusion_map < 0.95, 0, occlusion_map) + + if occlusion_map is not None: + if out.shape[2] != occlusion_map.shape[2] or out.shape[3] != occlusion_map.shape[3]: + occlusion_map = F.interpolate(occlusion_map, size=out.shape[2:], mode='bilinear') + out = out * occlusion_map + + # Decoding part + out = self.decoder(out) + + output_dict["prediction"] = out + + return output_dict \ No newline at end of file diff --git a/src/facerender/modules/keypoint_detector.py b/src/facerender/modules/keypoint_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..62a38a962b2f1a4326aac771aced353ec5e22a96 --- /dev/null +++ b/src/facerender/modules/keypoint_detector.py @@ -0,0 +1,179 @@ +from torch import nn +import torch +import torch.nn.functional as F + +from src.facerender.sync_batchnorm import SynchronizedBatchNorm2d as BatchNorm2d +from src.facerender.modules.util import KPHourglass, make_coordinate_grid, AntiAliasInterpolation2d, ResBottleneck + + +class KPDetector(nn.Module): + """ + Detecting canonical keypoints. Return keypoint position and jacobian near each keypoint. + """ + + def __init__(self, block_expansion, feature_channel, num_kp, image_channel, max_features, reshape_channel, reshape_depth, + num_blocks, temperature, estimate_jacobian=False, scale_factor=1, single_jacobian_map=False): + super(KPDetector, self).__init__() + + self.predictor = KPHourglass(block_expansion, in_features=image_channel, + max_features=max_features, reshape_features=reshape_channel, reshape_depth=reshape_depth, num_blocks=num_blocks) + + # self.kp = nn.Conv3d(in_channels=self.predictor.out_filters, out_channels=num_kp, kernel_size=7, padding=3) + self.kp = nn.Conv3d(in_channels=self.predictor.out_filters, out_channels=num_kp, kernel_size=3, padding=1) + + if estimate_jacobian: + self.num_jacobian_maps = 1 if single_jacobian_map else num_kp + # self.jacobian = nn.Conv3d(in_channels=self.predictor.out_filters, out_channels=9 * self.num_jacobian_maps, kernel_size=7, padding=3) + self.jacobian = nn.Conv3d(in_channels=self.predictor.out_filters, out_channels=9 * self.num_jacobian_maps, kernel_size=3, padding=1) + ''' + initial as: + [[1 0 0] + [0 1 0] + [0 0 1]] + ''' + self.jacobian.weight.data.zero_() + self.jacobian.bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0, 0, 0, 1] * self.num_jacobian_maps, dtype=torch.float)) + else: + self.jacobian = None + + self.temperature = temperature + self.scale_factor = scale_factor + if self.scale_factor != 1: + self.down = AntiAliasInterpolation2d(image_channel, self.scale_factor) + + def gaussian2kp(self, heatmap): + """ + Extract the mean from a heatmap + """ + shape = heatmap.shape + heatmap = heatmap.unsqueeze(-1) + grid = make_coordinate_grid(shape[2:], heatmap.type()).unsqueeze_(0).unsqueeze_(0) + value = (heatmap * grid).sum(dim=(2, 3, 4)) + kp = {'value': value} + + return kp + + def forward(self, x): + if self.scale_factor != 1: + x = self.down(x) + + feature_map = self.predictor(x) + prediction = self.kp(feature_map) + + final_shape = prediction.shape + heatmap = prediction.view(final_shape[0], final_shape[1], -1) + heatmap = F.softmax(heatmap / self.temperature, dim=2) + heatmap = heatmap.view(*final_shape) + + out = self.gaussian2kp(heatmap) + + if self.jacobian is not None: + jacobian_map = self.jacobian(feature_map) + jacobian_map = jacobian_map.reshape(final_shape[0], self.num_jacobian_maps, 9, final_shape[2], + final_shape[3], final_shape[4]) + heatmap = heatmap.unsqueeze(2) + + jacobian = heatmap * jacobian_map + jacobian = jacobian.view(final_shape[0], final_shape[1], 9, -1) + jacobian = jacobian.sum(dim=-1) + jacobian = jacobian.view(jacobian.shape[0], jacobian.shape[1], 3, 3) + out['jacobian'] = jacobian + + return out + + +class HEEstimator(nn.Module): + """ + Estimating head pose and expression. + """ + + def __init__(self, block_expansion, feature_channel, num_kp, image_channel, max_features, num_bins=66, estimate_jacobian=True): + super(HEEstimator, self).__init__() + + self.conv1 = nn.Conv2d(in_channels=image_channel, out_channels=block_expansion, kernel_size=7, padding=3, stride=2) + self.norm1 = BatchNorm2d(block_expansion, affine=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + self.conv2 = nn.Conv2d(in_channels=block_expansion, out_channels=256, kernel_size=1) + self.norm2 = BatchNorm2d(256, affine=True) + + self.block1 = nn.Sequential() + for i in range(3): + self.block1.add_module('b1_'+ str(i), ResBottleneck(in_features=256, stride=1)) + + self.conv3 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=1) + self.norm3 = BatchNorm2d(512, affine=True) + self.block2 = ResBottleneck(in_features=512, stride=2) + + self.block3 = nn.Sequential() + for i in range(3): + self.block3.add_module('b3_'+ str(i), ResBottleneck(in_features=512, stride=1)) + + self.conv4 = nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=1) + self.norm4 = BatchNorm2d(1024, affine=True) + self.block4 = ResBottleneck(in_features=1024, stride=2) + + self.block5 = nn.Sequential() + for i in range(5): + self.block5.add_module('b5_'+ str(i), ResBottleneck(in_features=1024, stride=1)) + + self.conv5 = nn.Conv2d(in_channels=1024, out_channels=2048, kernel_size=1) + self.norm5 = BatchNorm2d(2048, affine=True) + self.block6 = ResBottleneck(in_features=2048, stride=2) + + self.block7 = nn.Sequential() + for i in range(2): + self.block7.add_module('b7_'+ str(i), ResBottleneck(in_features=2048, stride=1)) + + self.fc_roll = nn.Linear(2048, num_bins) + self.fc_pitch = nn.Linear(2048, num_bins) + self.fc_yaw = nn.Linear(2048, num_bins) + + self.fc_t = nn.Linear(2048, 3) + + self.fc_exp = nn.Linear(2048, 3*num_kp) + + def forward(self, x): + out = self.conv1(x) + out = self.norm1(out) + out = F.relu(out) + out = self.maxpool(out) + + out = self.conv2(out) + out = self.norm2(out) + out = F.relu(out) + + out = self.block1(out) + + out = self.conv3(out) + out = self.norm3(out) + out = F.relu(out) + out = self.block2(out) + + out = self.block3(out) + + out = self.conv4(out) + out = self.norm4(out) + out = F.relu(out) + out = self.block4(out) + + out = self.block5(out) + + out = self.conv5(out) + out = self.norm5(out) + out = F.relu(out) + out = self.block6(out) + + out = self.block7(out) + + out = F.adaptive_avg_pool2d(out, 1) + out = out.view(out.shape[0], -1) + + yaw = self.fc_roll(out) + pitch = self.fc_pitch(out) + roll = self.fc_yaw(out) + t = self.fc_t(out) + exp = self.fc_exp(out) + + return {'yaw': yaw, 'pitch': pitch, 'roll': roll, 't': t, 'exp': exp} + diff --git a/src/facerender/modules/make_animation.py b/src/facerender/modules/make_animation.py new file mode 100644 index 0000000000000000000000000000000000000000..3360c53501a064f35d7db21a5361f89aa9658b42 --- /dev/null +++ b/src/facerender/modules/make_animation.py @@ -0,0 +1,170 @@ +from scipy.spatial import ConvexHull +import torch +import torch.nn.functional as F +import numpy as np +from tqdm import tqdm + +def normalize_kp(kp_source, kp_driving, kp_driving_initial, adapt_movement_scale=False, + use_relative_movement=False, use_relative_jacobian=False): + if adapt_movement_scale: + source_area = ConvexHull(kp_source['value'][0].data.cpu().numpy()).volume + driving_area = ConvexHull(kp_driving_initial['value'][0].data.cpu().numpy()).volume + adapt_movement_scale = np.sqrt(source_area) / np.sqrt(driving_area) + else: + adapt_movement_scale = 1 + + kp_new = {k: v for k, v in kp_driving.items()} + + if use_relative_movement: + kp_value_diff = (kp_driving['value'] - kp_driving_initial['value']) + kp_value_diff *= adapt_movement_scale + kp_new['value'] = kp_value_diff + kp_source['value'] + + if use_relative_jacobian: + jacobian_diff = torch.matmul(kp_driving['jacobian'], torch.inverse(kp_driving_initial['jacobian'])) + kp_new['jacobian'] = torch.matmul(jacobian_diff, kp_source['jacobian']) + + return kp_new + +def headpose_pred_to_degree(pred): + device = pred.device + idx_tensor = [idx for idx in range(66)] + idx_tensor = torch.FloatTensor(idx_tensor).type_as(pred).to(device) + pred = F.softmax(pred) + degree = torch.sum(pred*idx_tensor, 1) * 3 - 99 + return degree + +def get_rotation_matrix(yaw, pitch, roll): + yaw = yaw / 180 * 3.14 + pitch = pitch / 180 * 3.14 + roll = roll / 180 * 3.14 + + roll = roll.unsqueeze(1) + pitch = pitch.unsqueeze(1) + yaw = yaw.unsqueeze(1) + + pitch_mat = torch.cat([torch.ones_like(pitch), torch.zeros_like(pitch), torch.zeros_like(pitch), + torch.zeros_like(pitch), torch.cos(pitch), -torch.sin(pitch), + torch.zeros_like(pitch), torch.sin(pitch), torch.cos(pitch)], dim=1) + pitch_mat = pitch_mat.view(pitch_mat.shape[0], 3, 3) + + yaw_mat = torch.cat([torch.cos(yaw), torch.zeros_like(yaw), torch.sin(yaw), + torch.zeros_like(yaw), torch.ones_like(yaw), torch.zeros_like(yaw), + -torch.sin(yaw), torch.zeros_like(yaw), torch.cos(yaw)], dim=1) + yaw_mat = yaw_mat.view(yaw_mat.shape[0], 3, 3) + + roll_mat = torch.cat([torch.cos(roll), -torch.sin(roll), torch.zeros_like(roll), + torch.sin(roll), torch.cos(roll), torch.zeros_like(roll), + torch.zeros_like(roll), torch.zeros_like(roll), torch.ones_like(roll)], dim=1) + roll_mat = roll_mat.view(roll_mat.shape[0], 3, 3) + + rot_mat = torch.einsum('bij,bjk,bkm->bim', pitch_mat, yaw_mat, roll_mat) + + return rot_mat + +def keypoint_transformation(kp_canonical, he, wo_exp=False): + kp = kp_canonical['value'] # (bs, k, 3) + yaw, pitch, roll= he['yaw'], he['pitch'], he['roll'] + yaw = headpose_pred_to_degree(yaw) + pitch = headpose_pred_to_degree(pitch) + roll = headpose_pred_to_degree(roll) + + if 'yaw_in' in he: + yaw = he['yaw_in'] + if 'pitch_in' in he: + pitch = he['pitch_in'] + if 'roll_in' in he: + roll = he['roll_in'] + + rot_mat = get_rotation_matrix(yaw, pitch, roll) # (bs, 3, 3) + + t, exp = he['t'], he['exp'] + if wo_exp: + exp = exp*0 + + # keypoint rotation + kp_rotated = torch.einsum('bmp,bkp->bkm', rot_mat, kp) + + # keypoint translation + t[:, 0] = t[:, 0]*0 + t[:, 2] = t[:, 2]*0 + t = t.unsqueeze(1).repeat(1, kp.shape[1], 1) + kp_t = kp_rotated + t + + # add expression deviation + exp = exp.view(exp.shape[0], -1, 3) + kp_transformed = kp_t + exp + + return {'value': kp_transformed} + + + +def make_animation(source_image, source_semantics, target_semantics, + generator, kp_detector, he_estimator, mapping, + yaw_c_seq=None, pitch_c_seq=None, roll_c_seq=None, + use_exp=True, use_half=False): + with torch.no_grad(): + predictions = [] + + kp_canonical = kp_detector(source_image) + he_source = mapping(source_semantics) + kp_source = keypoint_transformation(kp_canonical, he_source) + + for frame_idx in tqdm(range(target_semantics.shape[1]), 'Face Renderer:'): + # still check the dimension + # print(target_semantics.shape, source_semantics.shape) + target_semantics_frame = target_semantics[:, frame_idx] + he_driving = mapping(target_semantics_frame) + if yaw_c_seq is not None: + he_driving['yaw_in'] = yaw_c_seq[:, frame_idx] + if pitch_c_seq is not None: + he_driving['pitch_in'] = pitch_c_seq[:, frame_idx] + if roll_c_seq is not None: + he_driving['roll_in'] = roll_c_seq[:, frame_idx] + + kp_driving = keypoint_transformation(kp_canonical, he_driving) + + kp_norm = kp_driving + out = generator(source_image, kp_source=kp_source, kp_driving=kp_norm) + ''' + source_image_new = out['prediction'].squeeze(1) + kp_canonical_new = kp_detector(source_image_new) + he_source_new = he_estimator(source_image_new) + kp_source_new = keypoint_transformation(kp_canonical_new, he_source_new, wo_exp=True) + kp_driving_new = keypoint_transformation(kp_canonical_new, he_driving, wo_exp=True) + out = generator(source_image_new, kp_source=kp_source_new, kp_driving=kp_driving_new) + ''' + predictions.append(out['prediction']) + predictions_ts = torch.stack(predictions, dim=1) + return predictions_ts + +class AnimateModel(torch.nn.Module): + """ + Merge all generator related updates into single model for better multi-gpu usage + """ + + def __init__(self, generator, kp_extractor, mapping): + super(AnimateModel, self).__init__() + self.kp_extractor = kp_extractor + self.generator = generator + self.mapping = mapping + + self.kp_extractor.eval() + self.generator.eval() + self.mapping.eval() + + def forward(self, x): + + source_image = x['source_image'] + source_semantics = x['source_semantics'] + target_semantics = x['target_semantics'] + yaw_c_seq = x['yaw_c_seq'] + pitch_c_seq = x['pitch_c_seq'] + roll_c_seq = x['roll_c_seq'] + + predictions_video = make_animation(source_image, source_semantics, target_semantics, + self.generator, self.kp_extractor, + self.mapping, use_exp = True, + yaw_c_seq=yaw_c_seq, pitch_c_seq=pitch_c_seq, roll_c_seq=roll_c_seq) + + return predictions_video \ No newline at end of file diff --git a/src/facerender/modules/mapping.py b/src/facerender/modules/mapping.py new file mode 100644 index 0000000000000000000000000000000000000000..0e3a1c2d1770996080c08e9daafb346f05d7bcdd --- /dev/null +++ b/src/facerender/modules/mapping.py @@ -0,0 +1,47 @@ +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class MappingNet(nn.Module): + def __init__(self, coeff_nc, descriptor_nc, layer, num_kp, num_bins): + super( MappingNet, self).__init__() + + self.layer = layer + nonlinearity = nn.LeakyReLU(0.1) + + self.first = nn.Sequential( + torch.nn.Conv1d(coeff_nc, descriptor_nc, kernel_size=7, padding=0, bias=True)) + + for i in range(layer): + net = nn.Sequential(nonlinearity, + torch.nn.Conv1d(descriptor_nc, descriptor_nc, kernel_size=3, padding=0, dilation=3)) + setattr(self, 'encoder' + str(i), net) + + self.pooling = nn.AdaptiveAvgPool1d(1) + self.output_nc = descriptor_nc + + self.fc_roll = nn.Linear(descriptor_nc, num_bins) + self.fc_pitch = nn.Linear(descriptor_nc, num_bins) + self.fc_yaw = nn.Linear(descriptor_nc, num_bins) + self.fc_t = nn.Linear(descriptor_nc, 3) + self.fc_exp = nn.Linear(descriptor_nc, 3*num_kp) + + def forward(self, input_3dmm): + out = self.first(input_3dmm) + for i in range(self.layer): + model = getattr(self, 'encoder' + str(i)) + out = model(out) + out[:,:,3:-3] + out = self.pooling(out) + out = out.view(out.shape[0], -1) + #print('out:', out.shape) + + yaw = self.fc_yaw(out) + pitch = self.fc_pitch(out) + roll = self.fc_roll(out) + t = self.fc_t(out) + exp = self.fc_exp(out) + + return {'yaw': yaw, 'pitch': pitch, 'roll': roll, 't': t, 'exp': exp} \ No newline at end of file diff --git a/src/facerender/modules/util.py b/src/facerender/modules/util.py new file mode 100644 index 0000000000000000000000000000000000000000..b916deefbb8b957ad6ab3cd7403c28513e5ae18e --- /dev/null +++ b/src/facerender/modules/util.py @@ -0,0 +1,564 @@ +from torch import nn + +import torch.nn.functional as F +import torch + +from src.facerender.sync_batchnorm import SynchronizedBatchNorm2d as BatchNorm2d +from src.facerender.sync_batchnorm import SynchronizedBatchNorm3d as BatchNorm3d + +import torch.nn.utils.spectral_norm as spectral_norm + + +def kp2gaussian(kp, spatial_size, kp_variance): + """ + Transform a keypoint into gaussian like representation + """ + mean = kp['value'] + + coordinate_grid = make_coordinate_grid(spatial_size, mean.type()) + number_of_leading_dimensions = len(mean.shape) - 1 + shape = (1,) * number_of_leading_dimensions + coordinate_grid.shape + coordinate_grid = coordinate_grid.view(*shape) + repeats = mean.shape[:number_of_leading_dimensions] + (1, 1, 1, 1) + coordinate_grid = coordinate_grid.repeat(*repeats) + + # Preprocess kp shape + shape = mean.shape[:number_of_leading_dimensions] + (1, 1, 1, 3) + mean = mean.view(*shape) + + mean_sub = (coordinate_grid - mean) + + out = torch.exp(-0.5 * (mean_sub ** 2).sum(-1) / kp_variance) + + return out + +def make_coordinate_grid_2d(spatial_size, type): + """ + Create a meshgrid [-1,1] x [-1,1] of given spatial_size. + """ + h, w = spatial_size + x = torch.arange(w).type(type) + y = torch.arange(h).type(type) + + x = (2 * (x / (w - 1)) - 1) + y = (2 * (y / (h - 1)) - 1) + + yy = y.view(-1, 1).repeat(1, w) + xx = x.view(1, -1).repeat(h, 1) + + meshed = torch.cat([xx.unsqueeze_(2), yy.unsqueeze_(2)], 2) + + return meshed + + +def make_coordinate_grid(spatial_size, type): + d, h, w = spatial_size + x = torch.arange(w).type(type) + y = torch.arange(h).type(type) + z = torch.arange(d).type(type) + + x = (2 * (x / (w - 1)) - 1) + y = (2 * (y / (h - 1)) - 1) + z = (2 * (z / (d - 1)) - 1) + + yy = y.view(1, -1, 1).repeat(d, 1, w) + xx = x.view(1, 1, -1).repeat(d, h, 1) + zz = z.view(-1, 1, 1).repeat(1, h, w) + + meshed = torch.cat([xx.unsqueeze_(3), yy.unsqueeze_(3), zz.unsqueeze_(3)], 3) + + return meshed + + +class ResBottleneck(nn.Module): + def __init__(self, in_features, stride): + super(ResBottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_channels=in_features, out_channels=in_features//4, kernel_size=1) + self.conv2 = nn.Conv2d(in_channels=in_features//4, out_channels=in_features//4, kernel_size=3, padding=1, stride=stride) + self.conv3 = nn.Conv2d(in_channels=in_features//4, out_channels=in_features, kernel_size=1) + self.norm1 = BatchNorm2d(in_features//4, affine=True) + self.norm2 = BatchNorm2d(in_features//4, affine=True) + self.norm3 = BatchNorm2d(in_features, affine=True) + + self.stride = stride + if self.stride != 1: + self.skip = nn.Conv2d(in_channels=in_features, out_channels=in_features, kernel_size=1, stride=stride) + self.norm4 = BatchNorm2d(in_features, affine=True) + + def forward(self, x): + out = self.conv1(x) + out = self.norm1(out) + out = F.relu(out) + out = self.conv2(out) + out = self.norm2(out) + out = F.relu(out) + out = self.conv3(out) + out = self.norm3(out) + if self.stride != 1: + x = self.skip(x) + x = self.norm4(x) + out += x + out = F.relu(out) + return out + + +class ResBlock2d(nn.Module): + """ + Res block, preserve spatial resolution. + """ + + def __init__(self, in_features, kernel_size, padding): + super(ResBlock2d, self).__init__() + self.conv1 = nn.Conv2d(in_channels=in_features, out_channels=in_features, kernel_size=kernel_size, + padding=padding) + self.conv2 = nn.Conv2d(in_channels=in_features, out_channels=in_features, kernel_size=kernel_size, + padding=padding) + self.norm1 = BatchNorm2d(in_features, affine=True) + self.norm2 = BatchNorm2d(in_features, affine=True) + + def forward(self, x): + out = self.norm1(x) + out = F.relu(out) + out = self.conv1(out) + out = self.norm2(out) + out = F.relu(out) + out = self.conv2(out) + out += x + return out + + +class ResBlock3d(nn.Module): + """ + Res block, preserve spatial resolution. + """ + + def __init__(self, in_features, kernel_size, padding): + super(ResBlock3d, self).__init__() + self.conv1 = nn.Conv3d(in_channels=in_features, out_channels=in_features, kernel_size=kernel_size, + padding=padding) + self.conv2 = nn.Conv3d(in_channels=in_features, out_channels=in_features, kernel_size=kernel_size, + padding=padding) + self.norm1 = BatchNorm3d(in_features, affine=True) + self.norm2 = BatchNorm3d(in_features, affine=True) + + def forward(self, x): + out = self.norm1(x) + out = F.relu(out) + out = self.conv1(out) + out = self.norm2(out) + out = F.relu(out) + out = self.conv2(out) + out += x + return out + + +class UpBlock2d(nn.Module): + """ + Upsampling block for use in decoder. + """ + + def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1): + super(UpBlock2d, self).__init__() + + self.conv = nn.Conv2d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, + padding=padding, groups=groups) + self.norm = BatchNorm2d(out_features, affine=True) + + def forward(self, x): + out = F.interpolate(x, scale_factor=2) + out = self.conv(out) + out = self.norm(out) + out = F.relu(out) + return out + +class UpBlock3d(nn.Module): + """ + Upsampling block for use in decoder. + """ + + def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1): + super(UpBlock3d, self).__init__() + + self.conv = nn.Conv3d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, + padding=padding, groups=groups) + self.norm = BatchNorm3d(out_features, affine=True) + + def forward(self, x): + # out = F.interpolate(x, scale_factor=(1, 2, 2), mode='trilinear') + out = F.interpolate(x, scale_factor=(1, 2, 2)) + out = self.conv(out) + out = self.norm(out) + out = F.relu(out) + return out + + +class DownBlock2d(nn.Module): + """ + Downsampling block for use in encoder. + """ + + def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1): + super(DownBlock2d, self).__init__() + self.conv = nn.Conv2d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, + padding=padding, groups=groups) + self.norm = BatchNorm2d(out_features, affine=True) + self.pool = nn.AvgPool2d(kernel_size=(2, 2)) + + def forward(self, x): + out = self.conv(x) + out = self.norm(out) + out = F.relu(out) + out = self.pool(out) + return out + + +class DownBlock3d(nn.Module): + """ + Downsampling block for use in encoder. + """ + + def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1): + super(DownBlock3d, self).__init__() + ''' + self.conv = nn.Conv3d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, + padding=padding, groups=groups, stride=(1, 2, 2)) + ''' + self.conv = nn.Conv3d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, + padding=padding, groups=groups) + self.norm = BatchNorm3d(out_features, affine=True) + self.pool = nn.AvgPool3d(kernel_size=(1, 2, 2)) + + def forward(self, x): + out = self.conv(x) + out = self.norm(out) + out = F.relu(out) + out = self.pool(out) + return out + + +class SameBlock2d(nn.Module): + """ + Simple block, preserve spatial resolution. + """ + + def __init__(self, in_features, out_features, groups=1, kernel_size=3, padding=1, lrelu=False): + super(SameBlock2d, self).__init__() + self.conv = nn.Conv2d(in_channels=in_features, out_channels=out_features, + kernel_size=kernel_size, padding=padding, groups=groups) + self.norm = BatchNorm2d(out_features, affine=True) + if lrelu: + self.ac = nn.LeakyReLU() + else: + self.ac = nn.ReLU() + + def forward(self, x): + out = self.conv(x) + out = self.norm(out) + out = self.ac(out) + return out + + +class Encoder(nn.Module): + """ + Hourglass Encoder + """ + + def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256): + super(Encoder, self).__init__() + + down_blocks = [] + for i in range(num_blocks): + down_blocks.append(DownBlock3d(in_features if i == 0 else min(max_features, block_expansion * (2 ** i)), + min(max_features, block_expansion * (2 ** (i + 1))), + kernel_size=3, padding=1)) + self.down_blocks = nn.ModuleList(down_blocks) + + def forward(self, x): + outs = [x] + for down_block in self.down_blocks: + outs.append(down_block(outs[-1])) + return outs + + +class Decoder(nn.Module): + """ + Hourglass Decoder + """ + + def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256): + super(Decoder, self).__init__() + + up_blocks = [] + + for i in range(num_blocks)[::-1]: + in_filters = (1 if i == num_blocks - 1 else 2) * min(max_features, block_expansion * (2 ** (i + 1))) + out_filters = min(max_features, block_expansion * (2 ** i)) + up_blocks.append(UpBlock3d(in_filters, out_filters, kernel_size=3, padding=1)) + + self.up_blocks = nn.ModuleList(up_blocks) + # self.out_filters = block_expansion + self.out_filters = block_expansion + in_features + + self.conv = nn.Conv3d(in_channels=self.out_filters, out_channels=self.out_filters, kernel_size=3, padding=1) + self.norm = BatchNorm3d(self.out_filters, affine=True) + + def forward(self, x): + out = x.pop() + # for up_block in self.up_blocks[:-1]: + for up_block in self.up_blocks: + out = up_block(out) + skip = x.pop() + out = torch.cat([out, skip], dim=1) + # out = self.up_blocks[-1](out) + out = self.conv(out) + out = self.norm(out) + out = F.relu(out) + return out + + +class Hourglass(nn.Module): + """ + Hourglass architecture. + """ + + def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256): + super(Hourglass, self).__init__() + self.encoder = Encoder(block_expansion, in_features, num_blocks, max_features) + self.decoder = Decoder(block_expansion, in_features, num_blocks, max_features) + self.out_filters = self.decoder.out_filters + + def forward(self, x): + return self.decoder(self.encoder(x)) + + +class KPHourglass(nn.Module): + """ + Hourglass architecture. + """ + + def __init__(self, block_expansion, in_features, reshape_features, reshape_depth, num_blocks=3, max_features=256): + super(KPHourglass, self).__init__() + + self.down_blocks = nn.Sequential() + for i in range(num_blocks): + self.down_blocks.add_module('down'+ str(i), DownBlock2d(in_features if i == 0 else min(max_features, block_expansion * (2 ** i)), + min(max_features, block_expansion * (2 ** (i + 1))), + kernel_size=3, padding=1)) + + in_filters = min(max_features, block_expansion * (2 ** num_blocks)) + self.conv = nn.Conv2d(in_channels=in_filters, out_channels=reshape_features, kernel_size=1) + + self.up_blocks = nn.Sequential() + for i in range(num_blocks): + in_filters = min(max_features, block_expansion * (2 ** (num_blocks - i))) + out_filters = min(max_features, block_expansion * (2 ** (num_blocks - i - 1))) + self.up_blocks.add_module('up'+ str(i), UpBlock3d(in_filters, out_filters, kernel_size=3, padding=1)) + + self.reshape_depth = reshape_depth + self.out_filters = out_filters + + def forward(self, x): + out = self.down_blocks(x) + out = self.conv(out) + bs, c, h, w = out.shape + out = out.view(bs, c//self.reshape_depth, self.reshape_depth, h, w) + out = self.up_blocks(out) + + return out + + + +class AntiAliasInterpolation2d(nn.Module): + """ + Band-limited downsampling, for better preservation of the input signal. + """ + def __init__(self, channels, scale): + super(AntiAliasInterpolation2d, self).__init__() + sigma = (1 / scale - 1) / 2 + kernel_size = 2 * round(sigma * 4) + 1 + self.ka = kernel_size // 2 + self.kb = self.ka - 1 if kernel_size % 2 == 0 else self.ka + + kernel_size = [kernel_size, kernel_size] + sigma = [sigma, sigma] + # The gaussian kernel is the product of the + # gaussian function of each dimension. + kernel = 1 + meshgrids = torch.meshgrid( + [ + torch.arange(size, dtype=torch.float32) + for size in kernel_size + ] + ) + for size, std, mgrid in zip(kernel_size, sigma, meshgrids): + mean = (size - 1) / 2 + kernel *= torch.exp(-(mgrid - mean) ** 2 / (2 * std ** 2)) + + # Make sure sum of values in gaussian kernel equals 1. + kernel = kernel / torch.sum(kernel) + # Reshape to depthwise convolutional weight + kernel = kernel.view(1, 1, *kernel.size()) + kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1)) + + self.register_buffer('weight', kernel) + self.groups = channels + self.scale = scale + inv_scale = 1 / scale + self.int_inv_scale = int(inv_scale) + + def forward(self, input): + if self.scale == 1.0: + return input + + out = F.pad(input, (self.ka, self.kb, self.ka, self.kb)) + out = F.conv2d(out, weight=self.weight, groups=self.groups) + out = out[:, :, ::self.int_inv_scale, ::self.int_inv_scale] + + return out + + +class SPADE(nn.Module): + def __init__(self, norm_nc, label_nc): + super().__init__() + + self.param_free_norm = nn.InstanceNorm2d(norm_nc, affine=False) + nhidden = 128 + + self.mlp_shared = nn.Sequential( + nn.Conv2d(label_nc, nhidden, kernel_size=3, padding=1), + nn.ReLU()) + self.mlp_gamma = nn.Conv2d(nhidden, norm_nc, kernel_size=3, padding=1) + self.mlp_beta = nn.Conv2d(nhidden, norm_nc, kernel_size=3, padding=1) + + def forward(self, x, segmap): + normalized = self.param_free_norm(x) + segmap = F.interpolate(segmap, size=x.size()[2:], mode='nearest') + actv = self.mlp_shared(segmap) + gamma = self.mlp_gamma(actv) + beta = self.mlp_beta(actv) + out = normalized * (1 + gamma) + beta + return out + + +class SPADEResnetBlock(nn.Module): + def __init__(self, fin, fout, norm_G, label_nc, use_se=False, dilation=1): + super().__init__() + # Attributes + self.learned_shortcut = (fin != fout) + fmiddle = min(fin, fout) + self.use_se = use_se + # create conv layers + self.conv_0 = nn.Conv2d(fin, fmiddle, kernel_size=3, padding=dilation, dilation=dilation) + self.conv_1 = nn.Conv2d(fmiddle, fout, kernel_size=3, padding=dilation, dilation=dilation) + if self.learned_shortcut: + self.conv_s = nn.Conv2d(fin, fout, kernel_size=1, bias=False) + # apply spectral norm if specified + if 'spectral' in norm_G: + self.conv_0 = spectral_norm(self.conv_0) + self.conv_1 = spectral_norm(self.conv_1) + if self.learned_shortcut: + self.conv_s = spectral_norm(self.conv_s) + # define normalization layers + self.norm_0 = SPADE(fin, label_nc) + self.norm_1 = SPADE(fmiddle, label_nc) + if self.learned_shortcut: + self.norm_s = SPADE(fin, label_nc) + + def forward(self, x, seg1): + x_s = self.shortcut(x, seg1) + dx = self.conv_0(self.actvn(self.norm_0(x, seg1))) + dx = self.conv_1(self.actvn(self.norm_1(dx, seg1))) + out = x_s + dx + return out + + def shortcut(self, x, seg1): + if self.learned_shortcut: + x_s = self.conv_s(self.norm_s(x, seg1)) + else: + x_s = x + return x_s + + def actvn(self, x): + return F.leaky_relu(x, 2e-1) + +class audio2image(nn.Module): + def __init__(self, generator, kp_extractor, he_estimator_video, he_estimator_audio, train_params): + super().__init__() + # Attributes + self.generator = generator + self.kp_extractor = kp_extractor + self.he_estimator_video = he_estimator_video + self.he_estimator_audio = he_estimator_audio + self.train_params = train_params + + def headpose_pred_to_degree(self, pred): + device = pred.device + idx_tensor = [idx for idx in range(66)] + idx_tensor = torch.FloatTensor(idx_tensor).to(device) + pred = F.softmax(pred) + degree = torch.sum(pred*idx_tensor, 1) * 3 - 99 + + return degree + + def get_rotation_matrix(self, yaw, pitch, roll): + yaw = yaw / 180 * 3.14 + pitch = pitch / 180 * 3.14 + roll = roll / 180 * 3.14 + + roll = roll.unsqueeze(1) + pitch = pitch.unsqueeze(1) + yaw = yaw.unsqueeze(1) + + roll_mat = torch.cat([torch.ones_like(roll), torch.zeros_like(roll), torch.zeros_like(roll), + torch.zeros_like(roll), torch.cos(roll), -torch.sin(roll), + torch.zeros_like(roll), torch.sin(roll), torch.cos(roll)], dim=1) + roll_mat = roll_mat.view(roll_mat.shape[0], 3, 3) + + pitch_mat = torch.cat([torch.cos(pitch), torch.zeros_like(pitch), torch.sin(pitch), + torch.zeros_like(pitch), torch.ones_like(pitch), torch.zeros_like(pitch), + -torch.sin(pitch), torch.zeros_like(pitch), torch.cos(pitch)], dim=1) + pitch_mat = pitch_mat.view(pitch_mat.shape[0], 3, 3) + + yaw_mat = torch.cat([torch.cos(yaw), -torch.sin(yaw), torch.zeros_like(yaw), + torch.sin(yaw), torch.cos(yaw), torch.zeros_like(yaw), + torch.zeros_like(yaw), torch.zeros_like(yaw), torch.ones_like(yaw)], dim=1) + yaw_mat = yaw_mat.view(yaw_mat.shape[0], 3, 3) + + rot_mat = torch.einsum('bij,bjk,bkm->bim', roll_mat, pitch_mat, yaw_mat) + + return rot_mat + + def keypoint_transformation(self, kp_canonical, he): + kp = kp_canonical['value'] # (bs, k, 3) + yaw, pitch, roll = he['yaw'], he['pitch'], he['roll'] + t, exp = he['t'], he['exp'] + + yaw = self.headpose_pred_to_degree(yaw) + pitch = self.headpose_pred_to_degree(pitch) + roll = self.headpose_pred_to_degree(roll) + + rot_mat = self.get_rotation_matrix(yaw, pitch, roll) # (bs, 3, 3) + + # keypoint rotation + kp_rotated = torch.einsum('bmp,bkp->bkm', rot_mat, kp) + + + + # keypoint translation + t = t.unsqueeze_(1).repeat(1, kp.shape[1], 1) + kp_t = kp_rotated + t + + # add expression deviation + exp = exp.view(exp.shape[0], -1, 3) + kp_transformed = kp_t + exp + + return {'value': kp_transformed} + + def forward(self, source_image, target_audio): + pose_source = self.he_estimator_video(source_image) + pose_generated = self.he_estimator_audio(target_audio) + kp_canonical = self.kp_extractor(source_image) + kp_source = self.keypoint_transformation(kp_canonical, pose_source) + kp_transformed_generated = self.keypoint_transformation(kp_canonical, pose_generated) + generated = self.generator(source_image, kp_source=kp_source, kp_driving=kp_transformed_generated) + return generated \ No newline at end of file diff --git a/src/facerender/sync_batchnorm/__init__.py b/src/facerender/sync_batchnorm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bc8709d92c610b36e0bcbd7da20c1eb41dc8cfcf --- /dev/null +++ b/src/facerender/sync_batchnorm/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# File : __init__.py +# Author : Jiayuan Mao +# Email : maojiayuan@gmail.com +# Date : 27/01/2018 +# +# This file is part of Synchronized-BatchNorm-PyTorch. +# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch +# Distributed under MIT License. + +from .batchnorm import SynchronizedBatchNorm1d, SynchronizedBatchNorm2d, SynchronizedBatchNorm3d +from .replicate import DataParallelWithCallback, patch_replication_callback diff --git a/src/facerender/sync_batchnorm/__pycache__/__init__.cpython-310.pyc b/src/facerender/sync_batchnorm/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e9a5ab4f86f72d8bbfcbd8421e8ed5d2271e3e3 Binary files /dev/null and b/src/facerender/sync_batchnorm/__pycache__/__init__.cpython-310.pyc differ diff --git a/src/facerender/sync_batchnorm/__pycache__/batchnorm.cpython-310.pyc b/src/facerender/sync_batchnorm/__pycache__/batchnorm.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66b01e4f79a7accf32e7933fc4960dbbd9cafa88 Binary files /dev/null and b/src/facerender/sync_batchnorm/__pycache__/batchnorm.cpython-310.pyc differ diff --git a/src/facerender/sync_batchnorm/__pycache__/comm.cpython-310.pyc b/src/facerender/sync_batchnorm/__pycache__/comm.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8a68d349ae8ed8e8ab09c0c728fff3688e04d61 Binary files /dev/null and b/src/facerender/sync_batchnorm/__pycache__/comm.cpython-310.pyc differ diff --git a/src/facerender/sync_batchnorm/__pycache__/replicate.cpython-310.pyc b/src/facerender/sync_batchnorm/__pycache__/replicate.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ddaf235337988ed9c813e4e3fe9376117a9f960a Binary files /dev/null and b/src/facerender/sync_batchnorm/__pycache__/replicate.cpython-310.pyc differ diff --git a/src/facerender/sync_batchnorm/batchnorm.py b/src/facerender/sync_batchnorm/batchnorm.py new file mode 100644 index 0000000000000000000000000000000000000000..5f4e763f0366dffa10320116413f8c7181a8aeb1 --- /dev/null +++ b/src/facerender/sync_batchnorm/batchnorm.py @@ -0,0 +1,315 @@ +# -*- coding: utf-8 -*- +# File : batchnorm.py +# Author : Jiayuan Mao +# Email : maojiayuan@gmail.com +# Date : 27/01/2018 +# +# This file is part of Synchronized-BatchNorm-PyTorch. +# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch +# Distributed under MIT License. + +import collections + +import torch +import torch.nn.functional as F + +from torch.nn.modules.batchnorm import _BatchNorm +from torch.nn.parallel._functions import ReduceAddCoalesced, Broadcast + +from .comm import SyncMaster + +__all__ = ['SynchronizedBatchNorm1d', 'SynchronizedBatchNorm2d', 'SynchronizedBatchNorm3d'] + + +def _sum_ft(tensor): + """sum over the first and last dimention""" + return tensor.sum(dim=0).sum(dim=-1) + + +def _unsqueeze_ft(tensor): + """add new dementions at the front and the tail""" + return tensor.unsqueeze(0).unsqueeze(-1) + + +_ChildMessage = collections.namedtuple('_ChildMessage', ['sum', 'ssum', 'sum_size']) +_MasterMessage = collections.namedtuple('_MasterMessage', ['sum', 'inv_std']) + + +class _SynchronizedBatchNorm(_BatchNorm): + def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True): + super(_SynchronizedBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine) + + self._sync_master = SyncMaster(self._data_parallel_master) + + self._is_parallel = False + self._parallel_id = None + self._slave_pipe = None + + def forward(self, input): + # If it is not parallel computation or is in evaluation mode, use PyTorch's implementation. + if not (self._is_parallel and self.training): + return F.batch_norm( + input, self.running_mean, self.running_var, self.weight, self.bias, + self.training, self.momentum, self.eps) + + # Resize the input to (B, C, -1). + input_shape = input.size() + input = input.view(input.size(0), self.num_features, -1) + + # Compute the sum and square-sum. + sum_size = input.size(0) * input.size(2) + input_sum = _sum_ft(input) + input_ssum = _sum_ft(input ** 2) + + # Reduce-and-broadcast the statistics. + if self._parallel_id == 0: + mean, inv_std = self._sync_master.run_master(_ChildMessage(input_sum, input_ssum, sum_size)) + else: + mean, inv_std = self._slave_pipe.run_slave(_ChildMessage(input_sum, input_ssum, sum_size)) + + # Compute the output. + if self.affine: + # MJY:: Fuse the multiplication for speed. + output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std * self.weight) + _unsqueeze_ft(self.bias) + else: + output = (input - _unsqueeze_ft(mean)) * _unsqueeze_ft(inv_std) + + # Reshape it. + return output.view(input_shape) + + def __data_parallel_replicate__(self, ctx, copy_id): + self._is_parallel = True + self._parallel_id = copy_id + + # parallel_id == 0 means master device. + if self._parallel_id == 0: + ctx.sync_master = self._sync_master + else: + self._slave_pipe = ctx.sync_master.register_slave(copy_id) + + def _data_parallel_master(self, intermediates): + """Reduce the sum and square-sum, compute the statistics, and broadcast it.""" + + # Always using same "device order" makes the ReduceAdd operation faster. + # Thanks to:: Tete Xiao (http://tetexiao.com/) + intermediates = sorted(intermediates, key=lambda i: i[1].sum.get_device()) + + to_reduce = [i[1][:2] for i in intermediates] + to_reduce = [j for i in to_reduce for j in i] # flatten + target_gpus = [i[1].sum.get_device() for i in intermediates] + + sum_size = sum([i[1].sum_size for i in intermediates]) + sum_, ssum = ReduceAddCoalesced.apply(target_gpus[0], 2, *to_reduce) + mean, inv_std = self._compute_mean_std(sum_, ssum, sum_size) + + broadcasted = Broadcast.apply(target_gpus, mean, inv_std) + + outputs = [] + for i, rec in enumerate(intermediates): + outputs.append((rec[0], _MasterMessage(*broadcasted[i*2:i*2+2]))) + + return outputs + + def _compute_mean_std(self, sum_, ssum, size): + """Compute the mean and standard-deviation with sum and square-sum. This method + also maintains the moving average on the master device.""" + assert size > 1, 'BatchNorm computes unbiased standard-deviation, which requires size > 1.' + mean = sum_ / size + sumvar = ssum - sum_ * mean + unbias_var = sumvar / (size - 1) + bias_var = sumvar / size + + self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.data + self.running_var = (1 - self.momentum) * self.running_var + self.momentum * unbias_var.data + + return mean, bias_var.clamp(self.eps) ** -0.5 + + +class SynchronizedBatchNorm1d(_SynchronizedBatchNorm): + r"""Applies Synchronized Batch Normalization over a 2d or 3d input that is seen as a + mini-batch. + + .. math:: + + y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta + + This module differs from the built-in PyTorch BatchNorm1d as the mean and + standard-deviation are reduced across all devices during training. + + For example, when one uses `nn.DataParallel` to wrap the network during + training, PyTorch's implementation normalize the tensor on each device using + the statistics only on that device, which accelerated the computation and + is also easy to implement, but the statistics might be inaccurate. + Instead, in this synchronized version, the statistics will be computed + over all training samples distributed on multiple devices. + + Note that, for one-GPU or CPU-only case, this module behaves exactly same + as the built-in PyTorch implementation. + + The mean and standard-deviation are calculated per-dimension over + the mini-batches and gamma and beta are learnable parameter vectors + of size C (where C is the input size). + + During training, this layer keeps a running estimate of its computed mean + and variance. The running sum is kept with a default momentum of 0.1. + + During evaluation, this running mean/variance is used for normalization. + + Because the BatchNorm is done over the `C` dimension, computing statistics + on `(N, L)` slices, it's common terminology to call this Temporal BatchNorm + + Args: + num_features: num_features from an expected input of size + `batch_size x num_features [x width]` + eps: a value added to the denominator for numerical stability. + Default: 1e-5 + momentum: the value used for the running_mean and running_var + computation. Default: 0.1 + affine: a boolean value that when set to ``True``, gives the layer learnable + affine parameters. Default: ``True`` + + Shape: + - Input: :math:`(N, C)` or :math:`(N, C, L)` + - Output: :math:`(N, C)` or :math:`(N, C, L)` (same shape as input) + + Examples: + >>> # With Learnable Parameters + >>> m = SynchronizedBatchNorm1d(100) + >>> # Without Learnable Parameters + >>> m = SynchronizedBatchNorm1d(100, affine=False) + >>> input = torch.autograd.Variable(torch.randn(20, 100)) + >>> output = m(input) + """ + + def _check_input_dim(self, input): + if input.dim() != 2 and input.dim() != 3: + raise ValueError('expected 2D or 3D input (got {}D input)' + .format(input.dim())) + super(SynchronizedBatchNorm1d, self)._check_input_dim(input) + + +class SynchronizedBatchNorm2d(_SynchronizedBatchNorm): + r"""Applies Batch Normalization over a 4d input that is seen as a mini-batch + of 3d inputs + + .. math:: + + y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta + + This module differs from the built-in PyTorch BatchNorm2d as the mean and + standard-deviation are reduced across all devices during training. + + For example, when one uses `nn.DataParallel` to wrap the network during + training, PyTorch's implementation normalize the tensor on each device using + the statistics only on that device, which accelerated the computation and + is also easy to implement, but the statistics might be inaccurate. + Instead, in this synchronized version, the statistics will be computed + over all training samples distributed on multiple devices. + + Note that, for one-GPU or CPU-only case, this module behaves exactly same + as the built-in PyTorch implementation. + + The mean and standard-deviation are calculated per-dimension over + the mini-batches and gamma and beta are learnable parameter vectors + of size C (where C is the input size). + + During training, this layer keeps a running estimate of its computed mean + and variance. The running sum is kept with a default momentum of 0.1. + + During evaluation, this running mean/variance is used for normalization. + + Because the BatchNorm is done over the `C` dimension, computing statistics + on `(N, H, W)` slices, it's common terminology to call this Spatial BatchNorm + + Args: + num_features: num_features from an expected input of + size batch_size x num_features x height x width + eps: a value added to the denominator for numerical stability. + Default: 1e-5 + momentum: the value used for the running_mean and running_var + computation. Default: 0.1 + affine: a boolean value that when set to ``True``, gives the layer learnable + affine parameters. Default: ``True`` + + Shape: + - Input: :math:`(N, C, H, W)` + - Output: :math:`(N, C, H, W)` (same shape as input) + + Examples: + >>> # With Learnable Parameters + >>> m = SynchronizedBatchNorm2d(100) + >>> # Without Learnable Parameters + >>> m = SynchronizedBatchNorm2d(100, affine=False) + >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45)) + >>> output = m(input) + """ + + def _check_input_dim(self, input): + if input.dim() != 4: + raise ValueError('expected 4D input (got {}D input)' + .format(input.dim())) + super(SynchronizedBatchNorm2d, self)._check_input_dim(input) + + +class SynchronizedBatchNorm3d(_SynchronizedBatchNorm): + r"""Applies Batch Normalization over a 5d input that is seen as a mini-batch + of 4d inputs + + .. math:: + + y = \frac{x - mean[x]}{ \sqrt{Var[x] + \epsilon}} * gamma + beta + + This module differs from the built-in PyTorch BatchNorm3d as the mean and + standard-deviation are reduced across all devices during training. + + For example, when one uses `nn.DataParallel` to wrap the network during + training, PyTorch's implementation normalize the tensor on each device using + the statistics only on that device, which accelerated the computation and + is also easy to implement, but the statistics might be inaccurate. + Instead, in this synchronized version, the statistics will be computed + over all training samples distributed on multiple devices. + + Note that, for one-GPU or CPU-only case, this module behaves exactly same + as the built-in PyTorch implementation. + + The mean and standard-deviation are calculated per-dimension over + the mini-batches and gamma and beta are learnable parameter vectors + of size C (where C is the input size). + + During training, this layer keeps a running estimate of its computed mean + and variance. The running sum is kept with a default momentum of 0.1. + + During evaluation, this running mean/variance is used for normalization. + + Because the BatchNorm is done over the `C` dimension, computing statistics + on `(N, D, H, W)` slices, it's common terminology to call this Volumetric BatchNorm + or Spatio-temporal BatchNorm + + Args: + num_features: num_features from an expected input of + size batch_size x num_features x depth x height x width + eps: a value added to the denominator for numerical stability. + Default: 1e-5 + momentum: the value used for the running_mean and running_var + computation. Default: 0.1 + affine: a boolean value that when set to ``True``, gives the layer learnable + affine parameters. Default: ``True`` + + Shape: + - Input: :math:`(N, C, D, H, W)` + - Output: :math:`(N, C, D, H, W)` (same shape as input) + + Examples: + >>> # With Learnable Parameters + >>> m = SynchronizedBatchNorm3d(100) + >>> # Without Learnable Parameters + >>> m = SynchronizedBatchNorm3d(100, affine=False) + >>> input = torch.autograd.Variable(torch.randn(20, 100, 35, 45, 10)) + >>> output = m(input) + """ + + def _check_input_dim(self, input): + if input.dim() != 5: + raise ValueError('expected 5D input (got {}D input)' + .format(input.dim())) + super(SynchronizedBatchNorm3d, self)._check_input_dim(input) diff --git a/src/facerender/sync_batchnorm/comm.py b/src/facerender/sync_batchnorm/comm.py new file mode 100644 index 0000000000000000000000000000000000000000..922f8c4a3adaa9b32fdcaef09583be03b0d7eb2b --- /dev/null +++ b/src/facerender/sync_batchnorm/comm.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# File : comm.py +# Author : Jiayuan Mao +# Email : maojiayuan@gmail.com +# Date : 27/01/2018 +# +# This file is part of Synchronized-BatchNorm-PyTorch. +# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch +# Distributed under MIT License. + +import queue +import collections +import threading + +__all__ = ['FutureResult', 'SlavePipe', 'SyncMaster'] + + +class FutureResult(object): + """A thread-safe future implementation. Used only as one-to-one pipe.""" + + def __init__(self): + self._result = None + self._lock = threading.Lock() + self._cond = threading.Condition(self._lock) + + def put(self, result): + with self._lock: + assert self._result is None, 'Previous result has\'t been fetched.' + self._result = result + self._cond.notify() + + def get(self): + with self._lock: + if self._result is None: + self._cond.wait() + + res = self._result + self._result = None + return res + + +_MasterRegistry = collections.namedtuple('MasterRegistry', ['result']) +_SlavePipeBase = collections.namedtuple('_SlavePipeBase', ['identifier', 'queue', 'result']) + + +class SlavePipe(_SlavePipeBase): + """Pipe for master-slave communication.""" + + def run_slave(self, msg): + self.queue.put((self.identifier, msg)) + ret = self.result.get() + self.queue.put(True) + return ret + + +class SyncMaster(object): + """An abstract `SyncMaster` object. + + - During the replication, as the data parallel will trigger an callback of each module, all slave devices should + call `register(id)` and obtain an `SlavePipe` to communicate with the master. + - During the forward pass, master device invokes `run_master`, all messages from slave devices will be collected, + and passed to a registered callback. + - After receiving the messages, the master device should gather the information and determine to message passed + back to each slave devices. + """ + + def __init__(self, master_callback): + """ + + Args: + master_callback: a callback to be invoked after having collected messages from slave devices. + """ + self._master_callback = master_callback + self._queue = queue.Queue() + self._registry = collections.OrderedDict() + self._activated = False + + def __getstate__(self): + return {'master_callback': self._master_callback} + + def __setstate__(self, state): + self.__init__(state['master_callback']) + + def register_slave(self, identifier): + """ + Register an slave device. + + Args: + identifier: an identifier, usually is the device id. + + Returns: a `SlavePipe` object which can be used to communicate with the master device. + + """ + if self._activated: + assert self._queue.empty(), 'Queue is not clean before next initialization.' + self._activated = False + self._registry.clear() + future = FutureResult() + self._registry[identifier] = _MasterRegistry(future) + return SlavePipe(identifier, self._queue, future) + + def run_master(self, master_msg): + """ + Main entry for the master device in each forward pass. + The messages were first collected from each devices (including the master device), and then + an callback will be invoked to compute the message to be sent back to each devices + (including the master device). + + Args: + master_msg: the message that the master want to send to itself. This will be placed as the first + message when calling `master_callback`. For detailed usage, see `_SynchronizedBatchNorm` for an example. + + Returns: the message to be sent back to the master device. + + """ + self._activated = True + + intermediates = [(0, master_msg)] + for i in range(self.nr_slaves): + intermediates.append(self._queue.get()) + + results = self._master_callback(intermediates) + assert results[0][0] == 0, 'The first result should belongs to the master.' + + for i, res in results: + if i == 0: + continue + self._registry[i].result.put(res) + + for i in range(self.nr_slaves): + assert self._queue.get() is True + + return results[0][1] + + @property + def nr_slaves(self): + return len(self._registry) diff --git a/src/facerender/sync_batchnorm/replicate.py b/src/facerender/sync_batchnorm/replicate.py new file mode 100644 index 0000000000000000000000000000000000000000..b71c7b8ed51a1d6c55b1f753bdd8d90bad79bd06 --- /dev/null +++ b/src/facerender/sync_batchnorm/replicate.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# File : replicate.py +# Author : Jiayuan Mao +# Email : maojiayuan@gmail.com +# Date : 27/01/2018 +# +# This file is part of Synchronized-BatchNorm-PyTorch. +# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch +# Distributed under MIT License. + +import functools + +from torch.nn.parallel.data_parallel import DataParallel + +__all__ = [ + 'CallbackContext', + 'execute_replication_callbacks', + 'DataParallelWithCallback', + 'patch_replication_callback' +] + + +class CallbackContext(object): + pass + + +def execute_replication_callbacks(modules): + """ + Execute an replication callback `__data_parallel_replicate__` on each module created by original replication. + + The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` + + Note that, as all modules are isomorphism, we assign each sub-module with a context + (shared among multiple copies of this module on different devices). + Through this context, different copies can share some information. + + We guarantee that the callback on the master copy (the first copy) will be called ahead of calling the callback + of any slave copies. + """ + master_copy = modules[0] + nr_modules = len(list(master_copy.modules())) + ctxs = [CallbackContext() for _ in range(nr_modules)] + + for i, module in enumerate(modules): + for j, m in enumerate(module.modules()): + if hasattr(m, '__data_parallel_replicate__'): + m.__data_parallel_replicate__(ctxs[j], i) + + +class DataParallelWithCallback(DataParallel): + """ + Data Parallel with a replication callback. + + An replication callback `__data_parallel_replicate__` of each module will be invoked after being created by + original `replicate` function. + The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` + + Examples: + > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) + > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) + # sync_bn.__data_parallel_replicate__ will be invoked. + """ + + def replicate(self, module, device_ids): + modules = super(DataParallelWithCallback, self).replicate(module, device_ids) + execute_replication_callbacks(modules) + return modules + + +def patch_replication_callback(data_parallel): + """ + Monkey-patch an existing `DataParallel` object. Add the replication callback. + Useful when you have customized `DataParallel` implementation. + + Examples: + > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) + > sync_bn = DataParallel(sync_bn, device_ids=[0, 1]) + > patch_replication_callback(sync_bn) + # this is equivalent to + > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) + > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) + """ + + assert isinstance(data_parallel, DataParallel) + + old_replicate = data_parallel.replicate + + @functools.wraps(old_replicate) + def new_replicate(module, device_ids): + modules = old_replicate(module, device_ids) + execute_replication_callbacks(modules) + return modules + + data_parallel.replicate = new_replicate diff --git a/src/facerender/sync_batchnorm/unittest.py b/src/facerender/sync_batchnorm/unittest.py new file mode 100644 index 0000000000000000000000000000000000000000..0675c022e4ba85d38d1f813490f6740150909524 --- /dev/null +++ b/src/facerender/sync_batchnorm/unittest.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# File : unittest.py +# Author : Jiayuan Mao +# Email : maojiayuan@gmail.com +# Date : 27/01/2018 +# +# This file is part of Synchronized-BatchNorm-PyTorch. +# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch +# Distributed under MIT License. + +import unittest + +import numpy as np +from torch.autograd import Variable + + +def as_numpy(v): + if isinstance(v, Variable): + v = v.data + return v.cpu().numpy() + + +class TorchTestCase(unittest.TestCase): + def assertTensorClose(self, a, b, atol=1e-3, rtol=1e-3): + npa, npb = as_numpy(a), as_numpy(b) + self.assertTrue( + np.allclose(npa, npb, atol=atol), + 'Tensor close check failed\n{}\n{}\nadiff={}, rdiff={}'.format(a, b, np.abs(npa - npb).max(), np.abs((npa - npb) / np.fmax(npa, 1e-5)).max()) + ) diff --git a/src/generate_batch.py b/src/generate_batch.py new file mode 100644 index 0000000000000000000000000000000000000000..95f21526feea846977707e97394132d43225c02a --- /dev/null +++ b/src/generate_batch.py @@ -0,0 +1,120 @@ +import os + +from tqdm import tqdm +import torch +import numpy as np +import random +import scipy.io as scio +import src.utils.audio as audio + +def crop_pad_audio(wav, audio_length): + if len(wav) > audio_length: + wav = wav[:audio_length] + elif len(wav) < audio_length: + wav = np.pad(wav, [0, audio_length - len(wav)], mode='constant', constant_values=0) + return wav + +def parse_audio_length(audio_length, sr, fps): + bit_per_frames = sr / fps + + num_frames = int(audio_length / bit_per_frames) + audio_length = int(num_frames * bit_per_frames) + + return audio_length, num_frames + +def generate_blink_seq(num_frames): + ratio = np.zeros((num_frames,1)) + frame_id = 0 + while frame_id in range(num_frames): + start = 80 + if frame_id+start+9<=num_frames - 1: + ratio[frame_id+start:frame_id+start+9, 0] = [0.5,0.6,0.7,0.9,1, 0.9, 0.7,0.6,0.5] + frame_id = frame_id+start+9 + else: + break + return ratio + +def generate_blink_seq_randomly(num_frames): + ratio = np.zeros((num_frames,1)) + if num_frames<=20: + return ratio + frame_id = 0 + while frame_id in range(num_frames): + start = random.choice(range(min(10,num_frames), min(int(num_frames/2), 70))) + if frame_id+start+5<=num_frames - 1: + ratio[frame_id+start:frame_id+start+5, 0] = [0.5, 0.9, 1.0, 0.9, 0.5] + frame_id = frame_id+start+5 + else: + break + return ratio + +def get_data(first_coeff_path, audio_path, device, ref_eyeblink_coeff_path, still=False, idlemode=False, length_of_audio=False, use_blink=True): + + syncnet_mel_step_size = 16 + fps = 25 + + pic_name = os.path.splitext(os.path.split(first_coeff_path)[-1])[0] + audio_name = os.path.splitext(os.path.split(audio_path)[-1])[0] + + + if idlemode: + num_frames = int(length_of_audio * 25) + indiv_mels = np.zeros((num_frames, 80, 16)) + else: + wav = audio.load_wav(audio_path, 16000) + wav_length, num_frames = parse_audio_length(len(wav), 16000, 25) + wav = crop_pad_audio(wav, wav_length) + orig_mel = audio.melspectrogram(wav).T + spec = orig_mel.copy() # nframes 80 + indiv_mels = [] + + for i in tqdm(range(num_frames), 'mel:'): + start_frame_num = i-2 + start_idx = int(80. * (start_frame_num / float(fps))) + end_idx = start_idx + syncnet_mel_step_size + seq = list(range(start_idx, end_idx)) + seq = [ min(max(item, 0), orig_mel.shape[0]-1) for item in seq ] + m = spec[seq, :] + indiv_mels.append(m.T) + indiv_mels = np.asarray(indiv_mels) # T 80 16 + + ratio = generate_blink_seq_randomly(num_frames) # T + source_semantics_path = first_coeff_path + source_semantics_dict = scio.loadmat(source_semantics_path) + ref_coeff = source_semantics_dict['coeff_3dmm'][:1,:70] #1 70 + ref_coeff = np.repeat(ref_coeff, num_frames, axis=0) + + if ref_eyeblink_coeff_path is not None: + ratio[:num_frames] = 0 + refeyeblink_coeff_dict = scio.loadmat(ref_eyeblink_coeff_path) + refeyeblink_coeff = refeyeblink_coeff_dict['coeff_3dmm'][:,:64] + refeyeblink_num_frames = refeyeblink_coeff.shape[0] + if refeyeblink_num_frames frame_num: + new_degree_list = new_degree_list[:frame_num] + elif len(new_degree_list) < frame_num: + for _ in range(frame_num-len(new_degree_list)): + new_degree_list.append(new_degree_list[-1]) + print(len(new_degree_list)) + print(frame_num) + + remainder = frame_num%batch_size + if remainder!=0: + for _ in range(batch_size-remainder): + new_degree_list.append(new_degree_list[-1]) + new_degree_np = np.array(new_degree_list).reshape(batch_size, -1) + return new_degree_np + diff --git a/src/gradio_demo.py b/src/gradio_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..1e70005831b9f29dc3c7f39642364bc325a4c8a4 --- /dev/null +++ b/src/gradio_demo.py @@ -0,0 +1,155 @@ +import torch, uuid +import os, sys, shutil +from src.utils.preprocess import CropAndExtract +from src.test_audio2coeff import Audio2Coeff +from src.facerender.animate import AnimateFromCoeff +from src.generate_batch import get_data +from src.generate_facerender_batch import get_facerender_data + +from src.utils.init_path import init_path + +from pydub import AudioSegment + + +def mp3_to_wav(mp3_filename,wav_filename,frame_rate): + mp3_file = AudioSegment.from_file(file=mp3_filename) + mp3_file.set_frame_rate(frame_rate).export(wav_filename,format="wav") + + +class SadTalker(): + + def __init__(self, checkpoint_path='checkpoints', config_path='src/config', lazy_load=False): + + if torch.cuda.is_available() : + device = "cuda" + else: + device = "cpu" + + self.device = device + + os.environ['TORCH_HOME']= checkpoint_path + + self.checkpoint_path = checkpoint_path + self.config_path = config_path + + + def test(self, source_image, driven_audio, preprocess='crop', + still_mode=False, use_enhancer=False, batch_size=1, size=256, + pose_style = 0, exp_scale=1.0, + use_ref_video = False, + ref_video = None, + ref_info = None, + use_idle_mode = False, + length_of_audio = 0, use_blink=True, + result_dir='./results/'): + + self.sadtalker_paths = init_path(self.checkpoint_path, self.config_path, size, False, preprocess) + print(self.sadtalker_paths) + + self.audio_to_coeff = Audio2Coeff(self.sadtalker_paths, self.device) + self.preprocess_model = CropAndExtract(self.sadtalker_paths, self.device) + self.animate_from_coeff = AnimateFromCoeff(self.sadtalker_paths, self.device) + + time_tag = str(uuid.uuid4()) + save_dir = os.path.join(result_dir, time_tag) + os.makedirs(save_dir, exist_ok=True) + + input_dir = os.path.join(save_dir, 'input') + os.makedirs(input_dir, exist_ok=True) + + print(source_image) + pic_path = os.path.join(input_dir, os.path.basename(source_image)) + shutil.move(source_image, input_dir) + + if driven_audio is not None and os.path.isfile(driven_audio): + audio_path = os.path.join(input_dir, os.path.basename(driven_audio)) + + #### mp3 to wav + if '.mp3' in audio_path: + mp3_to_wav(driven_audio, audio_path.replace('.mp3', '.wav'), 16000) + audio_path = audio_path.replace('.mp3', '.wav') + else: + shutil.move(driven_audio, input_dir) + + elif use_idle_mode: + audio_path = os.path.join(input_dir, 'idlemode_'+str(length_of_audio)+'.wav') ## generate audio from this new audio_path + from pydub import AudioSegment + one_sec_segment = AudioSegment.silent(duration=1000*length_of_audio) #duration in milliseconds + one_sec_segment.export(audio_path, format="wav") + else: + print(use_ref_video, ref_info) + assert use_ref_video == True and ref_info == 'all' + + if use_ref_video and ref_info == 'all': # full ref mode + ref_video_videoname = os.path.basename(ref_video) + audio_path = os.path.join(save_dir, ref_video_videoname+'.wav') + print('new audiopath:',audio_path) + # if ref_video contains audio, set the audio from ref_video. + cmd = r"ffmpeg -y -hide_banner -loglevel error -i %s %s"%(ref_video, audio_path) + os.system(cmd) + + os.makedirs(save_dir, exist_ok=True) + + #crop image and extract 3dmm from image + first_frame_dir = os.path.join(save_dir, 'first_frame_dir') + os.makedirs(first_frame_dir, exist_ok=True) + first_coeff_path, crop_pic_path, crop_info = self.preprocess_model.generate(pic_path, first_frame_dir, preprocess, True, size) + + if first_coeff_path is None: + raise AttributeError("No face is detected") + + if use_ref_video: + print('using ref video for genreation') + ref_video_videoname = os.path.splitext(os.path.split(ref_video)[-1])[0] + ref_video_frame_dir = os.path.join(save_dir, ref_video_videoname) + os.makedirs(ref_video_frame_dir, exist_ok=True) + print('3DMM Extraction for the reference video providing pose') + ref_video_coeff_path, _, _ = self.preprocess_model.generate(ref_video, ref_video_frame_dir, preprocess, source_image_flag=False) + else: + ref_video_coeff_path = None + + if use_ref_video: + if ref_info == 'pose': + ref_pose_coeff_path = ref_video_coeff_path + ref_eyeblink_coeff_path = None + elif ref_info == 'blink': + ref_pose_coeff_path = None + ref_eyeblink_coeff_path = ref_video_coeff_path + elif ref_info == 'pose+blink': + ref_pose_coeff_path = ref_video_coeff_path + ref_eyeblink_coeff_path = ref_video_coeff_path + elif ref_info == 'all': + ref_pose_coeff_path = None + ref_eyeblink_coeff_path = None + else: + raise('error in refinfo') + else: + ref_pose_coeff_path = None + ref_eyeblink_coeff_path = None + + #audio2ceoff + if use_ref_video and ref_info == 'all': + coeff_path = ref_video_coeff_path # self.audio_to_coeff.generate(batch, save_dir, pose_style, ref_pose_coeff_path) + else: + batch = get_data(first_coeff_path, audio_path, self.device, ref_eyeblink_coeff_path=ref_eyeblink_coeff_path, still=still_mode, idlemode=use_idle_mode, length_of_audio=length_of_audio, use_blink=use_blink) # longer audio? + coeff_path = self.audio_to_coeff.generate(batch, save_dir, pose_style, ref_pose_coeff_path) + + #coeff2video + data = get_facerender_data(coeff_path, crop_pic_path, first_coeff_path, audio_path, batch_size, still_mode=still_mode, preprocess=preprocess, size=size, expression_scale = exp_scale) + return_path = self.animate_from_coeff.generate(data, save_dir, pic_path, crop_info, enhancer='gfpgan' if use_enhancer else None, preprocess=preprocess, img_size=size) + video_name = data['video_name'] + print(f'The generated video is named {video_name} in {save_dir}') + + del self.preprocess_model + del self.audio_to_coeff + del self.animate_from_coeff + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.synchronize() + + import gc; gc.collect() + + return return_path + + \ No newline at end of file diff --git a/src/test_audio2coeff.py b/src/test_audio2coeff.py new file mode 100644 index 0000000000000000000000000000000000000000..bbf19f494e2127b4ae9d6074b172fddb694d6e34 --- /dev/null +++ b/src/test_audio2coeff.py @@ -0,0 +1,123 @@ +import os +import torch +import numpy as np +from scipy.io import savemat, loadmat +from yacs.config import CfgNode as CN +from scipy.signal import savgol_filter + +import safetensors +import safetensors.torch + +from src.audio2pose_models.audio2pose import Audio2Pose +from src.audio2exp_models.networks import SimpleWrapperV2 +from src.audio2exp_models.audio2exp import Audio2Exp +from src.utils.safetensor_helper import load_x_from_safetensor + +def load_cpk(checkpoint_path, model=None, optimizer=None, device="cpu"): + checkpoint = torch.load(checkpoint_path, map_location=torch.device(device)) + if model is not None: + model.load_state_dict(checkpoint['model']) + if optimizer is not None: + optimizer.load_state_dict(checkpoint['optimizer']) + + return checkpoint['epoch'] + +class Audio2Coeff(): + + def __init__(self, sadtalker_path, device): + #load config + fcfg_pose = open(sadtalker_path['audio2pose_yaml_path']) + cfg_pose = CN.load_cfg(fcfg_pose) + cfg_pose.freeze() + fcfg_exp = open(sadtalker_path['audio2exp_yaml_path']) + cfg_exp = CN.load_cfg(fcfg_exp) + cfg_exp.freeze() + + # load audio2pose_model + self.audio2pose_model = Audio2Pose(cfg_pose, None, device=device) + self.audio2pose_model = self.audio2pose_model.to(device) + self.audio2pose_model.eval() + for param in self.audio2pose_model.parameters(): + param.requires_grad = False + + try: + if sadtalker_path['use_safetensor']: + checkpoints = safetensors.torch.load_file(sadtalker_path['checkpoint']) + self.audio2pose_model.load_state_dict(load_x_from_safetensor(checkpoints, 'audio2pose')) + else: + load_cpk(sadtalker_path['audio2pose_checkpoint'], model=self.audio2pose_model, device=device) + except: + raise Exception("Failed in loading audio2pose_checkpoint") + + # load audio2exp_model + netG = SimpleWrapperV2() + netG = netG.to(device) + for param in netG.parameters(): + netG.requires_grad = False + netG.eval() + try: + if sadtalker_path['use_safetensor']: + checkpoints = safetensors.torch.load_file(sadtalker_path['checkpoint']) + netG.load_state_dict(load_x_from_safetensor(checkpoints, 'audio2exp')) + else: + load_cpk(sadtalker_path['audio2exp_checkpoint'], model=netG, device=device) + except: + raise Exception("Failed in loading audio2exp_checkpoint") + self.audio2exp_model = Audio2Exp(netG, cfg_exp, device=device, prepare_training_loss=False) + self.audio2exp_model = self.audio2exp_model.to(device) + for param in self.audio2exp_model.parameters(): + param.requires_grad = False + self.audio2exp_model.eval() + + self.device = device + + def generate(self, batch, coeff_save_dir, pose_style, ref_pose_coeff_path=None): + + with torch.no_grad(): + #test + results_dict_exp= self.audio2exp_model.test(batch) + exp_pred = results_dict_exp['exp_coeff_pred'] #bs T 64 + + #for class_id in range(1): + #class_id = 0#(i+10)%45 + #class_id = random.randint(0,46) #46 styles can be selected + batch['class'] = torch.LongTensor([pose_style]).to(self.device) + results_dict_pose = self.audio2pose_model.test(batch) + pose_pred = results_dict_pose['pose_pred'] #bs T 6 + + pose_len = pose_pred.shape[1] + if pose_len<13: + pose_len = int((pose_len-1)/2)*2+1 + pose_pred = torch.Tensor(savgol_filter(np.array(pose_pred.cpu()), pose_len, 2, axis=1)).to(self.device) + else: + pose_pred = torch.Tensor(savgol_filter(np.array(pose_pred.cpu()), 13, 2, axis=1)).to(self.device) + + coeffs_pred = torch.cat((exp_pred, pose_pred), dim=-1) #bs T 70 + + coeffs_pred_numpy = coeffs_pred[0].clone().detach().cpu().numpy() + + if ref_pose_coeff_path is not None: + coeffs_pred_numpy = self.using_refpose(coeffs_pred_numpy, ref_pose_coeff_path) + + savemat(os.path.join(coeff_save_dir, '%s##%s.mat'%(batch['pic_name'], batch['audio_name'])), + {'coeff_3dmm': coeffs_pred_numpy}) + + return os.path.join(coeff_save_dir, '%s##%s.mat'%(batch['pic_name'], batch['audio_name'])) + + def using_refpose(self, coeffs_pred_numpy, ref_pose_coeff_path): + num_frames = coeffs_pred_numpy.shape[0] + refpose_coeff_dict = loadmat(ref_pose_coeff_path) + refpose_coeff = refpose_coeff_dict['coeff_3dmm'][:,64:70] + refpose_num_frames = refpose_coeff.shape[0] + if refpose_num_frames= 0 + if hp.symmetric_mels: + return (2 * hp.max_abs_value) * ((S - hp.min_level_db) / (-hp.min_level_db)) - hp.max_abs_value + else: + return hp.max_abs_value * ((S - hp.min_level_db) / (-hp.min_level_db)) + +def _denormalize(D): + if hp.allow_clipping_in_normalization: + if hp.symmetric_mels: + return (((np.clip(D, -hp.max_abs_value, + hp.max_abs_value) + hp.max_abs_value) * -hp.min_level_db / (2 * hp.max_abs_value)) + + hp.min_level_db) + else: + return ((np.clip(D, 0, hp.max_abs_value) * -hp.min_level_db / hp.max_abs_value) + hp.min_level_db) + + if hp.symmetric_mels: + return (((D + hp.max_abs_value) * -hp.min_level_db / (2 * hp.max_abs_value)) + hp.min_level_db) + else: + return ((D * -hp.min_level_db / hp.max_abs_value) + hp.min_level_db) diff --git a/src/utils/croper.py b/src/utils/croper.py new file mode 100644 index 0000000000000000000000000000000000000000..3d9a0ac58f97afdc95d40f2a400272b11fe38093 --- /dev/null +++ b/src/utils/croper.py @@ -0,0 +1,144 @@ +import os +import cv2 +import time +import glob +import argparse +import scipy +import numpy as np +from PIL import Image +import torch +from tqdm import tqdm +from itertools import cycle + +from src.face3d.extract_kp_videos_safe import KeypointExtractor +from facexlib.alignment import landmark_98_to_68 + +import numpy as np +from PIL import Image + +class Preprocesser: + def __init__(self, device='cuda'): + self.predictor = KeypointExtractor(device) + + def get_landmark(self, img_np): + """get landmark with dlib + :return: np.array shape=(68, 2) + """ + with torch.no_grad(): + dets = self.predictor.det_net.detect_faces(img_np, 0.97) + + if len(dets) == 0: + return None + det = dets[0] + + img = img_np[int(det[1]):int(det[3]), int(det[0]):int(det[2]), :] + lm = landmark_98_to_68(self.predictor.detector.get_landmarks(img)) # [0] + + #### keypoints to the original location + lm[:,0] += int(det[0]) + lm[:,1] += int(det[1]) + + return lm + + def align_face(self, img, lm, output_size=1024): + """ + :param filepath: str + :return: PIL Image + """ + lm_chin = lm[0: 17] # left-right + lm_eyebrow_left = lm[17: 22] # left-right + lm_eyebrow_right = lm[22: 27] # left-right + lm_nose = lm[27: 31] # top-down + lm_nostrils = lm[31: 36] # top-down + lm_eye_left = lm[36: 42] # left-clockwise + lm_eye_right = lm[42: 48] # left-clockwise + lm_mouth_outer = lm[48: 60] # left-clockwise + lm_mouth_inner = lm[60: 68] # left-clockwise + + # Calculate auxiliary vectors. + eye_left = np.mean(lm_eye_left, axis=0) + eye_right = np.mean(lm_eye_right, axis=0) + eye_avg = (eye_left + eye_right) * 0.5 + eye_to_eye = eye_right - eye_left + mouth_left = lm_mouth_outer[0] + mouth_right = lm_mouth_outer[6] + mouth_avg = (mouth_left + mouth_right) * 0.5 + eye_to_mouth = mouth_avg - eye_avg + + # Choose oriented crop rectangle. + x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1] # Addition of binocular difference and double mouth difference + x /= np.hypot(*x) # hypot函数计算直角三角形的斜边长，用斜边长对三角形两条直边做归一化 + x *= max(np.hypot(*eye_to_eye) * 2.0, np.hypot(*eye_to_mouth) * 1.8) # 双眼差和眼嘴差，选较大的作为基准尺度 + y = np.flipud(x) * [-1, 1] + c = eye_avg + eye_to_mouth * 0.1 + quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y]) # 定义四边形，以面部基准位置为中心上下左右平移得到四个顶点 + qsize = np.hypot(*x) * 2 # 定义四边形的大小（边长），为基准尺度的2倍 + + # Shrink. + # 如果计算出的四边形太大了，就按比例缩小它 + shrink = int(np.floor(qsize / output_size * 0.5)) + if shrink > 1: + rsize = (int(np.rint(float(img.size[0]) / shrink)), int(np.rint(float(img.size[1]) / shrink))) + img = img.resize(rsize, Image.ANTIALIAS) + quad /= shrink + qsize /= shrink + else: + rsize = (int(np.rint(float(img.size[0]))), int(np.rint(float(img.size[1])))) + + # Crop. + border = max(int(np.rint(qsize * 0.1)), 3) + crop = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))), + int(np.ceil(max(quad[:, 1])))) + crop = (max(crop[0] - border, 0), max(crop[1] - border, 0), min(crop[2] + border, img.size[0]), + min(crop[3] + border, img.size[1])) + if crop[2] - crop[0] < img.size[0] or crop[3] - crop[1] < img.size[1]: + # img = img.crop(crop) + quad -= crop[0:2] + + # Pad. + pad = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))), + int(np.ceil(max(quad[:, 1])))) + pad = (max(-pad[0] + border, 0), max(-pad[1] + border, 0), max(pad[2] - img.size[0] + border, 0), + max(pad[3] - img.size[1] + border, 0)) + # if enable_padding and max(pad) > border - 4: + # pad = np.maximum(pad, int(np.rint(qsize * 0.3))) + # img = np.pad(np.float32(img), ((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)), 'reflect') + # h, w, _ = img.shape + # y, x, _ = np.ogrid[:h, :w, :1] + # mask = np.maximum(1.0 - np.minimum(np.float32(x) / pad[0], np.float32(w - 1 - x) / pad[2]), + # 1.0 - np.minimum(np.float32(y) / pad[1], np.float32(h - 1 - y) / pad[3])) + # blur = qsize * 0.02 + # img += (scipy.ndimage.gaussian_filter(img, [blur, blur, 0]) - img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0) + # img += (np.median(img, axis=(0, 1)) - img) * np.clip(mask, 0.0, 1.0) + # img = Image.fromarray(np.uint8(np.clip(np.rint(img), 0, 255)), 'RGB') + # quad += pad[:2] + + # Transform. + quad = (quad + 0.5).flatten() + lx = max(min(quad[0], quad[2]), 0) + ly = max(min(quad[1], quad[7]), 0) + rx = min(max(quad[4], quad[6]), img.size[0]) + ry = min(max(quad[3], quad[5]), img.size[0]) + + # Save aligned image. + return rsize, crop, [lx, ly, rx, ry] + + def crop(self, img_np_list, still=False, xsize=512): # first frame for all video + img_np = img_np_list[0] + lm = self.get_landmark(img_np) + + if lm is None: + raise 'can not detect the landmark from source image' + rsize, crop, quad = self.align_face(img=Image.fromarray(img_np), lm=lm, output_size=xsize) + clx, cly, crx, cry = crop + lx, ly, rx, ry = quad + lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry) + for _i in range(len(img_np_list)): + _inp = img_np_list[_i] + _inp = cv2.resize(_inp, (rsize[0], rsize[1])) + _inp = _inp[cly:cry, clx:crx] + if not still: + _inp = _inp[ly:ry, lx:rx] + img_np_list[_i] = _inp + return img_np_list, crop, quad + diff --git a/src/utils/face_enhancer.py b/src/utils/face_enhancer.py new file mode 100644 index 0000000000000000000000000000000000000000..15851a15966c963d7bd04f35eebdaa6b22a3d966 --- /dev/null +++ b/src/utils/face_enhancer.py @@ -0,0 +1,123 @@ +import os +import torch + +from gfpgan import GFPGANer + +from tqdm import tqdm + +from src.utils.videoio import load_video_to_cv2 + +import cv2 + + +class GeneratorWithLen(object): + """ From https://stackoverflow.com/a/7460929 """ + + def __init__(self, gen, length): + self.gen = gen + self.length = length + + def __len__(self): + return self.length + + def __iter__(self): + return self.gen + +def enhancer_list(images, method='gfpgan', bg_upsampler='realesrgan'): + gen = enhancer_generator_no_len(images, method=method, bg_upsampler=bg_upsampler) + return list(gen) + +def enhancer_generator_with_len(images, method='gfpgan', bg_upsampler='realesrgan'): + """ Provide a generator with a __len__ method so that it can passed to functions that + call len()""" + + if os.path.isfile(images): # handle video to images + # TODO: Create a generator version of load_video_to_cv2 + images = load_video_to_cv2(images) + + gen = enhancer_generator_no_len(images, method=method, bg_upsampler=bg_upsampler) + gen_with_len = GeneratorWithLen(gen, len(images)) + return gen_with_len + +def enhancer_generator_no_len(images, method='gfpgan', bg_upsampler='realesrgan'): + """ Provide a generator function so that all of the enhanced images don't need + to be stored in memory at the same time. This can save tons of RAM compared to + the enhancer function. """ + + print('face enhancer....') + if not isinstance(images, list) and os.path.isfile(images): # handle video to images + images = load_video_to_cv2(images) + + # ------------------------ set up GFPGAN restorer ------------------------ + if method == 'gfpgan': + arch = 'clean' + channel_multiplier = 2 + model_name = 'GFPGANv1.4' + url = 'https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth' + elif method == 'RestoreFormer': + arch = 'RestoreFormer' + channel_multiplier = 2 + model_name = 'RestoreFormer' + url = 'https://github.com/TencentARC/GFPGAN/releases/download/v1.3.4/RestoreFormer.pth' + elif method == 'codeformer': # TODO: + arch = 'CodeFormer' + channel_multiplier = 2 + model_name = 'CodeFormer' + url = 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth' + else: + raise ValueError(f'Wrong model version {method}.') + + + # ------------------------ set up background upsampler ------------------------ + if bg_upsampler == 'realesrgan': + if not torch.cuda.is_available(): # CPU + import warnings + warnings.warn('The unoptimized RealESRGAN is slow on CPU. We do not use it. ' + 'If you really want to use it, please modify the corresponding codes.') + bg_upsampler = None + else: + from basicsr.archs.rrdbnet_arch import RRDBNet + from realesrgan import RealESRGANer + model = RRDBNet(num_in_ch=3, num_out_ch=3, num_feat=64, num_block=23, num_grow_ch=32, scale=2) + bg_upsampler = RealESRGANer( + scale=2, + model_path='https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.1/RealESRGAN_x2plus.pth', + model=model, + tile=400, + tile_pad=10, + pre_pad=0, + half=True) # need to set False in CPU mode + else: + bg_upsampler = None + + # determine model paths + model_path = os.path.join('gfpgan/weights', model_name + '.pth') + + if not os.path.isfile(model_path): + model_path = os.path.join('checkpoints', model_name + '.pth') + + if not os.path.isfile(model_path): + # download pre-trained models from url + model_path = url + + restorer = GFPGANer( + model_path=model_path, + upscale=2, + arch=arch, + channel_multiplier=channel_multiplier, + bg_upsampler=bg_upsampler) + + # ------------------------ restore ------------------------ + for idx in tqdm(range(len(images)), 'Face Enhancer:'): + + img = cv2.cvtColor(images[idx], cv2.COLOR_RGB2BGR) + + # restore faces and background if necessary + cropped_faces, restored_faces, r_img = restorer.enhance( + img, + has_aligned=False, + only_center_face=False, + paste_back=True) + + r_img = cv2.cvtColor(r_img, cv2.COLOR_BGR2RGB) + yield r_img diff --git a/src/utils/hparams.py b/src/utils/hparams.py new file mode 100644 index 0000000000000000000000000000000000000000..743c5c7d5a5a9e686f1ccd6fb3c2fb5cb382d62b --- /dev/null +++ b/src/utils/hparams.py @@ -0,0 +1,160 @@ +from glob import glob +import os + +class HParams: + def __init__(self, **kwargs): + self.data = {} + + for key, value in kwargs.items(): + self.data[key] = value + + def __getattr__(self, key): + if key not in self.data: + raise AttributeError("'HParams' object has no attribute %s" % key) + return self.data[key] + + def set_hparam(self, key, value): + self.data[key] = value + + +# Default hyperparameters +hparams = HParams( + num_mels=80, # Number of mel-spectrogram channels and local conditioning dimensionality + # network + rescale=True, # Whether to rescale audio prior to preprocessing + rescaling_max=0.9, # Rescaling value + + # Use LWS (https://github.com/Jonathan-LeRoux/lws) for STFT and phase reconstruction + # It"s preferred to set True to use with https://github.com/r9y9/wavenet_vocoder + # Does not work if n_ffit is not multiple of hop_size!! + use_lws=False, + + n_fft=800, # Extra window size is filled with 0 paddings to match this parameter + hop_size=200, # For 16000Hz, 200 = 12.5 ms (0.0125 * sample_rate) + win_size=800, # For 16000Hz, 800 = 50 ms (If None, win_size = n_fft) (0.05 * sample_rate) + sample_rate=16000, # 16000Hz (corresponding to librispeech) (sox --i ) + + frame_shift_ms=None, # Can replace hop_size parameter. (Recommended: 12.5) + + # Mel and Linear spectrograms normalization/scaling and clipping + signal_normalization=True, + # Whether to normalize mel spectrograms to some predefined range (following below parameters) + allow_clipping_in_normalization=True, # Only relevant if mel_normalization = True + symmetric_mels=True, + # Whether to scale the data to be symmetric around 0. (Also multiplies the output range by 2, + # faster and cleaner convergence) + max_abs_value=4., + # max absolute value of data. If symmetric, data will be [-max, max] else [0, max] (Must not + # be too big to avoid gradient explosion, + # not too small for fast convergence) + # Contribution by @begeekmyfriend + # Spectrogram Pre-Emphasis (Lfilter: Reduce spectrogram noise and helps model certitude + # levels. Also allows for better G&L phase reconstruction) + preemphasize=True, # whether to apply filter + preemphasis=0.97, # filter coefficient. + + # Limits + min_level_db=-100, + ref_level_db=20, + fmin=55, + # Set this to 55 if your speaker is male! if female, 95 should help taking off noise. (To + # test depending on dataset. Pitch info: male~[65, 260], female~[100, 525]) + fmax=7600, # To be increased/reduced depending on data. + + ###################### Our training parameters ################################# + img_size=96, + fps=25, + + batch_size=16, + initial_learning_rate=1e-4, + nepochs=300000, ### ctrl + c, stop whenever eval loss is consistently greater than train loss for ~10 epochs + num_workers=20, + checkpoint_interval=3000, + eval_interval=3000, + writer_interval=300, + save_optimizer_state=True, + + syncnet_wt=0.0, # is initially zero, will be set automatically to 0.03 later. Leads to faster convergence. + syncnet_batch_size=64, + syncnet_lr=1e-4, + syncnet_eval_interval=1000, + syncnet_checkpoint_interval=10000, + + disc_wt=0.07, + disc_initial_learning_rate=1e-4, +) + + + +# Default hyperparameters +hparamsdebug = HParams( + num_mels=80, # Number of mel-spectrogram channels and local conditioning dimensionality + # network + rescale=True, # Whether to rescale audio prior to preprocessing + rescaling_max=0.9, # Rescaling value + + # Use LWS (https://github.com/Jonathan-LeRoux/lws) for STFT and phase reconstruction + # It"s preferred to set True to use with https://github.com/r9y9/wavenet_vocoder + # Does not work if n_ffit is not multiple of hop_size!! + use_lws=False, + + n_fft=800, # Extra window size is filled with 0 paddings to match this parameter + hop_size=200, # For 16000Hz, 200 = 12.5 ms (0.0125 * sample_rate) + win_size=800, # For 16000Hz, 800 = 50 ms (If None, win_size = n_fft) (0.05 * sample_rate) + sample_rate=16000, # 16000Hz (corresponding to librispeech) (sox --i ) + + frame_shift_ms=None, # Can replace hop_size parameter. (Recommended: 12.5) + + # Mel and Linear spectrograms normalization/scaling and clipping + signal_normalization=True, + # Whether to normalize mel spectrograms to some predefined range (following below parameters) + allow_clipping_in_normalization=True, # Only relevant if mel_normalization = True + symmetric_mels=True, + # Whether to scale the data to be symmetric around 0. (Also multiplies the output range by 2, + # faster and cleaner convergence) + max_abs_value=4., + # max absolute value of data. If symmetric, data will be [-max, max] else [0, max] (Must not + # be too big to avoid gradient explosion, + # not too small for fast convergence) + # Contribution by @begeekmyfriend + # Spectrogram Pre-Emphasis (Lfilter: Reduce spectrogram noise and helps model certitude + # levels. Also allows for better G&L phase reconstruction) + preemphasize=True, # whether to apply filter + preemphasis=0.97, # filter coefficient. + + # Limits + min_level_db=-100, + ref_level_db=20, + fmin=55, + # Set this to 55 if your speaker is male! if female, 95 should help taking off noise. (To + # test depending on dataset. Pitch info: male~[65, 260], female~[100, 525]) + fmax=7600, # To be increased/reduced depending on data. + + ###################### Our training parameters ################################# + img_size=96, + fps=25, + + batch_size=2, + initial_learning_rate=1e-3, + nepochs=100000, ### ctrl + c, stop whenever eval loss is consistently greater than train loss for ~10 epochs + num_workers=0, + checkpoint_interval=10000, + eval_interval=10, + writer_interval=5, + save_optimizer_state=True, + + syncnet_wt=0.0, # is initially zero, will be set automatically to 0.03 later. Leads to faster convergence. + syncnet_batch_size=64, + syncnet_lr=1e-4, + syncnet_eval_interval=10000, + syncnet_checkpoint_interval=10000, + + disc_wt=0.07, + disc_initial_learning_rate=1e-4, +) + + +def hparams_debug_string(): + values = hparams.values() + hp = [" %s: %s" % (name, values[name]) for name in sorted(values) if name != "sentences"] + return "Hyperparameters:\n" + "\n".join(hp) diff --git a/src/utils/init_path.py b/src/utils/init_path.py new file mode 100644 index 0000000000000000000000000000000000000000..5f38d11907bd0dc789992062ce7f02d8876c638f --- /dev/null +++ b/src/utils/init_path.py @@ -0,0 +1,47 @@ +import os +import glob + +def init_path(checkpoint_dir, config_dir, size=512, old_version=False, preprocess='crop'): + + if old_version: + #### load all the checkpoint of `pth` + sadtalker_paths = { + 'wav2lip_checkpoint' : os.path.join(checkpoint_dir, 'wav2lip.pth'), + 'audio2pose_checkpoint' : os.path.join(checkpoint_dir, 'auido2pose_00140-model.pth'), + 'audio2exp_checkpoint' : os.path.join(checkpoint_dir, 'auido2exp_00300-model.pth'), + 'free_view_checkpoint' : os.path.join(checkpoint_dir, 'facevid2vid_00189-model.pth.tar'), + 'path_of_net_recon_model' : os.path.join(checkpoint_dir, 'epoch_20.pth') + } + + use_safetensor = False + elif len(glob.glob(os.path.join(checkpoint_dir, '*.safetensors'))): + print('using safetensor as default') + sadtalker_paths = { + "checkpoint":os.path.join(checkpoint_dir, 'SadTalker_V0.0.2_'+str(size)+'.safetensors'), + } + use_safetensor = True + else: + print("WARNING: The new version of the model will be updated by safetensor, you may need to download it mannully. We run the old version of the checkpoint this time!") + use_safetensor = False + + sadtalker_paths = { + 'wav2lip_checkpoint' : os.path.join(checkpoint_dir, 'wav2lip.pth'), + 'audio2pose_checkpoint' : os.path.join(checkpoint_dir, 'auido2pose_00140-model.pth'), + 'audio2exp_checkpoint' : os.path.join(checkpoint_dir, 'auido2exp_00300-model.pth'), + 'free_view_checkpoint' : os.path.join(checkpoint_dir, 'facevid2vid_00189-model.pth.tar'), + 'path_of_net_recon_model' : os.path.join(checkpoint_dir, 'epoch_20.pth') + } + + sadtalker_paths['dir_of_BFM_fitting'] = os.path.join(config_dir) # , 'BFM_Fitting' + sadtalker_paths['audio2pose_yaml_path'] = os.path.join(config_dir, 'auido2pose.yaml') + sadtalker_paths['audio2exp_yaml_path'] = os.path.join(config_dir, 'auido2exp.yaml') + sadtalker_paths['use_safetensor'] = use_safetensor # os.path.join(config_dir, 'auido2exp.yaml') + + if 'full' in preprocess: + sadtalker_paths['mappingnet_checkpoint'] = os.path.join(checkpoint_dir, 'mapping_00109-model.pth.tar') + sadtalker_paths['facerender_yaml'] = os.path.join(config_dir, 'facerender_still.yaml') + else: + sadtalker_paths['mappingnet_checkpoint'] = os.path.join(checkpoint_dir, 'mapping_00229-model.pth.tar') + sadtalker_paths['facerender_yaml'] = os.path.join(config_dir, 'facerender.yaml') + + return sadtalker_paths \ No newline at end of file diff --git a/src/utils/model2safetensor.py b/src/utils/model2safetensor.py new file mode 100644 index 0000000000000000000000000000000000000000..50c485000d43ba9c230a0bc64ce8aeaaec6e2b29 --- /dev/null +++ b/src/utils/model2safetensor.py @@ -0,0 +1,141 @@ +import torch +import yaml +import os + +import safetensors +from safetensors.torch import save_file +from yacs.config import CfgNode as CN +import sys + +sys.path.append('/apdcephfs/private_shadowcun/SadTalker') + +from src.face3d.models import networks + +from src.facerender.modules.keypoint_detector import HEEstimator, KPDetector +from src.facerender.modules.mapping import MappingNet +from src.facerender.modules.generator import OcclusionAwareGenerator, OcclusionAwareSPADEGenerator + +from src.audio2pose_models.audio2pose import Audio2Pose +from src.audio2exp_models.networks import SimpleWrapperV2 +from src.test_audio2coeff import load_cpk + +size = 256 +############ face vid2vid +config_path = os.path.join('src', 'config', 'facerender.yaml') +current_root_path = '.' + +path_of_net_recon_model = os.path.join(current_root_path, 'checkpoints', 'epoch_20.pth') +net_recon = networks.define_net_recon(net_recon='resnet50', use_last_fc=False, init_path='') +checkpoint = torch.load(path_of_net_recon_model, map_location='cpu') +net_recon.load_state_dict(checkpoint['net_recon']) + +with open(config_path) as f: + config = yaml.safe_load(f) + +generator = OcclusionAwareSPADEGenerator(**config['model_params']['generator_params'], + **config['model_params']['common_params']) +kp_extractor = KPDetector(**config['model_params']['kp_detector_params'], + **config['model_params']['common_params']) +he_estimator = HEEstimator(**config['model_params']['he_estimator_params'], + **config['model_params']['common_params']) +mapping = MappingNet(**config['model_params']['mapping_params']) + +def load_cpk_facevid2vid(checkpoint_path, generator=None, discriminator=None, + kp_detector=None, he_estimator=None, optimizer_generator=None, + optimizer_discriminator=None, optimizer_kp_detector=None, + optimizer_he_estimator=None, device="cpu"): + + checkpoint = torch.load(checkpoint_path, map_location=torch.device(device)) + if generator is not None: + generator.load_state_dict(checkpoint['generator']) + if kp_detector is not None: + kp_detector.load_state_dict(checkpoint['kp_detector']) + if he_estimator is not None: + he_estimator.load_state_dict(checkpoint['he_estimator']) + if discriminator is not None: + try: + discriminator.load_state_dict(checkpoint['discriminator']) + except: + print ('No discriminator in the state-dict. Dicriminator will be randomly initialized') + if optimizer_generator is not None: + optimizer_generator.load_state_dict(checkpoint['optimizer_generator']) + if optimizer_discriminator is not None: + try: + optimizer_discriminator.load_state_dict(checkpoint['optimizer_discriminator']) + except RuntimeError as e: + print ('No discriminator optimizer in the state-dict. Optimizer will be not initialized') + if optimizer_kp_detector is not None: + optimizer_kp_detector.load_state_dict(checkpoint['optimizer_kp_detector']) + if optimizer_he_estimator is not None: + optimizer_he_estimator.load_state_dict(checkpoint['optimizer_he_estimator']) + + return checkpoint['epoch'] + + +def load_cpk_facevid2vid_safetensor(checkpoint_path, generator=None, + kp_detector=None, he_estimator=None, + device="cpu"): + + checkpoint = safetensors.torch.load_file(checkpoint_path) + + if generator is not None: + x_generator = {} + for k,v in checkpoint.items(): + if 'generator' in k: + x_generator[k.replace('generator.', '')] = v + generator.load_state_dict(x_generator) + if kp_detector is not None: + x_generator = {} + for k,v in checkpoint.items(): + if 'kp_extractor' in k: + x_generator[k.replace('kp_extractor.', '')] = v + kp_detector.load_state_dict(x_generator) + if he_estimator is not None: + x_generator = {} + for k,v in checkpoint.items(): + if 'he_estimator' in k: + x_generator[k.replace('he_estimator.', '')] = v + he_estimator.load_state_dict(x_generator) + + return None + +free_view_checkpoint = '/apdcephfs/private_shadowcun/SadTalker/checkpoints/facevid2vid_'+str(size)+'-model.pth.tar' +load_cpk_facevid2vid(free_view_checkpoint, kp_detector=kp_extractor, generator=generator, he_estimator=he_estimator) + +wav2lip_checkpoint = os.path.join(current_root_path, 'checkpoints', 'wav2lip.pth') + +audio2pose_checkpoint = os.path.join(current_root_path, 'checkpoints', 'auido2pose_00140-model.pth') +audio2pose_yaml_path = os.path.join(current_root_path, 'src', 'config', 'auido2pose.yaml') + +audio2exp_checkpoint = os.path.join(current_root_path, 'checkpoints', 'auido2exp_00300-model.pth') +audio2exp_yaml_path = os.path.join(current_root_path, 'src', 'config', 'auido2exp.yaml') + +fcfg_pose = open(audio2pose_yaml_path) +cfg_pose = CN.load_cfg(fcfg_pose) +cfg_pose.freeze() +audio2pose_model = Audio2Pose(cfg_pose, wav2lip_checkpoint) +audio2pose_model.eval() +load_cpk(audio2pose_checkpoint, model=audio2pose_model, device='cpu') + +# load audio2exp_model +netG = SimpleWrapperV2() +netG.eval() +load_cpk(audio2exp_checkpoint, model=netG, device='cpu') + +class SadTalker(torch.nn.Module): + def __init__(self, kp_extractor, generator, netG, audio2pose, face_3drecon): + super(SadTalker, self).__init__() + self.kp_extractor = kp_extractor + self.generator = generator + self.audio2exp = netG + self.audio2pose = audio2pose + self.face_3drecon = face_3drecon + + +model = SadTalker(kp_extractor, generator, netG, audio2pose_model, net_recon) + +# here, we want to convert it to safetensor +save_file(model.state_dict(), "checkpoints/SadTalker_V0.0.2_"+str(size)+".safetensors") + +### test +load_cpk_facevid2vid_safetensor('checkpoints/SadTalker_V0.0.2_'+str(size)+'.safetensors', kp_detector=kp_extractor, generator=generator, he_estimator=None) \ No newline at end of file diff --git a/src/utils/paste_pic.py b/src/utils/paste_pic.py new file mode 100644 index 0000000000000000000000000000000000000000..f9989e21e48e64f620f9b148e65fdfe806c53b14 --- /dev/null +++ b/src/utils/paste_pic.py @@ -0,0 +1,69 @@ +import cv2, os +import numpy as np +from tqdm import tqdm +import uuid + +from src.utils.videoio import save_video_with_watermark + +def paste_pic(video_path, pic_path, crop_info, new_audio_path, full_video_path, extended_crop=False): + + if not os.path.isfile(pic_path): + raise ValueError('pic_path must be a valid path to video/image file') + elif pic_path.split('.')[-1] in ['jpg', 'png', 'jpeg']: + # loader for first frame + full_img = cv2.imread(pic_path) + else: + # loader for videos + video_stream = cv2.VideoCapture(pic_path) + fps = video_stream.get(cv2.CAP_PROP_FPS) + full_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + break + full_img = frame + frame_h = full_img.shape[0] + frame_w = full_img.shape[1] + + video_stream = cv2.VideoCapture(video_path) + fps = video_stream.get(cv2.CAP_PROP_FPS) + crop_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + crop_frames.append(frame) + + if len(crop_info) != 3: + print("you didn't crop the image") + return + else: + r_w, r_h = crop_info[0] + clx, cly, crx, cry = crop_info[1] + lx, ly, rx, ry = crop_info[2] + lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry) + # oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rx + # oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rx + + if extended_crop: + oy1, oy2, ox1, ox2 = cly, cry, clx, crx + else: + oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rx + + tmp_path = str(uuid.uuid4())+'.mp4' + out_tmp = cv2.VideoWriter(tmp_path, cv2.VideoWriter_fourcc(*'MP4V'), fps, (frame_w, frame_h)) + for crop_frame in tqdm(crop_frames, 'seamlessClone:'): + p = cv2.resize(crop_frame.astype(np.uint8), (ox2-ox1, oy2 - oy1)) + + mask = 255*np.ones(p.shape, p.dtype) + location = ((ox1+ox2) // 2, (oy1+oy2) // 2) + gen_img = cv2.seamlessClone(p, full_img, mask, location, cv2.NORMAL_CLONE) + out_tmp.write(gen_img) + + out_tmp.release() + + save_video_with_watermark(tmp_path, new_audio_path, full_video_path, watermark=False) + os.remove(tmp_path) diff --git a/src/utils/preprocess.py b/src/utils/preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..0f784e6c3d8562e1db1bbd850b9f01843cee3c97 --- /dev/null +++ b/src/utils/preprocess.py @@ -0,0 +1,170 @@ +import numpy as np +import cv2, os, sys, torch +from tqdm import tqdm +from PIL import Image + +# 3dmm extraction +import safetensors +import safetensors.torch +from src.face3d.util.preprocess import align_img +from src.face3d.util.load_mats import load_lm3d +from src.face3d.models import networks + +from scipy.io import loadmat, savemat +from src.utils.croper import Preprocesser + + +import warnings + +from src.utils.safetensor_helper import load_x_from_safetensor +warnings.filterwarnings("ignore") + +def split_coeff(coeffs): + """ + Return: + coeffs_dict -- a dict of torch.tensors + + Parameters: + coeffs -- torch.tensor, size (B, 256) + """ + id_coeffs = coeffs[:, :80] + exp_coeffs = coeffs[:, 80: 144] + tex_coeffs = coeffs[:, 144: 224] + angles = coeffs[:, 224: 227] + gammas = coeffs[:, 227: 254] + translations = coeffs[:, 254:] + return { + 'id': id_coeffs, + 'exp': exp_coeffs, + 'tex': tex_coeffs, + 'angle': angles, + 'gamma': gammas, + 'trans': translations + } + + +class CropAndExtract(): + def __init__(self, sadtalker_path, device): + + self.propress = Preprocesser(device) + self.net_recon = networks.define_net_recon(net_recon='resnet50', use_last_fc=False, init_path='').to(device) + + if sadtalker_path['use_safetensor']: + checkpoint = safetensors.torch.load_file(sadtalker_path['checkpoint']) + self.net_recon.load_state_dict(load_x_from_safetensor(checkpoint, 'face_3drecon')) + else: + checkpoint = torch.load(sadtalker_path['path_of_net_recon_model'], map_location=torch.device(device)) + self.net_recon.load_state_dict(checkpoint['net_recon']) + + self.net_recon.eval() + self.lm3d_std = load_lm3d(sadtalker_path['dir_of_BFM_fitting']) + self.device = device + + def generate(self, input_path, save_dir, crop_or_resize='crop', source_image_flag=False, pic_size=256): + + pic_name = os.path.splitext(os.path.split(input_path)[-1])[0] + + landmarks_path = os.path.join(save_dir, pic_name+'_landmarks.txt') + coeff_path = os.path.join(save_dir, pic_name+'.mat') + png_path = os.path.join(save_dir, pic_name+'.png') + + #load input + if not os.path.isfile(input_path): + raise ValueError('input_path must be a valid path to video/image file') + elif input_path.split('.')[-1] in ['jpg', 'png', 'jpeg']: + # loader for first frame + full_frames = [cv2.imread(input_path)] + fps = 25 + else: + # loader for videos + video_stream = cv2.VideoCapture(input_path) + fps = video_stream.get(cv2.CAP_PROP_FPS) + full_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + full_frames.append(frame) + if source_image_flag: + break + + x_full_frames= [cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in full_frames] + + #### crop images as the + if 'crop' in crop_or_resize.lower(): # default crop + x_full_frames, crop, quad = self.propress.crop(x_full_frames, still=True if 'ext' in crop_or_resize.lower() else False, xsize=512) + clx, cly, crx, cry = crop + lx, ly, rx, ry = quad + lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry) + oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rx + crop_info = ((ox2 - ox1, oy2 - oy1), crop, quad) + elif 'full' in crop_or_resize.lower(): + x_full_frames, crop, quad = self.propress.crop(x_full_frames, still=True if 'ext' in crop_or_resize.lower() else False, xsize=512) + clx, cly, crx, cry = crop + lx, ly, rx, ry = quad + lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry) + oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rx + crop_info = ((ox2 - ox1, oy2 - oy1), crop, quad) + else: # resize mode + oy1, oy2, ox1, ox2 = 0, x_full_frames[0].shape[0], 0, x_full_frames[0].shape[1] + crop_info = ((ox2 - ox1, oy2 - oy1), None, None) + + frames_pil = [Image.fromarray(cv2.resize(frame,(pic_size, pic_size))) for frame in x_full_frames] + if len(frames_pil) == 0: + print('No face is detected in the input file') + return None, None + + # save crop info + for frame in frames_pil: + cv2.imwrite(png_path, cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR)) + + # 2. get the landmark according to the detected face. + if not os.path.isfile(landmarks_path): + lm = self.propress.predictor.extract_keypoint(frames_pil, landmarks_path) + else: + print(' Using saved landmarks.') + lm = np.loadtxt(landmarks_path).astype(np.float32) + lm = lm.reshape([len(x_full_frames), -1, 2]) + + if not os.path.isfile(coeff_path): + # load 3dmm paramter generator from Deep3DFaceRecon_pytorch + video_coeffs, full_coeffs = [], [] + for idx in tqdm(range(len(frames_pil)), desc='3DMM Extraction In Video:'): + frame = frames_pil[idx] + W,H = frame.size + lm1 = lm[idx].reshape([-1, 2]) + + if np.mean(lm1) == -1: + lm1 = (self.lm3d_std[:, :2]+1)/2. + lm1 = np.concatenate( + [lm1[:, :1]*W, lm1[:, 1:2]*H], 1 + ) + else: + lm1[:, -1] = H - 1 - lm1[:, -1] + + trans_params, im1, lm1, _ = align_img(frame, lm1, self.lm3d_std) + + trans_params = np.array([float(item) for item in np.hsplit(trans_params, 5)]).astype(np.float32) + im_t = torch.tensor(np.array(im1)/255., dtype=torch.float32).permute(2, 0, 1).to(self.device).unsqueeze(0) + + with torch.no_grad(): + full_coeff = self.net_recon(im_t) + coeffs = split_coeff(full_coeff) + + pred_coeff = {key:coeffs[key].cpu().numpy() for key in coeffs} + + pred_coeff = np.concatenate([ + pred_coeff['exp'], + pred_coeff['angle'], + pred_coeff['trans'], + trans_params[2:][None], + ], 1) + video_coeffs.append(pred_coeff) + full_coeffs.append(full_coeff.cpu().numpy()) + + semantic_npy = np.array(video_coeffs)[:,0] + + savemat(coeff_path, {'coeff_3dmm': semantic_npy, 'full_3dmm': np.array(full_coeffs)[0]}) + + return coeff_path, png_path, crop_info diff --git a/src/utils/safetensor_helper.py b/src/utils/safetensor_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..3cdbdd21e4ed656dfe2d31a57360afb3e96480b3 --- /dev/null +++ b/src/utils/safetensor_helper.py @@ -0,0 +1,8 @@ + + +def load_x_from_safetensor(checkpoint, key): + x_generator = {} + for k,v in checkpoint.items(): + if key in k: + x_generator[k.replace(key+'.', '')] = v + return x_generator \ No newline at end of file diff --git a/src/utils/text2speech.py b/src/utils/text2speech.py new file mode 100644 index 0000000000000000000000000000000000000000..00d165b6cc7774fd200929aafa0ff3b15916111e --- /dev/null +++ b/src/utils/text2speech.py @@ -0,0 +1,20 @@ +import os +import tempfile +from TTS.api import TTS + + +class TTSTalker(): + def __init__(self) -> None: + model_name = TTS.list_models()[0] + self.tts = TTS(model_name) + + def test(self, text, language='en'): + + tempf = tempfile.NamedTemporaryFile( + delete = False, + suffix = ('.'+'wav'), + ) + + self.tts.tts_to_file(text, speaker=self.tts.speakers[0], language=language, file_path=tempf.name) + + return tempf.name \ No newline at end of file diff --git a/src/utils/videoio.py b/src/utils/videoio.py new file mode 100644 index 0000000000000000000000000000000000000000..08bfbdd7d4be97dc17fea4ad7b2733e9eb0ef975 --- /dev/null +++ b/src/utils/videoio.py @@ -0,0 +1,41 @@ +import shutil +import uuid + +import os + +import cv2 + +def load_video_to_cv2(input_path): + video_stream = cv2.VideoCapture(input_path) + fps = video_stream.get(cv2.CAP_PROP_FPS) + full_frames = [] + while 1: + still_reading, frame = video_stream.read() + if not still_reading: + video_stream.release() + break + full_frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + return full_frames + +def save_video_with_watermark(video, audio, save_path, watermark=False): + temp_file = str(uuid.uuid4())+'.mp4' + cmd = r'ffmpeg -y -hide_banner -loglevel error -i "%s" -i "%s" -vcodec copy "%s"' % (video, audio, temp_file) + os.system(cmd) + + if watermark is False: + shutil.move(temp_file, save_path) + else: + # watermark + try: + ##### check if stable-diffusion-webui + import webui + from modules import paths + watarmark_path = paths.script_path+"/extensions/SadTalker/docs/sadtalker_logo.png" + except: + # get the root path of sadtalker. + dir_path = os.path.dirname(os.path.realpath(__file__)) + watarmark_path = dir_path+"/../../docs/sadtalker_logo.png" + + cmd = r'ffmpeg -y -hide_banner -loglevel error -i "%s" -i "%s" -filter_complex "[1]scale=100:-1[wm];[0][wm]overlay=(main_w-overlay_w)-10:10" "%s"' % (temp_file, watarmark_path, save_path) + os.system(cmd) + os.remove(temp_file) \ No newline at end of file diff --git a/tts_voice.py b/tts_voice.py new file mode 100644 index 0000000000000000000000000000000000000000..8ee194c252f82ada41ccc14f33adb592e1a00985 --- /dev/null +++ b/tts_voice.py @@ -0,0 +1,26 @@ +tts_order_voice = {'英语 (美国)-Jenny-女': 'en-US-JennyNeural', + '英语 (美国)-Guy-男': 'en-US-GuyNeural', + '英语 (美国)-Ana-女': 'en-US-AnaNeural', + '英语 (美国)-Aria-女': 'en-US-AriaNeural', + '英语 (美国)-Christopher-男': 'en-US-ChristopherNeural', + '英语 (美国)-Eric-男': 'en-US-EricNeural', + '英语 (美国)-Michelle-女': 'en-US-MichelleNeural', + '英语 (美国)-Roger-男': 'en-US-RogerNeural', + '韩语 (韩国)-Sun-Hi-女': 'ko-KR-SunHiNeural', + '韩语 (韩国)-InJoon-男': 'ko-KR-InJoonNeural', + '日语 (日本)-Nanami-女': 'ja-JP-NanamiNeural', + '日语 (日本)-Keita-男': 'ja-JP-KeitaNeural', + '普通话 (中国大陆)-Xiaoxiao-女': 'zh-CN-XiaoxiaoNeural', + '普通话 (中国大陆)-Yunyang-男': 'zh-CN-YunyangNeural', + '普通话 (中国大陆)-Yunxi-男': 'zh-CN-YunxiNeural', + '普通话 (中国大陆)-Xiaoyi-女': 'zh-CN-XiaoyiNeural', + '普通话 (中国大陆)-Yunjian-男': 'zh-CN-YunjianNeural', + '普通话 (中国大陆)-Yunxia-男': 'zh-CN-YunxiaNeural', + '东北话 (中国大陆)-Xiaobei-女': 'zh-CN-liaoning-XiaobeiNeural', + '中原官话 (中国陕西)-Xiaoni-女': 'zh-CN-shaanxi-XiaoniNeural', + '粤语 (中国香港)-HiuMaan-女': 'zh-HK-HiuMaanNeural', + '粤语 (中国香港)-HiuGaai-女': 'zh-HK-HiuGaaiNeural', + '粤语 (中国香港)-WanLung-男': 'zh-HK-WanLungNeural', + '台湾普通话-HsiaoChen-女': 'zh-TW-HsiaoChenNeural', + '台湾普通话-HsiaoYu-女': 'zh-TW-HsiaoYuNeural', + '台湾普通话-YunJhe-男': 'zh-TW-YunJheNeural'} diff --git a/utils.py b/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1bd5b6185af6c9f1c270b8ba345bfc36d059e081 --- /dev/null +++ b/utils.py @@ -0,0 +1,305 @@ +import os +import sys +import argparse +import logging +import json +import subprocess +import numpy as np +from scipy.io.wavfile import read +import torch +from torch.nn import functional as F +from commons import sequence_mask + +MATPLOTLIB_FLAG = False + +logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) +logger = logging + + +def get_cmodel(rank): + checkpoint = torch.load('wavlm/WavLM-Large.pt') + cfg = WavLMConfig(checkpoint['cfg']) + cmodel = WavLM(cfg).cuda(rank) + cmodel.load_state_dict(checkpoint['model']) + cmodel.eval() + return cmodel + + +def get_content(cmodel, y): + with torch.no_grad(): + c = cmodel.extract_features(y.squeeze(1))[0] + c = c.transpose(1, 2) + return c + + +def get_vocoder(rank): + with open("hifigan/config.json", "r") as f: + config = json.load(f) + config = hifigan.AttrDict(config) + vocoder = hifigan.Generator(config) + ckpt = torch.load("hifigan/generator_v1") + vocoder.load_state_dict(ckpt["generator"]) + vocoder.eval() + vocoder.remove_weight_norm() + vocoder.cuda(rank) + return vocoder + + +def transform(mel, height): # 68-92 + #r = np.random.random() + #rate = r * 0.3 + 0.85 # 0.85-1.15 + #height = int(mel.size(-2) * rate) + tgt = torchvision.transforms.functional.resize(mel, (height, mel.size(-1))) + if height >= mel.size(-2): + return tgt[:, :mel.size(-2), :] + else: + silence = tgt[:,-1:,:].repeat(1,mel.size(-2)-height,1) + silence += torch.randn_like(silence) / 10 + return torch.cat((tgt, silence), 1) + + +def stretch(mel, width): # 0.5-2 + return torchvision.transforms.functional.resize(mel, (mel.size(-2), width)) + + +def load_checkpoint(checkpoint_path, model, optimizer=None): + assert os.path.isfile(checkpoint_path) + checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') + iteration = checkpoint_dict['iteration'] + learning_rate = checkpoint_dict['learning_rate'] + if optimizer is not None: + optimizer.load_state_dict(checkpoint_dict['optimizer']) + saved_state_dict = checkpoint_dict['model'] + if hasattr(model, 'module'): + state_dict = model.module.state_dict() + else: + state_dict = model.state_dict() + new_state_dict= {} + for k, v in state_dict.items(): + try: + new_state_dict[k] = saved_state_dict[k] + except: + logger.info("%s is not in the checkpoint" % k) + new_state_dict[k] = v + if hasattr(model, 'module'): + model.module.load_state_dict(new_state_dict) + else: + model.load_state_dict(new_state_dict) + logger.info("Loaded checkpoint '{}' (iteration {})" .format( + checkpoint_path, iteration)) + return model, optimizer, learning_rate, iteration + + +def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path): + logger.info("Saving model and optimizer state at iteration {} to {}".format( + iteration, checkpoint_path)) + if hasattr(model, 'module'): + state_dict = model.module.state_dict() + else: + state_dict = model.state_dict() + torch.save({'model': state_dict, + 'iteration': iteration, + 'optimizer': optimizer.state_dict(), + 'learning_rate': learning_rate}, checkpoint_path) + + +def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050): + for k, v in scalars.items(): + writer.add_scalar(k, v, global_step) + for k, v in histograms.items(): + writer.add_histogram(k, v, global_step) + for k, v in images.items(): + writer.add_image(k, v, global_step, dataformats='HWC') + for k, v in audios.items(): + writer.add_audio(k, v, global_step, audio_sampling_rate) + + +def latest_checkpoint_path(dir_path, regex="G_*.pth"): + f_list = glob.glob(os.path.join(dir_path, regex)) + f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f)))) + x = f_list[-1] + print(x) + return x + + +def plot_spectrogram_to_numpy(spectrogram): + global MATPLOTLIB_FLAG + if not MATPLOTLIB_FLAG: + import matplotlib + matplotlib.use("Agg") + MATPLOTLIB_FLAG = True + mpl_logger = logging.getLogger('matplotlib') + mpl_logger.setLevel(logging.WARNING) + import matplotlib.pylab as plt + import numpy as np + + fig, ax = plt.subplots(figsize=(10,2)) + im = ax.imshow(spectrogram, aspect="auto", origin="lower", + interpolation='none') + plt.colorbar(im, ax=ax) + plt.xlabel("Frames") + plt.ylabel("Channels") + plt.tight_layout() + + fig.canvas.draw() + data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') + data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) + plt.close() + return data + + +def plot_alignment_to_numpy(alignment, info=None): + global MATPLOTLIB_FLAG + if not MATPLOTLIB_FLAG: + import matplotlib + matplotlib.use("Agg") + MATPLOTLIB_FLAG = True + mpl_logger = logging.getLogger('matplotlib') + mpl_logger.setLevel(logging.WARNING) + import matplotlib.pylab as plt + import numpy as np + + fig, ax = plt.subplots(figsize=(6, 4)) + im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower', + interpolation='none') + fig.colorbar(im, ax=ax) + xlabel = 'Decoder timestep' + if info is not None: + xlabel += '\n\n' + info + plt.xlabel(xlabel) + plt.ylabel('Encoder timestep') + plt.tight_layout() + + fig.canvas.draw() + data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') + data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) + plt.close() + return data + + +def load_wav_to_torch(full_path): + sampling_rate, data = read(full_path) + return torch.FloatTensor(data.astype(np.float32)), sampling_rate + + +def load_filepaths_and_text(filename, split="|"): + with open(filename, encoding='utf-8') as f: + filepaths_and_text = [line.strip().split(split) for line in f] + return filepaths_and_text + + +def get_hparams(init=True): + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, default="./configs/base.json", + help='JSON file for configuration') + parser.add_argument('-m', '--model', type=str, required=True, + help='Model name') + + args = parser.parse_args() + model_dir = os.path.join("./logs", args.model) + + if not os.path.exists(model_dir): + os.makedirs(model_dir) + + config_path = args.config + config_save_path = os.path.join(model_dir, "config.json") + if init: + with open(config_path, "r") as f: + data = f.read() + with open(config_save_path, "w") as f: + f.write(data) + else: + with open(config_save_path, "r") as f: + data = f.read() + config = json.loads(data) + + hparams = HParams(**config) + hparams.model_dir = model_dir + return hparams + + +def get_hparams_from_dir(model_dir): + config_save_path = os.path.join(model_dir, "config.json") + with open(config_save_path, "r") as f: + data = f.read() + config = json.loads(data) + + hparams =HParams(**config) + hparams.model_dir = model_dir + return hparams + + +def get_hparams_from_file(config_path): + with open(config_path, "r") as f: + data = f.read() + config = json.loads(data) + + hparams =HParams(**config) + return hparams + + +def check_git_hash(model_dir): + source_dir = os.path.dirname(os.path.realpath(__file__)) + if not os.path.exists(os.path.join(source_dir, ".git")): + logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format( + source_dir + )) + return + + cur_hash = subprocess.getoutput("git rev-parse HEAD") + + path = os.path.join(model_dir, "githash") + if os.path.exists(path): + saved_hash = open(path).read() + if saved_hash != cur_hash: + logger.warn("git hash values are different. {}(saved) != {}(current)".format( + saved_hash[:8], cur_hash[:8])) + else: + open(path, "w").write(cur_hash) + + +def get_logger(model_dir, filename="train.log"): + global logger + logger = logging.getLogger(os.path.basename(model_dir)) + logger.setLevel(logging.DEBUG) + + formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s") + if not os.path.exists(model_dir): + os.makedirs(model_dir) + h = logging.FileHandler(os.path.join(model_dir, filename)) + h.setLevel(logging.DEBUG) + h.setFormatter(formatter) + logger.addHandler(h) + return logger + + +class HParams(): + def __init__(self, **kwargs): + for k, v in kwargs.items(): + if type(v) == dict: + v = HParams(**v) + self[k] = v + + def keys(self): + return self.__dict__.keys() + + def items(self): + return self.__dict__.items() + + def values(self): + return self.__dict__.values() + + def __len__(self): + return len(self.__dict__) + + def __getitem__(self, key): + return getattr(self, key) + + def __setitem__(self, key, value): + return setattr(self, key, value) + + def __contains__(self, key): + return key in self.__dict__ + + def __repr__(self): + return self.__dict__.__repr__() diff --git a/webui.bat b/webui.bat new file mode 100644 index 0000000000000000000000000000000000000000..6ff83231242ac2260c38a2a4a7ba030aa707b1a3 --- /dev/null +++ b/webui.bat @@ -0,0 +1,17 @@ +@echo off + +IF NOT EXIST venv ( +python -m venv venv +) ELSE ( +echo venv folder already exists, skipping creation... +) +call .\venv\Scripts\activate.bat + +set PYTHON="venv\Scripts\Python.exe" +echo venv %PYTHON% + +%PYTHON% Launcher.py + +echo. +echo Launch unsuccessful. Exiting. +pause \ No newline at end of file diff --git a/webui.sh b/webui.sh new file mode 100644 index 0000000000000000000000000000000000000000..245750237954e140777c0bd20e6d26a1f9d1f74e --- /dev/null +++ b/webui.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash + + +# If run from macOS, load defaults from webui-macos-env.sh +if [[ "$OSTYPE" == "darwin"* ]]; then + export TORCH_COMMAND="pip install torch==1.12.1 torchvision==0.13.1" +fi + +# python3 executable +if [[ -z "${python_cmd}" ]] +then + python_cmd="python3" +fi + +# git executable +if [[ -z "${GIT}" ]] +then + export GIT="git" +fi + +# python3 venv without trailing slash (defaults to ${install_dir}/${clone_dir}/venv) +if [[ -z "${venv_dir}" ]] +then + venv_dir="venv" +fi + +if [[ -z "${LAUNCH_SCRIPT}" ]] +then + LAUNCH_SCRIPT="launcher.py" +fi + +# this script cannot be run as root by default +can_run_as_root=1 + +# read any command line flags to the webui.sh script +while getopts "f" flag > /dev/null 2>&1 +do + case ${flag} in + f) can_run_as_root=1;; + *) break;; + esac +done + +# Disable sentry logging +export ERROR_REPORTING=FALSE + +# Do not reinstall existing pip packages on Debian/Ubuntu +export PIP_IGNORE_INSTALLED=0 + +# Pretty print +delimiter="################################################################" + +printf "\n%s\n" "${delimiter}" +printf "\e[1m\e[32mInstall script for SadTalker + Web UI\n" +printf "\e[1m\e[34mTested on Debian 11 (Bullseye)\e[0m" +printf "\n%s\n" "${delimiter}" + +# Do not run as root +if [[ $(id -u) -eq 0 && can_run_as_root -eq 0 ]] +then + printf "\n%s\n" "${delimiter}" + printf "\e[1m\e[31mERROR: This script must not be launched as root, aborting...\e[0m" + printf "\n%s\n" "${delimiter}" + exit 1 +else + printf "\n%s\n" "${delimiter}" + printf "Running on \e[1m\e[32m%s\e[0m user" "$(whoami)" + printf "\n%s\n" "${delimiter}" +fi + +if [[ -d .git ]] +then + printf "\n%s\n" "${delimiter}" + printf "Repo already cloned, using it as install directory" + printf "\n%s\n" "${delimiter}" + install_dir="${PWD}/../" + clone_dir="${PWD##*/}" +fi + +# Check prerequisites +gpu_info=$(lspci 2>/dev/null | grep VGA) +case "$gpu_info" in + *"Navi 1"*|*"Navi 2"*) export HSA_OVERRIDE_GFX_VERSION=10.3.0 + ;; + *"Renoir"*) export HSA_OVERRIDE_GFX_VERSION=9.0.0 + printf "\n%s\n" "${delimiter}" + printf "Experimental support for Renoir: make sure to have at least 4GB of VRAM and 10GB of RAM or enable cpu mode: --use-cpu all --no-half" + printf "\n%s\n" "${delimiter}" + ;; + *) + ;; +esac +if echo "$gpu_info" | grep -q "AMD" && [[ -z "${TORCH_COMMAND}" ]] +then + export TORCH_COMMAND="pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/rocm5.2" +fi + +for preq in "${GIT}" "${python_cmd}" +do + if ! hash "${preq}" &>/dev/null + then + printf "\n%s\n" "${delimiter}" + printf "\e[1m\e[31mERROR: %s is not installed, aborting...\e[0m" "${preq}" + printf "\n%s\n" "${delimiter}" + exit 1 + fi +done + +if ! "${python_cmd}" -c "import venv" &>/dev/null +then + printf "\n%s\n" "${delimiter}" + printf "\e[1m\e[31mERROR: python3-venv is not installed, aborting...\e[0m" + printf "\n%s\n" "${delimiter}" + exit 1 +fi + +printf "\n%s\n" "${delimiter}" +printf "Create and activate python venv" +printf "\n%s\n" "${delimiter}" +cd "${install_dir}"/"${clone_dir}"/ || { printf "\e[1m\e[31mERROR: Can't cd to %s/%s/, aborting...\e[0m" "${install_dir}" "${clone_dir}"; exit 1; } +if [[ ! -d "${venv_dir}" ]] +then + "${python_cmd}" -m venv "${venv_dir}" + first_launch=1 +fi +# shellcheck source=/dev/null +if [[ -f "${venv_dir}"/bin/activate ]] +then + source "${venv_dir}"/bin/activate +else + printf "\n%s\n" "${delimiter}" + printf "\e[1m\e[31mERROR: Cannot activate python venv, aborting...\e[0m" + printf "\n%s\n" "${delimiter}" + exit 1 +fi + +printf "\n%s\n" "${delimiter}" +printf "Launching launcher.py..." +printf "\n%s\n" "${delimiter}" +exec "${python_cmd}" "${LAUNCH_SCRIPT}" "$@" \ No newline at end of file