from generation_utils import *
from utils import WriteTextMidiToFile, get_miditok
from load import LoadModel
from decoder import TextDecoder
from playback import get_music
class GenerateMidiText:
"""Generating music with Class
LOGIC:
FOR GENERATING FROM SCRATCH:
- self.generate_one_new_track()
it calls
- self.generate_until_track_end()
FOR GENERATING NEW BARS:
- self.generate_one_more_bar()
it calls
- self.process_prompt_for_next_bar()
- self.generate_until_track_end()"""
def __init__(self, model, tokenizer, piece_by_track=[]):
self.model = model
self.tokenizer = tokenizer
# default initialization
self.initialize_default_parameters()
self.initialize_dictionaries(piece_by_track)
"""Setters"""
def initialize_default_parameters(self):
self.set_device()
self.set_attention_length()
self.generate_until = "TRACK_END"
self.set_force_sequence_lenth()
self.set_nb_bars_generated()
self.set_improvisation_level(0)
def initialize_dictionaries(self, piece_by_track):
self.piece_by_track = piece_by_track
def set_device(self, device="cpu"):
self.device = ("cpu",)
def set_attention_length(self):
self.max_length = self.model.config.n_positions
print(
f"Attention length set to {self.max_length} -> 'model.config.n_positions'"
)
def set_force_sequence_lenth(self, force_sequence_length=True):
self.force_sequence_length = force_sequence_length
def set_improvisation_level(self, improvisation_value):
self.no_repeat_ngram_size = improvisation_value
print("--------------------")
print(f"no_repeat_ngram_size set to {improvisation_value}")
print("--------------------")
def reset_temperatures(self, track_id, temperature):
self.piece_by_track[track_id]["temperature"] = temperature
def set_nb_bars_generated(self, n_bars=8): # default is a 8 bar model
self.model_n_bar = n_bars
""" Generation Tools - Dictionnaries """
def initiate_track_dict(self, instr, density, temperature):
label = len(self.piece_by_track)
self.piece_by_track.append(
{
"label": f"track_{label}",
"instrument": instr,
"density": density,
"temperature": temperature,
"bars": [],
}
)
def update_track_dict__add_bars(self, bars, track_id):
"""Add bars to the track dictionnary"""
for bar in self.striping_track_ends(bars).split("BAR_START "):
if bar == "": # happens is there is one bar only
continue
else:
if "TRACK_START" in bar:
self.piece_by_track[track_id]["bars"].append(bar)
else:
self.piece_by_track[track_id]["bars"].append("BAR_START " + bar)
def get_all_instr_bars(self, track_id):
return self.piece_by_track[track_id]["bars"]
def striping_track_ends(self, text):
if "TRACK_END" in text:
# first get rid of extra space if any
# then gets rid of "TRACK_END"
text = text.rstrip(" ").rstrip("TRACK_END")
return text
def get_last_generated_track(self, full_piece):
track = (
"TRACK_START "
+ self.striping_track_ends(full_piece.split("TRACK_START ")[-1])
+ "TRACK_END "
) # forcing the space after track and
return track
def get_selected_track_as_text(self, track_id):
text = ""
for bar in self.piece_by_track[track_id]["bars"]:
text += bar
text += "TRACK_END "
return text
@staticmethod
def get_newly_generated_text(input_prompt, full_piece):
return full_piece[len(input_prompt) :]
def get_whole_piece_from_bar_dict(self):
text = "PIECE_START "
for track_id, _ in enumerate(self.piece_by_track):
text += self.get_selected_track_as_text(track_id)
return text
def delete_one_track(self, track): # TO BE TESTED
self.piece_by_track.pop(track)
# def update_piece_dict__add_track(self, track_id, track):
# self.piece_dict[track_id] = track
# def update_all_dictionnaries__add_track(self, track):
# self.update_piece_dict__add_track(track_id, track)
"""Basic generation tools"""
def tokenize_input_prompt(self, input_prompt, verbose=True):
"""Tokenizing prompt
Args:
- input_prompt (str): prompt to tokenize
Returns:
- input_prompt_ids (torch.tensor): tokenized prompt
"""
if verbose:
print("Tokenizing input_prompt...")
return self.tokenizer.encode(input_prompt, return_tensors="pt")
def generate_sequence_of_token_ids(
self,
input_prompt_ids,
temperature,
verbose=True,
):
"""
generate a sequence of token ids based on input_prompt_ids
The sequence length depends on the trained model (self.model_n_bar)
"""
generated_ids = self.model.generate(
input_prompt_ids,
max_length=self.max_length,
do_sample=True,
temperature=temperature,
no_repeat_ngram_size=self.no_repeat_ngram_size, # default = 0
eos_token_id=self.tokenizer.encode(self.generate_until)[0], # good
)
if verbose:
print("Generating a token_id sequence...")
return generated_ids
def convert_ids_to_text(self, generated_ids, verbose=True):
"""converts the token_ids to text"""
generated_text = self.tokenizer.decode(generated_ids[0])
if verbose:
print("Converting token sequence to MidiText...")
return generated_text
def generate_until_track_end(
self,
input_prompt="PIECE_START ",
instrument=None,
density=None,
temperature=None,
verbose=True,
expected_length=None,
):
"""generate until the TRACK_END token is reached
full_piece = input_prompt + generated"""
if expected_length is None:
expected_length = self.model_n_bar
if instrument is not None:
input_prompt = f"{input_prompt}TRACK_START INST={str(instrument)} "
if density is not None:
input_prompt = f"{input_prompt}DENSITY={str(density)} "
if instrument is None and density is not None:
print("Density cannot be defined without an input_prompt instrument #TOFIX")
if temperature is None:
ValueError("Temperature must be defined")
if verbose:
print("--------------------")
print(
f"Generating {instrument} - Density {density} - temperature {temperature}"
)
bar_count_checks = False
failed = 0
while not bar_count_checks: # regenerate until right length
input_prompt_ids = self.tokenize_input_prompt(input_prompt, verbose=verbose)
generated_tokens = self.generate_sequence_of_token_ids(
input_prompt_ids, temperature, verbose=verbose
)
full_piece = self.convert_ids_to_text(generated_tokens, verbose=verbose)
generated = self.get_newly_generated_text(input_prompt, full_piece)
# bar_count_checks
bar_count_checks, bar_count = bar_count_check(generated, expected_length)
if not self.force_sequence_length:
# set bar_count_checks to true to exist the while loop
bar_count_checks = True
if not bar_count_checks and self.force_sequence_length:
# if the generated sequence is not the expected length
if failed > -1: # deactivated for speed
full_piece, bar_count_checks = forcing_bar_count(
input_prompt,
generated,
bar_count,
expected_length,
)
else:
print('"--- Wrong length - Regenerating ---')
if not bar_count_checks:
failed += 1
if failed > 2:
bar_count_checks = True # TOFIX exit the while loop
return full_piece
def generate_one_new_track(
self,
instrument,
density,
temperature,
input_prompt="PIECE_START ",
):
self.initiate_track_dict(instrument, density, temperature)
full_piece = self.generate_until_track_end(
input_prompt=input_prompt,
instrument=instrument,
density=density,
temperature=temperature,
)
track = self.get_last_generated_track(full_piece)
self.update_track_dict__add_bars(track, -1)
full_piece = self.get_whole_piece_from_bar_dict()
return full_piece
""" Piece generation - Basics """
def generate_piece(self, instrument_list, density_list, temperature_list):
"""generate a sequence with mutiple tracks
- inst_list sets the list of instruments of the order of generation
- density is paired with inst_list
Each track/intrument is generated on a prompt which contains the previously generated track/instrument
This means that the first instrument is generated with less bias than the next one, and so on.
'generated_piece' keeps track of the entire piece
'generated_piece' is returned by self.generate_until_track_end
# it is returned by self.generate_until_track_end"""
generated_piece = "PIECE_START "
for instrument, density, temperature in zip(
instrument_list, density_list, temperature_list
):
generated_piece = self.generate_one_new_track(
instrument,
density,
temperature,
input_prompt=generated_piece,
)
# generated_piece = self.get_whole_piece_from_bar_dict()
self.check_the_piece_for_errors()
return generated_piece
""" Piece generation - Extra Bars """
@staticmethod
def process_prompt_for_next_bar(self, track_idx):
"""Processing the prompt for the model to generate one more bar only.
The prompt containts:
if not the first bar: the previous, already processed, bars of the track
the bar initialization (ex: "TRACK_START INST=DRUMS DENSITY=2 ")
the last (self.model_n_bar)-1 bars of the track
Args:
track_idx (int): the index of the track to be processed
Returns:
the processed prompt for generating the next bar
"""
track = self.piece_by_track[track_idx]
# for bars which are not the bar to prolong
pre_promt = "PIECE_START "
for i, othertrack in enumerate(self.piece_by_track):
if i != track_idx:
len_diff = len(othertrack["bars"]) - len(track["bars"])
if len_diff > 0:
# if other bars are longer, it mean that this one should catch up
pre_promt += othertrack["bars"][0]
for bar in track["bars"][-self.model_n_bar :]:
pre_promt += bar
pre_promt += "TRACK_END "
elif False: # len_diff <= 0: # THIS GENERATES EMPTINESS
# adding an empty bars at the end of the other tracks if they have not been processed yet
pre_promt += othertracks["bars"][0]
for bar in track["bars"][-(self.model_n_bar - 1) :]:
pre_promt += bar
for _ in range(abs(len_diff) + 1):
pre_promt += "BAR_START BAR_END "
pre_promt += "TRACK_END "
# for the bar to prolong
# initialization e.g TRACK_START INST=DRUMS DENSITY=2
processed_prompt = track["bars"][0]
for bar in track["bars"][-(self.model_n_bar - 1) :]:
# adding the "last" bars of the track
processed_prompt += bar
processed_prompt += "BAR_START "
print(
f"--- prompt length = {len((pre_promt + processed_prompt).split(' '))} ---"
)
return pre_promt + processed_prompt
def generate_one_more_bar(self, i):
"""Generate one more bar from the input_prompt"""
processed_prompt = self.process_prompt_for_next_bar(self, i)
prompt_plus_bar = self.generate_until_track_end(
input_prompt=processed_prompt,
temperature=self.piece_by_track[i]["temperature"],
expected_length=1,
verbose=False,
)
added_bar = self.get_newly_generated_bar(prompt_plus_bar)
self.update_track_dict__add_bars(added_bar, i)
def get_newly_generated_bar(self, prompt_plus_bar):
return "BAR_START " + self.striping_track_ends(
prompt_plus_bar.split("BAR_START ")[-1]
)
def generate_n_more_bars(self, n_bars, only_this_track=None, verbose=True):
"""Generate n more bars from the input_prompt"""
if only_this_track is None:
only_this_track
print(f"================== ")
print(f"Adding {n_bars} more bars to the piece ")
for bar_id in range(n_bars):
print(f"----- added bar #{bar_id+1} --")
for i, track in enumerate(self.piece_by_track):
if only_this_track is None or i == only_this_track:
print(f"--------- {track['label']}")
self.generate_one_more_bar(i)
self.check_the_piece_for_errors()
def check_the_piece_for_errors(self, piece: str = None):
if piece is None:
piece = generate_midi.get_whole_piece_from_bar_dict()
errors = []
errors.append(
[
(token, id)
for id, token in enumerate(piece.split(" "))
if token not in self.tokenizer.vocab or token == "UNK"
]
)
if len(errors) > 0:
# print(piece)
for er in errors:
er
print(f"Token not found in the piece at {er[0][1]}: {er[0][0]}")
print(piece.split(" ")[er[0][1] - 5 : er[0][1] + 5])
if __name__ == "__main__":
# worker
DEVICE = "cpu"
# define generation parameters
N_FILES_TO_GENERATE = 2
Temperatures_to_try = [0.7]
USE_FAMILIZED_MODEL = True
force_sequence_length = True
if USE_FAMILIZED_MODEL:
# model_repo = "misnaej/the-jam-machine-elec-famil"
# model_repo = "misnaej/the-jam-machine-elec-famil-ft32"
# model_repo = "JammyMachina/elec-gmusic-familized-model-13-12__17-35-53"
# n_bar_generated = 8
model_repo = "JammyMachina/improved_4bars-mdl"
n_bar_generated = 4
instrument_promt_list = ["4", "DRUMS", "3"]
# DRUMS = drums, 0 = piano, 1 = chromatic percussion, 2 = organ, 3 = guitar, 4 = bass, 5 = strings, 6 = ensemble, 7 = brass, 8 = reed, 9 = pipe, 10 = synth lead, 11 = synth pad, 12 = synth effects, 13 = ethnic, 14 = percussive, 15 = sound effects
density_list = [3, 2, 2]
# temperature_list = [0.7, 0.7, 0.75]
else:
model_repo = "misnaej/the-jam-machine"
instrument_promt_list = ["30"] # , "DRUMS", "0"]
density_list = [3] # , 2, 3]
# temperature_list = [0.7, 0.5, 0.75]
pass
# define generation directory
generated_sequence_files_path = define_generation_dir(model_repo)
# load model and tokenizer
model, tokenizer = LoadModel(
model_repo, from_huggingface=True
).load_model_and_tokenizer()
# does the prompt make sense
check_if_prompt_inst_in_tokenizer_vocab(tokenizer, instrument_promt_list)
for temperature in Temperatures_to_try:
print(f"================= TEMPERATURE {temperature} =======================")
for _ in range(N_FILES_TO_GENERATE):
print(f"========================================")
# 1 - instantiate
generate_midi = GenerateMidiText(model, tokenizer)
# 0 - set the n_bar for this model
generate_midi.set_nb_bars_generated(n_bars=n_bar_generated)
# 1 - defines the instruments, densities and temperatures
# 2- generate the first 8 bars for each instrument
generate_midi.set_improvisation_level(30)
generate_midi.generate_piece(
instrument_promt_list,
density_list,
[temperature for _ in density_list],
)
# 3 - force the model to improvise
# generate_midi.set_improvisation_level(20)
# 4 - generate the next 4 bars for each instrument
# generate_midi.generate_n_more_bars(n_bar_generated)
# 5 - lower the improvisation level
generate_midi.generated_piece = (
generate_midi.get_whole_piece_from_bar_dict()
)
# print the generated sequence in terminal
print("=========================================")
print(generate_midi.generated_piece)
print("=========================================")
# write to JSON file
filename = WriteTextMidiToFile(
generate_midi,
generated_sequence_files_path,
).text_midi_to_file()
# decode the sequence to MIDI """
decode_tokenizer = get_miditok()
TextDecoder(decode_tokenizer, USE_FAMILIZED_MODEL).get_midi(
generate_midi.generated_piece, filename=filename.split(".")[0] + ".mid"
)
inst_midi, mixed_audio = get_music(filename.split(".")[0] + ".mid")
max_time = get_max_time(inst_midi)
plot_piano_roll(inst_midi)
print("Et voilà! Your MIDI file is ready! GO JAM!")