Lint pygmalion
Browse files
src/axolotl/prompt_strategies/pygmalion.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
|
|
|
|
|
| 1 |
import copy
|
| 2 |
import logging
|
| 3 |
from collections import defaultdict
|
|
@@ -9,10 +11,14 @@ IGNORE_TOKEN_ID = -100
|
|
| 9 |
|
| 10 |
|
| 11 |
class PygmalionPromptTokenizingStrategy(PromptTokenizingStrategy):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
bot_prefix_token_ids = []
|
| 13 |
|
| 14 |
def __init__(self, prompter, tokenizer, *args, **kwargs):
|
| 15 |
-
super().__init__(prompter, tokenizer)
|
| 16 |
res = self._tokenize("<|model|>", add_eos_token=False, strip_bos_token=True)
|
| 17 |
self.bot_prefix_token_ids = res["input_ids"]
|
| 18 |
|
|
@@ -23,7 +29,7 @@ class PygmalionPromptTokenizingStrategy(PromptTokenizingStrategy):
|
|
| 23 |
"labels": [],
|
| 24 |
}
|
| 25 |
current_len = 0
|
| 26 |
-
for
|
| 27 |
role, message = part
|
| 28 |
if role == "system":
|
| 29 |
prefix = "<|system|>"
|
|
@@ -96,10 +102,16 @@ class PygmalionPromptTokenizingStrategy(PromptTokenizingStrategy):
|
|
| 96 |
|
| 97 |
|
| 98 |
class PygmalionPrompter:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
def __init__(self, *args, **kwargs):
|
| 100 |
pass
|
| 101 |
|
| 102 |
-
def build_prompt(
|
|
|
|
|
|
|
| 103 |
for msg in source:
|
| 104 |
yield msg["role"], msg["value"]
|
| 105 |
|
|
|
|
| 1 |
+
"""Module containing the PygmalionPromptTokenizingStrategy and PygmalionPrompter class"""
|
| 2 |
+
|
| 3 |
import copy
|
| 4 |
import logging
|
| 5 |
from collections import defaultdict
|
|
|
|
| 11 |
|
| 12 |
|
| 13 |
class PygmalionPromptTokenizingStrategy(PromptTokenizingStrategy):
|
| 14 |
+
"""
|
| 15 |
+
Tokenizing strategy for Pygmalion.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
bot_prefix_token_ids = []
|
| 19 |
|
| 20 |
def __init__(self, prompter, tokenizer, *args, **kwargs):
|
| 21 |
+
super().__init__(prompter, tokenizer, *args, **kwargs)
|
| 22 |
res = self._tokenize("<|model|>", add_eos_token=False, strip_bos_token=True)
|
| 23 |
self.bot_prefix_token_ids = res["input_ids"]
|
| 24 |
|
|
|
|
| 29 |
"labels": [],
|
| 30 |
}
|
| 31 |
current_len = 0
|
| 32 |
+
for _, part in enumerate(self.prompter.build_prompt(prompt["conversations"])):
|
| 33 |
role, message = part
|
| 34 |
if role == "system":
|
| 35 |
prefix = "<|system|>"
|
|
|
|
| 102 |
|
| 103 |
|
| 104 |
class PygmalionPrompter:
|
| 105 |
+
"""
|
| 106 |
+
Prompter for Pygmalion.
|
| 107 |
+
"""
|
| 108 |
+
|
| 109 |
def __init__(self, *args, **kwargs):
|
| 110 |
pass
|
| 111 |
|
| 112 |
+
def build_prompt(
|
| 113 |
+
self, source, *args, **kwargs # pylint: disable=unused-argument
|
| 114 |
+
) -> Generator[str, None, None]:
|
| 115 |
for msg in source:
|
| 116 |
yield msg["role"], msg["value"]
|
| 117 |
|