yujiepan commited on
Commit
c5aa03d
·
verified ·
1 Parent(s): ba086de

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. README.md +35 -0
README.md CHANGED
@@ -20,6 +20,41 @@ pipe = pipeline(
20
  trust_remote_code=True, max_new_tokens=3,
21
  )
22
  print(pipe("Hello World!"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  ```
24
 
25
  ### Codes to create this repo:
 
20
  trust_remote_code=True, max_new_tokens=3,
21
  )
22
  print(pipe("Hello World!"))
23
+
24
+
25
+ from transformers import AutoModelForCausalLM, AutoTokenizer
26
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
27
+ model = AutoModelForCausalLM.from_pretrained(
28
+ model_id,
29
+ torch_dtype="auto",
30
+ device_map="auto"
31
+ )
32
+ prompt = "Give me a short introduction to large language model."
33
+ messages = [
34
+ {"role": "user", "content": prompt}
35
+ ]
36
+ text = tokenizer.apply_chat_template(
37
+ messages,
38
+ tokenize=False,
39
+ add_generation_prompt=True,
40
+ enable_thinking=True # Switches between thinking and non-thinking modes. Default is True.
41
+ )
42
+ print(text)
43
+ model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
44
+ generated_ids = model.generate(
45
+ **model_inputs,
46
+ max_new_tokens=128
47
+ )
48
+ output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()
49
+ try:
50
+ # rindex finding 151668 (</think>)
51
+ index = len(output_ids) - output_ids[::-1].index(151668)
52
+ except ValueError:
53
+ index = 0
54
+ thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n")
55
+ content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n")
56
+ print("thinking content:", thinking_content)
57
+ print("content:", content)
58
  ```
59
 
60
  ### Codes to create this repo: