|
|
--- |
|
|
license: apache-2.0 |
|
|
base_model: |
|
|
- Delta-Vector/Austral-GLM4-KTO |
|
|
language: |
|
|
- en |
|
|
library_name: transformers |
|
|
tags: |
|
|
- roleplay |
|
|
- finetune |
|
|
- axolotl |
|
|
- adventure |
|
|
- creative-writing |
|
|
- GLM4 |
|
|
- 32B |
|
|
datasets: |
|
|
- Delta-Vector/Tauri-Rep-Remover-KTO |
|
|
- Delta-Vector/Orion-LN-V1-ShareGPT |
|
|
- Delta-Vector/Orion-Personamaxx-RP |
|
|
- Delta-Vector/Orion-Co-Writer-51K |
|
|
- Delta-Vector/Orion-Praxis-Co-Writer |
|
|
- Delta-Vector/Orion-Shoujo-AI-Filtered-ShareGPT |
|
|
- Delta-Vector/Orion-PIPPA-Cleaned-V2 |
|
|
- Delta-Vector/Orion-Alpindale-LN-ShareGPT |
|
|
- Delta-Vector/Orion-Deepseek-V3-RP-Filtered |
|
|
- Delta-Vector/Orion-Books-V2-ShareGPT |
|
|
- Delta-Vector/Orion-Light-Novels-Roleplay-Logs-Books-Oh-My-duplicate-turns-removed |
|
|
- Delta-Vector/Orion-RP-Guild |
|
|
- Delta-Vector/Orion-Creative_Writing-Complexity |
|
|
- Delta-Vector/Orion-Deepseek-R1-RP-Filtered |
|
|
- Delta-Vector/Orion-Storium-Prefixed-Clean |
|
|
- Delta-Vector/Orion-Misc-Sharegpt-Prefixed |
|
|
- Delta-Vector/Orion-LIMARP-Complexity |
|
|
- Delta-Vector/Orion-BlueSky-10K-Complexity |
|
|
- Delta-Vector/Orion-OpenCAI-ShareGPT |
|
|
- Delta-Vector/Orion-Roleplay-Logs-Sharegpt-Ngram-cleaned |
|
|
- Delta-Vector/Orion-vanilla-backrooms-claude-sharegpt |
|
|
--- |
|
|
<!DOCTYPE html> |
|
|
<html lang="en"> |
|
|
<head> |
|
|
<meta charset="UTF-8"> |
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
|
<title>Austral 24B Winton</title> |
|
|
<link href="" rel="stylesheet"> |
|
|
<style> |
|
|
body { |
|
|
font-family: 'Roboto Slab', serif; |
|
|
background: linear-gradient(135deg, #8B4513 0%, #A0522D 25%, #CD853F 50%, #D2691E 75%, #8B4513 100%); |
|
|
background-size: 400% 400%; |
|
|
animation: prehistoricShift 20s ease-in-out infinite; |
|
|
color: #2F1B14; |
|
|
margin: 0; |
|
|
padding: 0; |
|
|
font-size: 16px; |
|
|
min-height: 100vh; |
|
|
} |
|
|
@keyframes prehistoricShift { |
|
|
0%, 100% { background-position: 0% 50%; } |
|
|
50% { background-position: 100% 50%; } |
|
|
} |
|
|
.container { |
|
|
margin: 20px; |
|
|
background: linear-gradient(145deg, #F4E4BC 0%, #DEB887 100%); |
|
|
padding: 20px; |
|
|
border-radius: 15px; |
|
|
box-shadow: 0 8px 25px rgba(0, 0, 0, 0.4), inset 0 2px 5px rgba(255, 255, 255, 0.3); |
|
|
border: 4px solid #8B4513; |
|
|
position: relative; |
|
|
overflow: hidden; |
|
|
} |
|
|
.container::before { |
|
|
content: ''; |
|
|
position: absolute; |
|
|
top: 0; |
|
|
left: 0; |
|
|
right: 0; |
|
|
bottom: 0; |
|
|
background-image: |
|
|
radial-gradient(circle at 20% 80%, rgba(139, 69, 19, 0.1) 0%, transparent 50%), |
|
|
radial-gradient(circle at 80% 20%, rgba(160, 82, 45, 0.1) 0%, transparent 50%); |
|
|
pointer-events: none; |
|
|
} |
|
|
.header h1 { |
|
|
font-family: 'Cinzel', serif; |
|
|
font-size: 32px; |
|
|
color: #5D2E0C; |
|
|
margin: 0 0 20px 0; |
|
|
text-align: center; |
|
|
text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.3); |
|
|
letter-spacing: 2px; |
|
|
position: relative; |
|
|
} |
|
|
.section { |
|
|
margin-top: 30px; |
|
|
position: relative; |
|
|
} |
|
|
.section h2 { |
|
|
font-family: 'Cinzel', serif; |
|
|
font-size: 26px; |
|
|
color: #5D2E0C; |
|
|
text-align: center; |
|
|
margin-bottom: 20px; |
|
|
text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.2); |
|
|
letter-spacing: 1px; |
|
|
} |
|
|
.info p { |
|
|
color: #2F1B14; |
|
|
line-height: 1.7; |
|
|
font-size: 16px; |
|
|
text-shadow: 0 1px 1px rgba(255, 255, 255, 0.5); |
|
|
} |
|
|
.info img { |
|
|
width: 85%; |
|
|
border-radius: 12px; |
|
|
margin: 0 auto 15px; |
|
|
display: block; |
|
|
box-shadow: 0 0 25px rgba(0, 0, 0, 0.4); |
|
|
border: 3px solid #8B4513; |
|
|
filter: sepia(20%) contrast(110%); |
|
|
} |
|
|
a { |
|
|
color: #5D2E0C; |
|
|
text-decoration: none; |
|
|
transition: all 0.3s ease; |
|
|
font-weight: 500; |
|
|
} |
|
|
a:hover { |
|
|
color: #8B4513; |
|
|
text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.2); |
|
|
} |
|
|
.button { |
|
|
display: inline-block; |
|
|
background: linear-gradient(145deg, #CD853F, #D2691E); |
|
|
color: #2F1B14; |
|
|
padding: 12px 24px; |
|
|
border-radius: 8px; |
|
|
cursor: pointer; |
|
|
text-decoration: none; |
|
|
transition: all 0.3s ease; |
|
|
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); |
|
|
border: 2px solid #8B4513; |
|
|
} |
|
|
.button:hover { |
|
|
background: linear-gradient(145deg, #D2691E, #CD853F); |
|
|
box-shadow: 0 6px 15px rgba(139, 69, 19, 0.4); |
|
|
transform: translateY(-2px); |
|
|
} |
|
|
pre { |
|
|
background: linear-gradient(145deg, #F5DEB3, #DEB887); |
|
|
padding: 20px; |
|
|
border-radius: 8px; |
|
|
overflow-x: auto; |
|
|
border: 2px solid #8B4513; |
|
|
box-shadow: inset 0 2px 5px rgba(0, 0, 0, 0.1); |
|
|
} |
|
|
code { |
|
|
font-family: 'Courier New', monospace; |
|
|
color: #2F1B14; |
|
|
} |
|
|
.info-card { |
|
|
background: linear-gradient(145deg, #F5DEB3, #DEB887); |
|
|
border: 3px solid #8B4513; |
|
|
border-radius: 12px; |
|
|
overflow: hidden; |
|
|
box-shadow: 0 6px 15px rgba(0, 0, 0, 0.2); |
|
|
} |
|
|
.info-header { |
|
|
background: linear-gradient(145deg, #CD853F, #D2691E); |
|
|
padding: 25px; |
|
|
border-bottom: 2px solid #8B4513; |
|
|
} |
|
|
.info-header h3 { |
|
|
font-family: 'Cinzel', serif; |
|
|
color: #2F1B14; |
|
|
margin: 0 0 15px 0; |
|
|
font-size: 22px; |
|
|
text-align: center; |
|
|
text-shadow: 1px 1px 2px rgba(0, 0, 0, 0.2); |
|
|
letter-spacing: 1px; |
|
|
} |
|
|
.model-tags { |
|
|
display: flex; |
|
|
gap: 10px; |
|
|
flex-wrap: wrap; |
|
|
justify-content: center; |
|
|
} |
|
|
.model-tag { |
|
|
background: linear-gradient(145deg, #DEB887, #CD853F); |
|
|
color: #2F1B14; |
|
|
padding: 6px 12px; |
|
|
border-radius: 6px; |
|
|
font-size: 12px; |
|
|
border: 2px solid #8B4513; |
|
|
font-weight: 500; |
|
|
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); |
|
|
} |
|
|
.model-composition { |
|
|
padding: 25px; |
|
|
border-bottom: 2px solid #8B4513; |
|
|
} |
|
|
.model-composition h4 { |
|
|
font-family: 'Cinzel', serif; |
|
|
color: #5D2E0C; |
|
|
margin: 0 0 20px 0; |
|
|
font-size: 18px; |
|
|
text-align: center; |
|
|
letter-spacing: 1px; |
|
|
} |
|
|
.composition-list { |
|
|
list-style: none; |
|
|
padding: 0; |
|
|
margin: 0; |
|
|
display: grid; |
|
|
gap: 15px; |
|
|
} |
|
|
.composition-list li { |
|
|
color: #2F1B14; |
|
|
display: flex; |
|
|
align-items: baseline; |
|
|
gap: 12px; |
|
|
padding: 10px; |
|
|
background: rgba(245, 222, 179, 0.5); |
|
|
border-radius: 6px; |
|
|
border-left: 4px solid #8B4513; |
|
|
} |
|
|
.model-component { |
|
|
font-weight: 600; |
|
|
min-width: 120px; |
|
|
} |
|
|
.model-description { |
|
|
padding: 25px; |
|
|
background: linear-gradient(145deg, #F5DEB3, #F4E4BC); |
|
|
} |
|
|
.metrics-section { |
|
|
margin-bottom: 30px; |
|
|
} |
|
|
.metrics-section details { |
|
|
background: linear-gradient(145deg, #F5DEB3, #DEB887); |
|
|
border: 3px solid #8B4513; |
|
|
border-radius: 10px; |
|
|
padding: 20px; |
|
|
margin-bottom: 20px; |
|
|
box-shadow: 0 4px 10px rgba(0, 0, 0, 0.2); |
|
|
} |
|
|
.metrics-section summary { |
|
|
font-family: 'Cinzel', serif; |
|
|
color: #5D2E0C; |
|
|
font-size: 18px; |
|
|
cursor: pointer; |
|
|
outline: none; |
|
|
padding: 10px 0; |
|
|
text-align: center; |
|
|
font-weight: 500; |
|
|
letter-spacing: 1px; |
|
|
} |
|
|
.creator-section { |
|
|
margin: 25px 0; |
|
|
text-align: center; |
|
|
} |
|
|
.creator-badge { |
|
|
display: inline-flex; |
|
|
align-items: center; |
|
|
background: linear-gradient(145deg, #CD853F, #D2691E); |
|
|
border: 3px solid #8B4513; |
|
|
border-radius: 10px; |
|
|
padding: 15px 20px; |
|
|
box-shadow: 0 4px 10px rgba(0, 0, 0, 0.2); |
|
|
} |
|
|
.creator-label { |
|
|
color: #2F1B14; |
|
|
font-size: 14px; |
|
|
margin-right: 10px; |
|
|
font-weight: 500; |
|
|
} |
|
|
.creator-link { |
|
|
display: flex; |
|
|
align-items: center; |
|
|
gap: 8px; |
|
|
color: #2F1B14; |
|
|
text-decoration: none; |
|
|
transition: all 0.3s ease; |
|
|
} |
|
|
.creator-name { |
|
|
font-weight: 600; |
|
|
} |
|
|
.creator-arrow { |
|
|
font-size: 16px; |
|
|
transition: transform 0.3s ease; |
|
|
} |
|
|
.creator-link:hover .creator-arrow { |
|
|
transform: translateX(5px); |
|
|
} |
|
|
.link-arrow { |
|
|
display: inline-block; |
|
|
transition: transform 0.3s ease; |
|
|
} |
|
|
a:hover .link-arrow { |
|
|
transform: translateX(3px); |
|
|
} |
|
|
.axolotl-container { |
|
|
text-align: center; |
|
|
margin: 35px 0; |
|
|
} |
|
|
.axolotl-container img { |
|
|
max-width: 300px; |
|
|
border-radius: 10px; |
|
|
box-shadow: 0 6px 15px rgba(0, 0, 0, 0.3); |
|
|
border: 3px solid #8B4513; |
|
|
filter: sepia(30%) contrast(110%); |
|
|
} |
|
|
.fossil-texture { |
|
|
position: relative; |
|
|
} |
|
|
.fossil-texture::after { |
|
|
content: ''; |
|
|
position: absolute; |
|
|
top: 0; |
|
|
left: 0; |
|
|
right: 0; |
|
|
bottom: 0; |
|
|
background-image: |
|
|
radial-gradient(circle at 25% 25%, rgba(139, 69, 19, 0.05) 2px, transparent 2px), |
|
|
radial-gradient(circle at 75% 75%, rgba(160, 82, 45, 0.05) 1px, transparent 1px); |
|
|
background-size: 50px 50px, 30px 30px; |
|
|
pointer-events: none; |
|
|
} |
|
|
</style> |
|
|
</head> |
|
|
<body> |
|
|
<div class="container fossil-texture"> |
|
|
<div class="header"> |
|
|
<h1>Austral 32B GLM4 Winton</h1> |
|
|
</p> |
|
|
</div> |
|
|
<div class="info"> |
|
|
<img src="https://cdn-uploads.huggingface.co/production/uploads/66c26b6fb01b19d8c3c2467b/jxUvuFK1bdOdAPiYIcBW5.jpeg" alt="Model banner"> |
|
|
<div style="text-align: center;"> |
|
|
<div class="creator-section"> |
|
|
<div class="creator-badge"> |
|
|
<span class="creator-label">Trained by</span> |
|
|
<a href="https://huggingface.co/Delta-Vector" target="_blank" class="creator-link"> |
|
|
<span class="creator-name">Delta-Vector</span> |
|
|
</a> |
|
|
</div> |
|
|
</div> |
|
|
<div class="model-info"> |
|
|
<h2>Overview</h2> |
|
|
<div class="info-card"> |
|
|
<div class="info-header"> |
|
|
<h3>Austral 32B - Winton</h3> |
|
|
<div class="model-tags"> |
|
|
<span class="model-tag">Codex Finetune</span> |
|
|
<span class="model-tag">GLM-4-Tulu Based</span> |
|
|
<span class ="model-tag">KTO enhanced</span> |
|
|
<span class ="model-tag">Adventure/Roleplay generalist</span> |
|
|
<span class="model-tag">32B Sized model</span> |
|
|
</div> |
|
|
</div> |
|
|
<div class="model-description"> |
|
|
<p style="font-weight: bold; font-style: italic;">More than 1.5-metres tall, about six-metres long and up to 1000-kilograms heavy, Australovenator Wintonensis was a fast and agile hunter. The largest known Australian theropod.</p> |
|
|
<p>This is a finetune of Delta-Vector/GLM-4-32B-Tulu-Instruct to be a generalist Roleplay/Adventure model. I've removed some of the "slops" that i noticed in an otherwise great model aswell as improving the general writing of the model, This was a multi-stage finetune, all previous checkpoints are released aswell. In testing it has shown to be a great model for Adventure cards & Roleplay, Often pushing the plot forward better then other models, While avoiding some of the slops you'd find in models from Drummer and Co.</p> |
|
|
<p>Support my finetunes / Me on Kofi: https://Ko-fi.com/deltavector | Thank you to Auri for helping/Testing ♥</p> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
<div class="section"> |
|
|
<h2>Quants</h2> |
|
|
<div class="info-card"> |
|
|
<div class="model-composition"> |
|
|
<h4>Quants Formats</h4> |
|
|
<ul class="composition-list"> |
|
|
<li><span class="model-component"><a href="https://huggingface.co/bartowski/Delta-Vector_Austral-32B-GLM4-Winton-GGUF" target="_blank">GGUF</a></span>For use with LLama.cpp & Forks</li> |
|
|
<li><span class="model-component"><a href="https://huggingface.co/ArtusDev/Delta-Vector_Austral-32B-GLM4-Winton-EXL3" target="_blank">EXL3</a></span>For use with TabbyAPI</li> |
|
|
</ul> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
<div class="section"> |
|
|
<h2>Chat Format</h2> |
|
|
<p>This model utilizes ChatML.</p> |
|
|
<pre><code><|im_start|>user |
|
|
Hi there!<|im_end|> |
|
|
<|im_start|>assistant |
|
|
Nice to meet you!<|im_end|> |
|
|
<|im_start|>user |
|
|
Can I ask a question?<|im_end|> |
|
|
<|im_start|>assistant</code></pre> |
|
|
</div> |
|
|
<div class="section"> |
|
|
<h2>Training</h2> |
|
|
<p>As is known by now, I trained off my own Instruct tune from base. `Delta-Vector/GLM-4-32B-Tulu-Instruct`, After which it was trained for 4 epochs on a datamix of Light Novels/Natural Human writing datasets, etc, The resulting model is kinda incoherent, so we end up having to KTO the model to improve coherency and cohesiveness but that resulted in the model not being as "creative" as once thought, So usually i'd SFT with Pocketdoc's rep-remover data, however this time, I decided to convert the dataset into a KTO format and that resulted in a better model. Thankies to Pocket for that dataset.</p> |
|
|
<div class="metrics-section"> |
|
|
<details> |
|
|
<summary>Config(Post-KTO V2)</summary> |
|
|
<pre><code>https://wandb.ai/new-eden/Austral-32B/runs/zlhv6tfw?nw=nwuserdeltavector</code></pre> |
|
|
</details> |
|
|
</div> |
|
|
</div> |
|
|
<div class="section"> |
|
|
<h2Fossil Formation Process</h2> |
|
|
<p>This model was trained over 4 epochs using 8 x A100s (Ty to my work, Quixi.AI) for the base SFT, Then i used KTO to clean up some coherency issues for 1 epoch, then finally training for another 1 epoch on Rep_Remover to delete slops. Total was roughly 80 hours total.</p> |
|
|
<p style="text-align: center; margin-top: 20px;"> |
|
|
<div class="axolotl-container"> |
|
|
<a href="https://github.com/OpenAccess-AI-Collective/axolotl" target="_blank"> |
|
|
<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl"> |
|
|
</a> |
|
|
</div> |
|
|
<div class="section"> |
|
|
<h2>Credits</h2> |
|
|
<p>TYSM to my friends: Auri, Lucy, Trappu, Alicat, Kubernetes Bad, Intervitens, NyxKrage & Kalomaze</p> |
|
|
</p> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</body> |
|
|
</html> |
