Spaces:

djuna
/

Chat-Template-Tester

Running

App Files Files Community

Chat-Template-Tester / index.html

djuna

feat: fetch directly from hf

1a81b28 verified 11 months ago

raw

history blame contribute delete

16.5 kB

	<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="UTF-8">
	<meta name="viewport" content="width=device-width, initial-scale=1.0">
	<title>Chat Template Tester</title>
	<script src="https://mozilla.github.io/nunjucks/files/nunjucks.min.js"></script>
	<style>
	/* CSS Reset and Base styles */
	* {
	margin: 0;
	padding: 0;
	box-sizing: border-box;
	}

	body {
	font-family: system-ui, -apple-system, sans-serif;
	background-color: #f5f5f5;
	line-height: 1.6;
	color: #333;
	}

	/* Container */
	.container {
	max-width: 1200px;
	margin: 0 auto;
	padding: 20px;
	}

	@media (max-width: 768px) {
	.container {
	padding: 12px;
	}
	}

	/* Typography */
	h1 {
	font-size: clamp(1.5rem, 4vw, 2rem);
	margin-bottom: 1.5rem;
	color: #333;
	}

	h2 {
	font-size: clamp(1.2rem, 3vw, 1.5rem);
	margin-bottom: 1rem;
	color: #444;
	}

	/* Card Layout */
	.card {
	background: white;
	padding: clamp(16px, 3vw, 24px);
	border-radius: 12px;
	box-shadow: 0 2px 8px rgba(0,0,0,0.1);
	margin-bottom: 20px;
	}

	/* Form Elements */
	.input-group {
	display: flex;
	gap: 8px;
	margin-bottom: 12px;
	}

	@media (max-width: 600px) {
	.input-group {
	flex-direction: column;
	}

	.input-group select,
	.input-group input,
	.input-group button {
	width: 100%;
	}
	}

	textarea,
	input[type="text"],
	select {
	padding: 12px;
	border: 1.5px solid #ddd;
	border-radius: 8px;
	font-size: 14px;
	background: #fff;
	transition: border-color 0.2s;
	}

	textarea:focus,
	input[type="text"]:focus,
	select:focus {
	outline: none;
	border-color: #4CAF50;
	}

	select {
	min-width: 120px;
	}

	@media (max-width: 600px) {
	select {
	min-width: 100%;
	}
	}

	textarea {
	width: 100%;
	min-height: 120px;
	resize: vertical;
	font-family: inherit;
	}

	/* Buttons */
	button {
	background-color: #4CAF50;
	color: white;
	padding: 12px 20px;
	border: none;
	border-radius: 8px;
	cursor: pointer;
	font-size: 14px;
	font-weight: 500;
	transition: all 0.2s;
	white-space: nowrap;
	}

	button:hover {
	background-color: #45a049;
	transform: translateY(-1px);
	}

	button:active {
	transform: translateY(0);
	}

	button.secondary {
	background-color: #6c757d;
	}

	button.danger {
	background-color: #dc3545;
	}

	button.small {
	padding: 6px 12px;
	font-size: 13px;
	}

	/* Messages Section */
	.message {
	background-color: #f8f9fa;
	padding: 16px;
	border-radius: 8px;
	margin-bottom: 12px;
	display: flex;
	align-items: center;
	gap: 12px;
	flex-wrap: wrap;
	}

	@media (max-width: 600px) {
	.message {
	flex-direction: column;
	align-items: stretch;
	}
	}

	.message-content {
	flex-grow: 1;
	min-width: 200px;
	word-break: break-word;
	}

	.message strong {
	color: #495057;
	margin-right: 8px;
	display: inline-block;
	}

	.message-actions {
	display: flex;
	gap: 6px;
	flex-wrap: wrap;
	}

	@media (max-width: 600px) {
	.message-actions {
	justify-content: flex-end;
	margin-top: 8px;
	}
	}

	/* Configuration section */
	.config-group {
	display: flex;
	align-items: center;
	gap: 16px;
	margin-bottom: 16px;
	flex-wrap: wrap;
	}

	@media (max-width: 600px) {
	.config-group {
	flex-direction: column;
	align-items: stretch;
	}
	}

	/* Output section */
	#output {
	white-space: pre-wrap;
	background-color: #f8f9fa;
	padding: 16px;
	border-radius: 8px;
	font-family: monospace;
	border: 1.5px solid #ddd;
	overflow-x: auto;
	font-size: 14px;
	line-height: 1.5;
	}

	#error {
	white-space: pre-wrap;
	color: #ffffff;
	background-color: #f8090a;
	padding: 16px;
	border-radius: 8px;
	font-family: monospace;
	border: 1.5px solid #ddd;
	overflow-x: auto;
	font-size: 14px;
	line-height: 1.5;
	}

	/* Checkbox styling */
	.checkbox-wrapper {
	display: flex;
	align-items: center;
	gap: 8px;
	}

	input[type="checkbox"] {
	width: 16px;
	height: 16px;
	}

	/* Loading state */
	.loading {
	opacity: 0.7;
	pointer-events: none;
	}

	/* Focus styles for accessibility */
	:focus-visible {
	outline: 2px solid #4CAF50;
	outline-offset: 2px;
	}
	</style>
	</head>
	<body>
	<div class="container">
	<h1>Chat Template Tester</h1>

	<div class="card">
	<h2>Messages</h2>
	<div class="input-group">
	<select id="role">
	<option value="user">User</option>
	<option value="assistant">Assistant</option>
	<option value="system">System</option>
	</select>
	<input type="text" id="content" placeholder="Type your message here..." style="flex-grow: 1">
	<button onclick="addMessage()">Add Message</button>
	</div>
	<div id="messages"></div>
	</div>

	<div class="card">
	<h2>Configuration</h2>
	<div class="config-group">
	<div class="checkbox-wrapper">
	<input type="checkbox" id="addGenerationPrompt">
	<label for="addGenerationPrompt">Add Generation Prompt</label>
	</div>
	<div class="input-group" style="margin-bottom: 0; flex-grow: 1;">
	<input type="text" id="bosToken" placeholder="BOS Token (e.g., <s>)" style="flex-grow: 1;">
	<input type="text" id="eosToken" placeholder="EOS Token (e.g., </s>)" style="flex-grow: 1;">
	</div>
	</div>
	<div class="input-group">
	<input type="text" id="repoUrl" placeholder="Enter Hugging Face Repo URL" style="flex-grow: 1">
	<button onclick="handleFetchConfig()">Fetch Config</button>
	</div>

	<h2>Template</h2>
	<textarea id="template" placeholder="Enter your template here..."></textarea>

	<button onclick="applyTemplate()" style="margin-top: 16px">Apply Template</button>
	</div>

	<div class="card">
	<h2>Output</h2>
	<pre id="output"></pre>
	<pre id="error" style="display: none"></pre>
	</div>
	</div>

	<script>
	let messages = [];
	let editingIndex = null;

	function addMessage() {
	const role = document.getElementById('role').value;
	const content = document.getElementById('content').value;
	if (content.trim()) {
	messages.push({ role, content });
	updateMessageDisplay();
	document.getElementById('content').value = '';
	document.getElementById('content').focus();
	}
	}

	function updateMessageDisplay() {
	const messagesDiv = document.getElementById('messages');
	messagesDiv.innerHTML = messages.map((msg, index) => `
	<div class="message">
	<div class="message-content">
	<strong>${msg.role}:</strong>
	${editingIndex === index
	? `<input type="text" id="editContent" value="${msg.content}" style="width: 100%;">`
	: msg.content}
	</div>
	<div class="message-actions">
	${editingIndex === index
	? `<button class="small" onclick="saveEdit(${index})">Save</button>`
	: `<button class="small secondary" onclick="editMessage(${index})">Edit</button>`}
	<button class="small danger" onclick="removeMessage(${index})">Remove</button>
	<button class="small" onclick="moveMessageUp(${index})" ${index === 0 ? 'disabled' : ''}>↑</button>
	<button class="small" onclick="moveMessageDown(${index})" ${index === messages.length - 1 ? 'disabled' : ''}>↓</button>
	</div>
	</div>
	`).join('');
	}

	function removeMessage(index) {
	messages.splice(index, 1);
	updateMessageDisplay();
	}

	function editMessage(index) {
	editingIndex = index;
	updateMessageDisplay();
	setTimeout(() => {
	const editInput = document.getElementById('editContent');
	if (editInput) {
	editInput.focus();
	editInput.select();
	}
	}, 0);
	}

	function saveEdit(index) {
	const newContent = document.getElementById('editContent').value;
	if (newContent.trim()) {
	messages[index].content = newContent;
	editingIndex = null;
	updateMessageDisplay();
	}
	}

	function moveMessageUp(index) {
	if (index > 0) {
	[messages[index - 1], messages[index]] = [messages[index], messages[index - 1]];
	updateMessageDisplay();
	}
	}

	function moveMessageDown(index) {
	if (index < messages.length - 1) {
	[messages[index], messages[index + 1]] = [messages[index + 1], messages[index]];
	updateMessageDisplay();
	}
	}
	function raiseException(string) {
	document.getElementById("output").style.display = "none";
	document.getElementById("error").style.display = "block";
	document.getElementById('error').textContent = `Error: ${string}`;
	}

	function applyTemplate() {
	document.getElementById("output").style.display = "block";
	document.getElementById("error").style.display = "none";
	const template = document.getElementById('template').value;
	const addGenerationPrompt = document.getElementById('addGenerationPrompt').checked;
	const bosToken = document.getElementById('bosToken').value;
	const eosToken = document.getElementById('eosToken').value;

	const context = {
	messages: messages,
	add_generation_prompt: addGenerationPrompt,
	bos_token: bosToken,
	eos_token: eosToken,
	raise_exception: raiseException,
	};

	try {
	document.getElementById('output').parentElement.classList.add('loading');
	const result = nunjucks.renderString(template, context);
	const decodedResult = result.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&');
	document.getElementById('output').textContent = decodedResult;
	} catch (error) {
	document.getElementById('output').textContent = `Error: ${error.message}`;
	} finally {
	document.getElementById('output').parentElement.classList.remove('loading');
	}
	}

	// Keyboard shortcuts and accessibility
	document.addEventListener('keydown', function(e) {
	// Add message on Enter in the content input
	if (e.target.id === 'content' && e.key === 'Enter' && !e.shiftKey) {
	e.preventDefault();
	addMessage();
	}

	// Save edit on Enter in the edit input
	if (e.target.id === 'editContent' && e.key === 'Enter' && !e.shiftKey) {
	e.preventDefault();
	saveEdit(editingIndex);
	}

	// Cancel edit on Escape
	if (e.key === 'Escape' && editingIndex !== null) {
	editingIndex = null;
	updateMessageDisplay();
	}
	});

	// Initialize
	updateMessageDisplay();

	// New Functions for fetching tokenizer config
	function parseRepoUrl(url) {
	const hfUrlRegex = /^(?:https?:\/\/)?(?:huggingface\.co\|hf\.co)\/([^/]+)\/([^/]+)$/;
	const shortRegex = /^([^/]+)\/([^/]+)$/;

	let match = url.match(hfUrlRegex);
	if (!match) {
	match = url.match(shortRegex);
	}

	if (match) {
	return { user: match[1], repo: match[2] };
	}
	return null;
	}

	async function fetchTokenizerConfig(user, repo) {
	const apiUrl = `https://huggingface.co/${user}/${repo}/raw/main/tokenizer_config.json`;
	try {
	const response = await fetch(apiUrl);
	if (!response.ok) {
	throw new Error(`HTTP error! status: ${response.status}`);
	}
	return await response.json();
	} catch (error) {
	console.error("Error fetching tokenizer config:", error);
	displayError(`Failed to fetch tokenizer config: ${error.message}`);
	return null;
	}
	}

	function displayError(message) {
	console.error(message);
	}

	function populateConfigFields(config) {
	const bosTokenInput = document.getElementById('bosToken');
	const eosTokenInput = document.getElementById('eosToken');
	const templateTextarea = document.getElementById('template');

	bosTokenInput.value = config?.bos_token ?? "";
	eosTokenInput.value = config?.eos_token ?? "";

	let chatTemplate = config?.chat_template ?? "";

	// Decode HTML entities
	if (chatTemplate) {
	const tempElement = document.createElement('div');
	tempElement.innerHTML = chatTemplate;
	chatTemplate = tempElement.textContent;
	}

	templateTextarea.value = chatTemplate;
	}


	async function handleFetchConfig() {
	const repoUrl = document.getElementById('repoUrl').value;
	const repoInfo = parseRepoUrl(repoUrl);

	if (!repoInfo) {
	displayError("Invalid Hugging Face repository URL format.");
	return;
	}

	document.getElementById('repoUrl').parentElement.classList.add('loading');

	const { user, repo } = repoInfo;
	const config = await fetchTokenizerConfig(user, repo);

	document.getElementById('repoUrl').parentElement.classList.remove('loading');

	if (config) {
	console.log("Tokenizer Config:", JSON.stringify(config, null, 2));
	populateConfigFields(config);
	}
	}
	</script>
	</body>
	</html>