feat: fetch directly from hf

index.html

@@ -276,7 +276,7 @@
 
         <div class="card">
             <h2>Configuration</h2>
-            <div class="config-group">
+             <div class="config-group">
                 <div class="checkbox-wrapper">
                     <input type="checkbox" id="addGenerationPrompt">
                     <label for="addGenerationPrompt">Add Generation Prompt</label>
@@ -286,6 +286,10 @@
                      <input type="text" id="eosToken" placeholder="EOS Token (e.g., </s>)" style="flex-grow: 1;">
                 </div>
             </div>
+             <div class="input-group">
+                <input type="text" id="repoUrl" placeholder="Enter Hugging Face Repo URL" style="flex-grow: 1">
+                <button onclick="handleFetchConfig()">Fetch Config</button>
+            </div>
             
             <h2>Template</h2>
             <textarea id="template" placeholder="Enter your template here..."></textarea>
@@ -433,6 +437,84 @@
 
         // Initialize
         updateMessageDisplay();
+
+        // New Functions for fetching tokenizer config
+        function parseRepoUrl(url) {
+            const hfUrlRegex = /^(?:https?:\/\/)?(?:huggingface\.co|hf\.co)\/([^/]+)\/([^/]+)$/;
+            const shortRegex = /^([^/]+)\/([^/]+)$/;
+
+            let match = url.match(hfUrlRegex);
+            if (!match) {
+                match = url.match(shortRegex);
+            }
+
+            if (match) {
+                return { user: match[1], repo: match[2] };
+            }
+            return null;
+        }
+
+        async function fetchTokenizerConfig(user, repo) {
+            const apiUrl = `https://huggingface.co/${user}/${repo}/raw/main/tokenizer_config.json`;
+            try {
+                const response = await fetch(apiUrl);
+                if (!response.ok) {
+                    throw new Error(`HTTP error! status: ${response.status}`);
+                }
+                return await response.json();
+            } catch (error) {
+                console.error("Error fetching tokenizer config:", error);
+                displayError(`Failed to fetch tokenizer config: ${error.message}`);
+                return null;
+            }
+        }
+
+        function displayError(message) {
+            console.error(message);
+        }
+
+        function populateConfigFields(config) {
+            const bosTokenInput = document.getElementById('bosToken');
+            const eosTokenInput = document.getElementById('eosToken');
+            const templateTextarea = document.getElementById('template');
+
+            bosTokenInput.value = config?.bos_token ?? "";
+            eosTokenInput.value = config?.eos_token ?? "";
+
+            let chatTemplate = config?.chat_template ?? "";
+            
+            // Decode HTML entities
+            if (chatTemplate) {
+                const tempElement = document.createElement('div');
+                tempElement.innerHTML = chatTemplate;
+                chatTemplate = tempElement.textContent;
+            }
+            
+            templateTextarea.value = chatTemplate;
+        }
+
+
+        async function handleFetchConfig() {
+            const repoUrl = document.getElementById('repoUrl').value;
+            const repoInfo = parseRepoUrl(repoUrl);
+
+            if (!repoInfo) {
+                displayError("Invalid Hugging Face repository URL format.");
+                return;
+            }
+
+            document.getElementById('repoUrl').parentElement.classList.add('loading');
+
+            const { user, repo } = repoInfo;
+            const config = await fetchTokenizerConfig(user, repo);
+
+            document.getElementById('repoUrl').parentElement.classList.remove('loading');
+
+            if (config) {
+                console.log("Tokenizer Config:", JSON.stringify(config, null, 2));
+                populateConfigFields(config);
+            }
+        }
     </script>
 </body>
 </html>