Update app.py

app.py

@@ -11,8 +11,16 @@ from collections import Counter
 from typing import List, Tuple, Dict
 import random
 import math
-from datasets import load_dataset
-from transformers import AutoTokenizer
+try:
+    from datasets import load_dataset
+except ImportError:
+    print("datasets non disponibile, usando solo dati sintetici")
+    load_dataset = None
+try:
+    from transformers import AutoTokenizer
+except ImportError:
+    print("transformers non disponibile, usando tokenizer personalizzato")
+    AutoTokenizer = None
 import gradio as gr
 
 class SelfOrganizingTokenizer:
@@ -151,56 +159,54 @@ class AITrainer:
         """Carica dataset pubblici senza API key"""
         datasets = []
         
-        try:
-            # Wikipedia in italiano
-            wiki = load_dataset("wikipedia", "20220301.it", split="train[:10000]")
-            for item in wiki:
-                if len(item['text']) > 100:
-                    datasets.append(item['text'])
-        except:
-            pass
-            
-        try:
-            # Common Crawl
-            cc = load_dataset("cc100", lang="it", split="train[:5000]")
-            for item in cc:
-                if len(item['text']) > 100:
-                    datasets.append(item['text'])
-        except:
-            pass
-            
-        try:
-            # OSCAR
-            oscar = load_dataset("oscar-corpus/OSCAR-2201", "it", split="train[:5000]")
-            for item in oscar:
-                if len(item['text']) > 100:
-                    datasets.append(item['text'])
-        except:
-            pass
+        if load_dataset:
+            try:
+                # Wikipedia in italiano
+                wiki = load_dataset("wikipedia", "20220301.it", split="train[:1000]", trust_remote_code=True)
+                for item in wiki:
+                    if len(item['text']) > 100:
+                        datasets.append(item['text'])
+                print(f"Caricati {len(datasets)} esempi da Wikipedia")
+            except Exception as e:
+                print(f"Wikipedia non disponibile: {e}")
+                
+            try:
+                # Common Crawl
+                cc = load_dataset("cc100", lang="it", split="train[:500]", trust_remote_code=True)
+                for item in cc:
+                    if len(item['text']) > 100:
+                        datasets.append(item['text'])
+                print(f"Caricati esempi da Common Crawl")
+            except Exception as e:
+                print(f"Common Crawl non disponibile: {e}")
         
         # Dataset di testo semplice da URL pubblici
         urls = [
             "https://www.gutenberg.org/files/2000/2000-0.txt",  # Divina Commedia
-            "https://www.gutenberg.org/files/1065/1065-0.txt"   # I Promessi Sposi
         ]
         
         for url in urls:
             try:
-                response = requests.get(url, timeout=30)
+                response = requests.get(url, timeout=10)
                 if response.status_code == 200:
                     text = response.text
-                    chunks = [text[i:i+2000] for i in range(0, len(text), 2000)]
-                    datasets.extend(chunks[:500])
-            except:
+                    # Filtra contenuto utile
+                    lines = text.split('\n')
+                    filtered_lines = [line.strip() for line in lines if len(line.strip()) > 50]
+                    chunks = filtered_lines[:1000]  # Primi 1000 chunk
+                    datasets.extend(chunks)
+                    print(f"Caricati {len(chunks)} chunk da {url}")
+            except Exception as e:
+                print(f"Errore caricamento {url}: {e}")
                 continue
         
-        # Genera dati sintetici se necessario
-        if len(datasets) < 1000:
-            synthetic_texts = self.generate_synthetic_data(5000)
-            datasets.extend(synthetic_texts)
+        # Genera dati sintetici
+        print("Generazione dati sintetici...")
+        synthetic_texts = self.generate_synthetic_data(8000)
+        datasets.extend(synthetic_texts)
         
         self.datasets = datasets[:10000]  # Limita a 10k esempi
-        print(f"Caricati {len(self.datasets)} esempi di training")
+        print(f"Dataset finale: {len(self.datasets)} esempi")
         
     def generate_synthetic_data(self, num_samples):
         """Genera dati sintetici per il training"""