import os import sys import time import subprocess from pathlib import Path import logging import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry from typing import Optional from dotenv import load_dotenv from huggingface_hub import HfApi, create_repo # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def setup_requests_session( retries: int = 5, backoff_factor: float = 1.0, status_forcelist: Optional[list] = None ) -> requests.Session: """Configure requests session with retries.""" if status_forcelist is None: status_forcelist = [408, 429, 500, 502, 503, 504] session = requests.Session() retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=backoff_factor, status_forcelist=status_forcelist, ) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) return session def check_network_connectivity(host: str = "8.8.8.8", timeout: int = 5) -> bool: """Check if network is accessible.""" try: # Try DNS resolution first subprocess.run( ["ping", "-c", "1", "-W", str(timeout), host], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True ) return True except subprocess.CalledProcessError: return False def check_huggingface_connectivity(timeout: int = 5) -> bool: """Check if Hugging Face is accessible.""" session = setup_requests_session() try: response = session.get("https://huggingface.co", timeout=timeout) return response.status_code == 200 except: return False def wait_for_network( max_attempts: int = 5, delay: int = 10, hosts: Optional[list] = None ) -> bool: """Wait for network connectivity.""" if hosts is None: hosts = ["8.8.8.8", "1.1.1.1"] for attempt in range(max_attempts): logger.info(f"Checking network connectivity (attempt {attempt + 1}/{max_attempts})") # Try different DNS servers for host in hosts: if check_network_connectivity(host): logger.info(f"Network connectivity established via {host}") return True # Check Hugging Face specifically if check_huggingface_connectivity(): logger.info("Hugging Face is accessible") return True if attempt < max_attempts - 1: logger.warning(f"Network check failed. Waiting {delay} seconds before retry...") time.sleep(delay) return False def upload_to_huggingface(): """Upload the project to Hugging Face.""" creds_path = None try: # Load environment variables load_dotenv() token = os.getenv("HUGGINGFACE_TOKEN") if not token: raise ValueError("HUGGINGFACE_TOKEN not found in environment variables") # Check network connectivity with increased timeout if not wait_for_network(max_attempts=10, delay=15): raise ConnectionError("Failed to establish network connectivity") # Initialize Hugging Face API with retry session session = setup_requests_session(retries=7, backoff_factor=2.0) api = HfApi(token=token, endpoint="https://huggingface.co") # Define Space name (modify as needed) space_name = "agentic-system" space_id = f"nananie143/{space_name}" # Create or get existing Space with retries and force hardware restart max_attempts = 3 for attempt in range(max_attempts): try: space_info = api.create_repo( repo_id=space_id, repo_type="space", space_sdk="gradio", private=False, exist_ok=True, hardware={"accelerator": "t4-medium"}, storage={"hf": {"root": "/data"}}, ) logger.info(f"Space ready: {space_info.url}") # Force hardware restart to ensure clean environment try: api.request_space_hardware( repo_id=space_id, hardware="t4-medium", sleep_time=2 ) logger.info("Requested hardware restart") except Exception as e: logger.warning(f"Hardware restart request failed: {e}") break except Exception as e: if attempt == max_attempts - 1: logger.error(f"Error creating/accessing Space after {max_attempts} attempts: {e}") raise logger.warning(f"Attempt {attempt + 1} failed, retrying...") time.sleep(5 * (attempt + 1)) # Add .gitattributes to ensure proper file handling gitattributes_content = """ *.py text eol=lf *.sh text eol=lf *.yml text eol=lf *.txt text eol=lf requirements.txt text eol=lf """ with open(".gitattributes", "w") as f: f.write(gitattributes_content.strip()) # Files to exclude from upload exclude_patterns = [ "__pycache__", "*.pyc", ".git", ".env", ".env.example", "models/*", "flagged/*", ".pytest_cache", "*.log", "*.gguf", ".gitignore", "*.backup", "*.bak*", "*.patch", "*.temp", ".DS_Store" ] # Important files to ensure are included important_files = [ "app.py", "agentic_system.py", "requirements.txt", "space.yml", "download_models_space.py", "app_space.sh", "orchestrator.py", "team_management.py", "meta_learning.py", "config.py", "upload_to_hub.py", ".gitattributes" ] # Prepare files for upload with validation files_to_upload = [] root_path = Path(".") # First add important files with validation for file in important_files: file_path = Path(file) if file_path.is_file(): if file_path.stat().st_size > 0: # Check if file is not empty files_to_upload.append(str(file_path)) else: logger.warning(f"Skipping empty file: {file}") else: logger.warning(f"Important file not found: {file}") # Then add other files with validation for path in root_path.rglob("*"): if path.is_file(): relative_path = str(path.relative_to(root_path)) if relative_path not in files_to_upload: # Skip if already added skip = False for pattern in exclude_patterns: if Path(relative_path).match(pattern): skip = True break if not skip and path.stat().st_size > 0: # Check if file is not empty files_to_upload.append(relative_path) # Upload files with retry mechanism logger.info("Starting file upload...") total_files = len(files_to_upload) for idx, file_path in enumerate(files_to_upload, 1): max_retries = 3 retry_count = 0 while retry_count < max_retries: try: logger.info(f"[{idx}/{total_files}] Uploading: {file_path}") api.upload_file( path_or_fileobj=file_path, path_in_repo=file_path, repo_id=space_id, repo_type="space" ) logger.info(f"✓ Uploaded: {file_path}") break except Exception as e: retry_count += 1 if retry_count == max_retries: logger.error(f"Failed to upload {file_path} after {max_retries} attempts: {e}") else: logger.warning(f"Retry {retry_count}/{max_retries} for {file_path}: {e}") time.sleep(5 * retry_count) logger.info(f"Space updated successfully! Visit: https://huggingface.co/spaces/{space_id}") except Exception as e: logger.error(f"Error uploading to Hugging Face: {e}") raise finally: if creds_path and os.path.exists(creds_path): os.remove(creds_path) if __name__ == "__main__": upload_to_huggingface()