Prathamesh Sarjerao Vaidya
commited on
Commit
Β·
785835e
1
Parent(s):
3b1fac6
fixed cache hf_spaces issue
Browse files- .dockerignore +58 -0
- Dockerfile +5 -1
- requirements.txt +2 -1
- run_app.py +29 -167
- startup.py +101 -28
.dockerignore
ADDED
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Ignore cache and temporary files
|
2 |
+
__pycache__/
|
3 |
+
*.pyc
|
4 |
+
*.pyo
|
5 |
+
*.pyd
|
6 |
+
.Python
|
7 |
+
*.so
|
8 |
+
.pytest_cache/
|
9 |
+
.coverage
|
10 |
+
htmlcov/
|
11 |
+
.tox/
|
12 |
+
.cache
|
13 |
+
nosetests.xml
|
14 |
+
coverage.xml
|
15 |
+
*.cover
|
16 |
+
.hypothesis/
|
17 |
+
|
18 |
+
# Ignore model cache (will be downloaded fresh)
|
19 |
+
model_cache/
|
20 |
+
outputs/
|
21 |
+
uploads/
|
22 |
+
temp_files/
|
23 |
+
demo_results/
|
24 |
+
|
25 |
+
# Ignore development files
|
26 |
+
.env
|
27 |
+
.env.local
|
28 |
+
.env.development.local
|
29 |
+
.env.test.local
|
30 |
+
.env.production.local
|
31 |
+
|
32 |
+
# Ignore OS files
|
33 |
+
.DS_Store
|
34 |
+
Thumbs.db
|
35 |
+
|
36 |
+
# Ignore IDE files
|
37 |
+
.vscode/
|
38 |
+
.idea/
|
39 |
+
*.swp
|
40 |
+
*.swo
|
41 |
+
|
42 |
+
# Ignore git
|
43 |
+
.git/
|
44 |
+
.gitignore
|
45 |
+
|
46 |
+
# Ignore documentation
|
47 |
+
*.md
|
48 |
+
!README.md
|
49 |
+
|
50 |
+
# Ignore test files
|
51 |
+
test_*.py
|
52 |
+
*_test.py
|
53 |
+
tests/
|
54 |
+
|
55 |
+
# Ignore temporary files
|
56 |
+
*.tmp
|
57 |
+
*.temp
|
58 |
+
*.log
|
Dockerfile
CHANGED
@@ -68,7 +68,11 @@ ENV PYTHONPATH=/app \
|
|
68 |
CTRANSLATE2_FORCE_CPU_ISA=generic \
|
69 |
# Disable problematic features
|
70 |
TF_CPP_MIN_LOG_LEVEL=2 \
|
71 |
-
TOKENIZERS_PARALLELISM=false
|
|
|
|
|
|
|
|
|
72 |
|
73 |
# Expose port for Hugging Face Spaces
|
74 |
EXPOSE 7860
|
|
|
68 |
CTRANSLATE2_FORCE_CPU_ISA=generic \
|
69 |
# Disable problematic features
|
70 |
TF_CPP_MIN_LOG_LEVEL=2 \
|
71 |
+
TOKENIZERS_PARALLELISM=false \
|
72 |
+
# Fix executable stack issues
|
73 |
+
ONNX_EXECUTION_PROVIDER=cpu \
|
74 |
+
# Disable problematic optimizations
|
75 |
+
OMP_NUM_THREADS=1
|
76 |
|
77 |
# Expose port for Hugging Face Spaces
|
78 |
EXPOSE 7860
|
requirements.txt
CHANGED
@@ -24,7 +24,8 @@ pyannote.metrics==3.2.1
|
|
24 |
|
25 |
# Performance & Optimization
|
26 |
numba==0.58.1
|
27 |
-
onnxruntime
|
|
|
28 |
accelerate==0.20.3
|
29 |
|
30 |
# Core Utilities
|
|
|
24 |
|
25 |
# Performance & Optimization
|
26 |
numba==0.58.1
|
27 |
+
# Use CPU-only onnxruntime to avoid executable stack issues
|
28 |
+
onnxruntime-cpu==1.16.3
|
29 |
accelerate==0.20.3
|
30 |
|
31 |
# Core Utilities
|
run_app.py
CHANGED
@@ -1,180 +1,42 @@
|
|
1 |
#!/usr/bin/env python3
|
2 |
"""
|
3 |
-
|
4 |
-
|
5 |
-
This script provides a unified way to run the system with different modes:
|
6 |
-
- Web App Mode: Interactive web interface
|
7 |
-
- Demo Mode: Test system capabilities
|
8 |
-
- CLI Mode: Command-line processing
|
9 |
-
- Test Mode: System validation
|
10 |
-
|
11 |
-
Usage:
|
12 |
-
python run_app.py [--mode web|demo|cli|test] [--port PORT] [--host HOST]
|
13 |
"""
|
14 |
|
|
|
15 |
import os
|
16 |
import sys
|
17 |
-
import argparse
|
18 |
-
import logging
|
19 |
from pathlib import Path
|
20 |
|
21 |
-
# Configure logging
|
22 |
-
logging.basicConfig(
|
23 |
-
level=logging.INFO,
|
24 |
-
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
25 |
-
)
|
26 |
-
logger = logging.getLogger(__name__)
|
27 |
-
|
28 |
-
def run_web_app(host: str = "0.0.0.0", port: int = 8000, debug: bool = False):
|
29 |
-
"""Run the web application."""
|
30 |
-
logger.info("π Starting Web Application...")
|
31 |
-
|
32 |
-
try:
|
33 |
-
# Use the working web_app.py directly
|
34 |
-
import uvicorn
|
35 |
-
from web_app import app
|
36 |
-
|
37 |
-
uvicorn.run(app, host=host, port=port, log_level="info" if debug else "warning")
|
38 |
-
|
39 |
-
except Exception as e:
|
40 |
-
logger.error(f"β Failed to start web app: {e}")
|
41 |
-
sys.exit(1)
|
42 |
-
|
43 |
-
def run_demo():
|
44 |
-
"""Run the demo system."""
|
45 |
-
logger.info("π΅ Starting Demo System...")
|
46 |
-
|
47 |
-
try:
|
48 |
-
from src.demo import main
|
49 |
-
main()
|
50 |
-
|
51 |
-
except Exception as e:
|
52 |
-
logger.error(f"β Failed to run demo: {e}")
|
53 |
-
sys.exit(1)
|
54 |
-
|
55 |
-
def run_tests():
|
56 |
-
"""Run system tests."""
|
57 |
-
logger.info("π§ͺ Running System Tests...")
|
58 |
-
|
59 |
-
try:
|
60 |
-
from src.test_system import main
|
61 |
-
main()
|
62 |
-
|
63 |
-
except Exception as e:
|
64 |
-
logger.error(f"β Failed to run tests: {e}")
|
65 |
-
sys.exit(1)
|
66 |
-
|
67 |
-
def run_cli_mode():
|
68 |
-
"""Run CLI processing mode."""
|
69 |
-
logger.info("π» Starting CLI Mode...")
|
70 |
-
|
71 |
-
try:
|
72 |
-
from src.main import main
|
73 |
-
main()
|
74 |
-
|
75 |
-
except Exception as e:
|
76 |
-
logger.error(f"β Failed to start CLI mode: {e}")
|
77 |
-
sys.exit(1)
|
78 |
-
|
79 |
-
def check_dependencies():
|
80 |
-
"""Check if all required dependencies are available."""
|
81 |
-
logger.info("π Checking dependencies...")
|
82 |
-
|
83 |
-
required_modules = [
|
84 |
-
'src.translator',
|
85 |
-
'src.audio_processor',
|
86 |
-
'src.main',
|
87 |
-
'web_app'
|
88 |
-
]
|
89 |
-
|
90 |
-
missing = []
|
91 |
-
for module in required_modules:
|
92 |
-
try:
|
93 |
-
__import__(module)
|
94 |
-
logger.info(f"β
{module}")
|
95 |
-
except ImportError as e:
|
96 |
-
logger.error(f"β {module}: {e}")
|
97 |
-
missing.append(module)
|
98 |
-
|
99 |
-
if missing:
|
100 |
-
logger.error(f"β Missing modules: {', '.join(missing)}")
|
101 |
-
logger.error("Install dependencies with: pip install -r requirements.txt")
|
102 |
-
return False
|
103 |
-
|
104 |
-
logger.info("β
All dependencies available")
|
105 |
-
return True
|
106 |
-
|
107 |
def main():
|
108 |
-
"""Main
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
)
|
122 |
-
|
123 |
-
parser.add_argument(
|
124 |
-
"--mode",
|
125 |
-
choices=["web", "demo", "cli", "test"],
|
126 |
-
default="web",
|
127 |
-
help="Run mode (default: web)"
|
128 |
-
)
|
129 |
-
|
130 |
-
parser.add_argument(
|
131 |
-
"--port",
|
132 |
-
type=int,
|
133 |
-
default=8000,
|
134 |
-
help="Port for web app (default: 8000)"
|
135 |
-
)
|
136 |
-
|
137 |
-
parser.add_argument(
|
138 |
-
"--host",
|
139 |
-
default="0.0.0.0",
|
140 |
-
help="Host for web app (default: 0.0.0.0)"
|
141 |
-
)
|
142 |
-
|
143 |
-
parser.add_argument(
|
144 |
-
"--debug",
|
145 |
-
action="store_true",
|
146 |
-
help="Enable debug mode"
|
147 |
-
)
|
148 |
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
153 |
)
|
154 |
-
|
155 |
-
args = parser.parse_args()
|
156 |
-
|
157 |
-
logger.info("π΅ Audio Intelligence System")
|
158 |
-
logger.info("=" * 50)
|
159 |
-
|
160 |
-
# Check dependencies unless skipped
|
161 |
-
if not args.skip_deps:
|
162 |
-
if not check_dependencies():
|
163 |
-
logger.error("β Critical dependencies missing. Exiting.")
|
164 |
-
sys.exit(1)
|
165 |
-
|
166 |
-
# Run selected mode
|
167 |
-
if args.mode == "web":
|
168 |
-
run_web_app(host=args.host, port=args.port, debug=args.debug)
|
169 |
-
elif args.mode == "demo":
|
170 |
-
run_demo()
|
171 |
-
elif args.mode == "test":
|
172 |
-
run_tests()
|
173 |
-
elif args.mode == "cli":
|
174 |
-
run_cli_mode()
|
175 |
-
else:
|
176 |
-
logger.error(f"β Unknown mode: {args.mode}")
|
177 |
-
sys.exit(1)
|
178 |
|
179 |
if __name__ == "__main__":
|
180 |
-
main()
|
|
|
1 |
#!/usr/bin/env python3
|
2 |
"""
|
3 |
+
Simple startup script for local development.
|
4 |
+
This is the main entry point for running the application locally.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
5 |
"""
|
6 |
|
7 |
+
import uvicorn
|
8 |
import os
|
9 |
import sys
|
|
|
|
|
10 |
from pathlib import Path
|
11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
12 |
def main():
|
13 |
+
"""Main function for local development."""
|
14 |
+
print("π Starting Audio Intelligence System (Local Development)")
|
15 |
+
|
16 |
+
# Create necessary directories
|
17 |
+
dirs = ['uploads', 'outputs', 'model_cache', 'temp_files', 'demo_results']
|
18 |
+
for d in dirs:
|
19 |
+
Path(d).mkdir(exist_ok=True)
|
20 |
+
print(f"β
Created directory: {d}")
|
21 |
+
|
22 |
+
# Check if web_app.py exists
|
23 |
+
if not Path("web_app.py").exists():
|
24 |
+
print("β web_app.py not found!")
|
25 |
+
sys.exit(1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
26 |
|
27 |
+
print("π Starting web application...")
|
28 |
+
print("π Access the application at: http://127.0.0.1:8000")
|
29 |
+
print("π API documentation at: http://127.0.0.1:8000/docs")
|
30 |
+
print("π Press Ctrl+C to stop the server")
|
31 |
+
|
32 |
+
# Start the server with reload for development
|
33 |
+
uvicorn.run(
|
34 |
+
"web_app:app",
|
35 |
+
host="127.0.0.1",
|
36 |
+
port=8000,
|
37 |
+
reload=True,
|
38 |
+
log_level="info"
|
39 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
40 |
|
41 |
if __name__ == "__main__":
|
42 |
+
main()
|
startup.py
CHANGED
@@ -1,18 +1,23 @@
|
|
1 |
#!/usr/bin/env python3
|
2 |
"""
|
3 |
Startup script for Hugging Face Spaces deployment.
|
|
|
4 |
"""
|
5 |
|
6 |
import os
|
7 |
import subprocess
|
8 |
-
import time
|
9 |
import sys
|
|
|
10 |
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
|
|
|
|
|
|
|
|
16 |
dirs = [
|
17 |
'uploads', 'outputs', 'model_cache', 'temp_files',
|
18 |
'demo_results', '/tmp/matplotlib', '/tmp/fontconfig'
|
@@ -21,37 +26,105 @@ def main():
|
|
21 |
for d in dirs:
|
22 |
try:
|
23 |
os.makedirs(d, mode=0o777, exist_ok=True)
|
24 |
-
|
25 |
except Exception as e:
|
26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
27 |
|
28 |
-
# Try to preload models
|
29 |
try:
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
43 |
except Exception as e:
|
44 |
-
|
45 |
-
|
|
|
|
|
|
|
|
|
46 |
|
47 |
-
# Start the web application
|
48 |
-
print('π Starting web application...')
|
49 |
try:
|
50 |
import uvicorn
|
51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
52 |
except Exception as e:
|
53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
54 |
sys.exit(1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
55 |
|
56 |
if __name__ == '__main__':
|
57 |
main()
|
|
|
1 |
#!/usr/bin/env python3
|
2 |
"""
|
3 |
Startup script for Hugging Face Spaces deployment.
|
4 |
+
Handles model preloading and graceful fallbacks for containerized environments.
|
5 |
"""
|
6 |
|
7 |
import os
|
8 |
import subprocess
|
|
|
9 |
import sys
|
10 |
+
import logging
|
11 |
|
12 |
+
# Configure logging
|
13 |
+
logging.basicConfig(
|
14 |
+
level=logging.INFO,
|
15 |
+
format='%(asctime)s - %(levelname)s - %(message)s'
|
16 |
+
)
|
17 |
+
logger = logging.getLogger(__name__)
|
18 |
+
|
19 |
+
def create_directories():
|
20 |
+
"""Create necessary directories with proper permissions."""
|
21 |
dirs = [
|
22 |
'uploads', 'outputs', 'model_cache', 'temp_files',
|
23 |
'demo_results', '/tmp/matplotlib', '/tmp/fontconfig'
|
|
|
26 |
for d in dirs:
|
27 |
try:
|
28 |
os.makedirs(d, mode=0o777, exist_ok=True)
|
29 |
+
logger.info(f'β
Created directory: {d}')
|
30 |
except Exception as e:
|
31 |
+
logger.warning(f'β οΈ Failed to create directory {d}: {e}')
|
32 |
+
|
33 |
+
def check_dependencies():
|
34 |
+
"""Check if critical dependencies are available."""
|
35 |
+
critical_deps = ['fastapi', 'uvicorn', 'torch', 'transformers']
|
36 |
+
missing_deps = []
|
37 |
+
|
38 |
+
for dep in critical_deps:
|
39 |
+
try:
|
40 |
+
__import__(dep)
|
41 |
+
logger.info(f'β
{dep} available')
|
42 |
+
except ImportError:
|
43 |
+
missing_deps.append(dep)
|
44 |
+
logger.warning(f'β οΈ {dep} not available')
|
45 |
+
|
46 |
+
if missing_deps:
|
47 |
+
logger.error(f'β Missing critical dependencies: {missing_deps}')
|
48 |
+
return False
|
49 |
+
|
50 |
+
return True
|
51 |
+
|
52 |
+
def preload_models():
|
53 |
+
"""Attempt to preload models with graceful fallback."""
|
54 |
+
logger.info('π Attempting model preloading...')
|
55 |
|
|
|
56 |
try:
|
57 |
+
# Check if model_preloader.py exists and is importable
|
58 |
+
import model_preloader
|
59 |
+
logger.info('β
Model preloader module found')
|
60 |
+
|
61 |
+
# Try to run the preloader
|
62 |
+
result = subprocess.run(
|
63 |
+
['python', 'model_preloader.py'],
|
64 |
+
capture_output=True,
|
65 |
+
text=True,
|
66 |
+
timeout=300 # 5 minute timeout
|
67 |
+
)
|
68 |
+
|
69 |
+
if result.returncode == 0:
|
70 |
+
logger.info('β
Models loaded successfully')
|
71 |
+
if result.stdout:
|
72 |
+
logger.info(f'Preloader output: {result.stdout[:500]}...')
|
73 |
+
return True
|
74 |
+
else:
|
75 |
+
logger.warning(f'β οΈ Model preloading failed with return code {result.returncode}')
|
76 |
+
if result.stderr:
|
77 |
+
logger.warning(f'Preloader stderr: {result.stderr[:500]}...')
|
78 |
+
return False
|
79 |
+
|
80 |
+
except subprocess.TimeoutExpired:
|
81 |
+
logger.warning('β οΈ Model preloading timed out, continuing...')
|
82 |
+
return False
|
83 |
except Exception as e:
|
84 |
+
logger.warning(f'β οΈ Model preloading failed: {e}')
|
85 |
+
return False
|
86 |
+
|
87 |
+
def start_web_app():
|
88 |
+
"""Start the web application."""
|
89 |
+
logger.info('π Starting web application...')
|
90 |
|
|
|
|
|
91 |
try:
|
92 |
import uvicorn
|
93 |
+
logger.info('β
Uvicorn imported successfully')
|
94 |
+
|
95 |
+
# Start the server
|
96 |
+
uvicorn.run(
|
97 |
+
'web_app:app',
|
98 |
+
host='0.0.0.0',
|
99 |
+
port=7860,
|
100 |
+
workers=1,
|
101 |
+
log_level='info',
|
102 |
+
access_log=True
|
103 |
+
)
|
104 |
+
except ImportError as e:
|
105 |
+
logger.error(f'β Failed to import uvicorn: {e}')
|
106 |
+
sys.exit(1)
|
107 |
except Exception as e:
|
108 |
+
logger.error(f'β Failed to start web application: {e}')
|
109 |
+
sys.exit(1)
|
110 |
+
|
111 |
+
def main():
|
112 |
+
"""Main startup function."""
|
113 |
+
logger.info('π Starting Multilingual Audio Intelligence System...')
|
114 |
+
|
115 |
+
# Check critical dependencies
|
116 |
+
if not check_dependencies():
|
117 |
+
logger.error('β Critical dependencies missing, exiting...')
|
118 |
sys.exit(1)
|
119 |
+
|
120 |
+
# Create necessary directories
|
121 |
+
create_directories()
|
122 |
+
|
123 |
+
# Attempt model preloading (non-blocking)
|
124 |
+
preload_models()
|
125 |
+
|
126 |
+
# Start the web application
|
127 |
+
start_web_app()
|
128 |
|
129 |
if __name__ == '__main__':
|
130 |
main()
|