feat: add MLflow experiment tracking to all 4 Gradio UIs
Each UI now logs per-request metrics to MLflow: - llm.py: latency, tokens/sec, prompt/completion tokens (gradio-llm-tuning) - embeddings.py: latency, text length, batch size (gradio-embeddings-tuning) - stt.py: latency, audio duration, real-time factor (gradio-stt-tuning) - tts.py: latency, text length, audio duration (gradio-tts-tuning) Uses try/except guarded imports so UIs still work if MLflow is unreachable. Persistent run per Gradio instance, batched metric logging via MlflowClient.log_batch().
This commit is contained in:
68
tts.py
68
tts.py
@@ -37,6 +37,66 @@ MLFLOW_TRACKING_URI = os.environ.get(
|
||||
"http://mlflow.mlflow.svc.cluster.local:80"
|
||||
)
|
||||
|
||||
# ─── MLflow experiment tracking ──────────────────────────────────────────
|
||||
try:
|
||||
import mlflow
|
||||
from mlflow.tracking import MlflowClient
|
||||
|
||||
mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)
|
||||
_mlflow_client = MlflowClient()
|
||||
|
||||
_experiment = _mlflow_client.get_experiment_by_name("gradio-tts-tuning")
|
||||
if _experiment is None:
|
||||
_experiment_id = _mlflow_client.create_experiment(
|
||||
"gradio-tts-tuning",
|
||||
artifact_location="/mlflow/artifacts/gradio-tts-tuning",
|
||||
)
|
||||
else:
|
||||
_experiment_id = _experiment.experiment_id
|
||||
|
||||
_mlflow_run = mlflow.start_run(
|
||||
experiment_id=_experiment_id,
|
||||
run_name=f"gradio-tts-{os.environ.get('HOSTNAME', 'local')}",
|
||||
tags={"service": "gradio-tts", "endpoint": TTS_URL},
|
||||
)
|
||||
_mlflow_run_id = _mlflow_run.info.run_id
|
||||
_mlflow_step = 0
|
||||
MLFLOW_ENABLED = True
|
||||
logger.info("MLflow tracking enabled: experiment=%s run=%s", _experiment_id, _mlflow_run_id)
|
||||
except Exception as exc:
|
||||
logger.warning("MLflow tracking disabled: %s", exc)
|
||||
_mlflow_client = None
|
||||
_mlflow_run_id = None
|
||||
_mlflow_step = 0
|
||||
MLFLOW_ENABLED = False
|
||||
|
||||
|
||||
def _log_tts_metrics(
|
||||
latency: float, audio_duration: float, text_chars: int, language: str,
|
||||
) -> None:
|
||||
"""Log TTS inference metrics to MLflow (non-blocking best-effort)."""
|
||||
global _mlflow_step
|
||||
if not MLFLOW_ENABLED or _mlflow_client is None:
|
||||
return
|
||||
try:
|
||||
_mlflow_step += 1
|
||||
ts = int(time.time() * 1000)
|
||||
rtf = latency / audio_duration if audio_duration > 0 else 0
|
||||
cps = text_chars / latency if latency > 0 else 0
|
||||
_mlflow_client.log_batch(
|
||||
_mlflow_run_id,
|
||||
metrics=[
|
||||
mlflow.entities.Metric("latency_s", latency, ts, _mlflow_step),
|
||||
mlflow.entities.Metric("audio_duration_s", audio_duration, ts, _mlflow_step),
|
||||
mlflow.entities.Metric("realtime_factor", rtf, ts, _mlflow_step),
|
||||
mlflow.entities.Metric("chars_per_second", cps, ts, _mlflow_step),
|
||||
mlflow.entities.Metric("text_chars", text_chars, ts, _mlflow_step),
|
||||
],
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("MLflow log failed", exc_info=True)
|
||||
|
||||
|
||||
# HTTP client with longer timeout for audio generation
|
||||
client = httpx.Client(timeout=120.0)
|
||||
|
||||
@@ -93,6 +153,14 @@ def synthesize_speech(text: str, language: str) -> tuple[str, tuple[int, np.ndar
|
||||
|
||||
# Status message
|
||||
status = f"✅ Generated {duration:.2f}s of audio in {latency*1000:.0f}ms"
|
||||
|
||||
# Log to MLflow
|
||||
_log_tts_metrics(
|
||||
latency=latency,
|
||||
audio_duration=duration,
|
||||
text_chars=len(text),
|
||||
language=lang_code,
|
||||
)
|
||||
|
||||
# Metrics
|
||||
metrics = f"""
|
||||
|
||||
Reference in New Issue
Block a user