fix: make mlflow_logger import optional with no-op fallback
All checks were successful
Build and Publish ray-serve-apps / build-and-publish (push) Successful in 11s
All checks were successful
Build and Publish ray-serve-apps / build-and-publish (push) Successful in 11s
The strixhalo LLM worker uses py_executable pointing to the Docker image venv which doesn't have the updated ray-serve-apps package. Wrap all InferenceLogger imports in try/except and guard usage with None checks so apps degrade gracefully without MLflow logging.
This commit is contained in:
@@ -10,7 +10,10 @@ from typing import Any
|
||||
|
||||
from ray import serve
|
||||
|
||||
from ray_serve.mlflow_logger import InferenceLogger
|
||||
try:
|
||||
from ray_serve.mlflow_logger import InferenceLogger
|
||||
except ImportError:
|
||||
InferenceLogger = None
|
||||
|
||||
|
||||
@serve.deployment(name="LLMDeployment", num_replicas=1)
|
||||
@@ -40,19 +43,22 @@ class LLMDeployment:
|
||||
print(f"Model {self.model_id} async engine created")
|
||||
|
||||
# MLflow metrics
|
||||
self._mlflow = InferenceLogger(
|
||||
experiment_name="ray-serve-llm",
|
||||
run_name=f"llm-{self.model_id.split('/')[-1]}",
|
||||
tags={"model.name": self.model_id, "model.framework": "vllm", "gpu": "strixhalo"},
|
||||
flush_every=5,
|
||||
)
|
||||
self._mlflow.initialize(
|
||||
params={
|
||||
"model_id": self.model_id,
|
||||
"max_model_len": str(self.max_model_len),
|
||||
"gpu_memory_utilization": str(self.gpu_memory_utilization),
|
||||
}
|
||||
)
|
||||
if InferenceLogger is not None:
|
||||
self._mlflow = InferenceLogger(
|
||||
experiment_name="ray-serve-llm",
|
||||
run_name=f"llm-{self.model_id.split('/')[-1]}",
|
||||
tags={"model.name": self.model_id, "model.framework": "vllm", "gpu": "strixhalo"},
|
||||
flush_every=5,
|
||||
)
|
||||
self._mlflow.initialize(
|
||||
params={
|
||||
"model_id": self.model_id,
|
||||
"max_model_len": str(self.max_model_len),
|
||||
"gpu_memory_utilization": str(self.gpu_memory_utilization),
|
||||
}
|
||||
)
|
||||
else:
|
||||
self._mlflow = None
|
||||
|
||||
async def __call__(self, request: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
@@ -96,15 +102,16 @@ class LLMDeployment:
|
||||
completion_tokens = len(generated_text.split())
|
||||
|
||||
# Log to MLflow
|
||||
self._mlflow.log_request(
|
||||
latency_s=latency,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
tokens_per_second=completion_tokens / latency if latency > 0 else 0,
|
||||
temperature=temperature,
|
||||
max_tokens_requested=max_tokens,
|
||||
)
|
||||
if self._mlflow:
|
||||
self._mlflow.log_request(
|
||||
latency_s=latency,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
tokens_per_second=completion_tokens / latency if latency > 0 else 0,
|
||||
temperature=temperature,
|
||||
max_tokens_requested=max_tokens,
|
||||
)
|
||||
|
||||
# Return OpenAI-compatible response
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user