style: apply ruff format to all files
Some checks failed
CI / Lint (push) Successful in 1m46s
CI / Test (push) Successful in 1m44s
CI / Publish (push) Failing after 19s
CI / Notify (push) Successful in 1s

This commit is contained in:
2026-02-13 11:05:26 -05:00
parent 1c841729a0
commit ca5bef9664
7 changed files with 89 additions and 222 deletions

View File

@@ -63,10 +63,7 @@ def cmd_list_experiments(args):
def cmd_compare(args):
"""Compare recent runs in an experiment."""
analyzer = ExperimentAnalyzer(
args.experiment,
tracking_uri=args.tracking_uri
)
analyzer = ExperimentAnalyzer(args.experiment, tracking_uri=args.tracking_uri)
if args.run_ids:
run_ids = args.run_ids.split(",")
@@ -82,10 +79,7 @@ def cmd_compare(args):
def cmd_best(args):
"""Find the best run by a metric."""
analyzer = ExperimentAnalyzer(
args.experiment,
tracking_uri=args.tracking_uri
)
analyzer = ExperimentAnalyzer(args.experiment, tracking_uri=args.tracking_uri)
best_run = analyzer.get_best_run(
metric=args.metric,
@@ -115,10 +109,7 @@ def cmd_best(args):
def cmd_summary(args):
"""Get metrics summary for an experiment."""
analyzer = ExperimentAnalyzer(
args.experiment,
tracking_uri=args.tracking_uri
)
analyzer = ExperimentAnalyzer(args.experiment, tracking_uri=args.tracking_uri)
summary = analyzer.get_metrics_summary(
hours=args.hours,
@@ -201,10 +192,7 @@ def cmd_promote(args):
def cmd_query(args):
"""Query runs with a filter."""
analyzer = ExperimentAnalyzer(
args.experiment,
tracking_uri=args.tracking_uri
)
analyzer = ExperimentAnalyzer(args.experiment, tracking_uri=args.tracking_uri)
runs = analyzer.search_runs(
filter_string=args.filter or "",

View File

@@ -22,41 +22,24 @@ class MLflowConfig:
# Tracking server URIs
tracking_uri: str = field(
default_factory=lambda: os.environ.get(
"MLFLOW_TRACKING_URI",
"http://mlflow.mlflow.svc.cluster.local:80"
)
default_factory=lambda: os.environ.get("MLFLOW_TRACKING_URI", "http://mlflow.mlflow.svc.cluster.local:80")
)
external_uri: str = field(
default_factory=lambda: os.environ.get(
"MLFLOW_EXTERNAL_URI",
"https://mlflow.lab.daviestechlabs.io"
)
default_factory=lambda: os.environ.get("MLFLOW_EXTERNAL_URI", "https://mlflow.lab.daviestechlabs.io")
)
# Artifact storage (NFS PVC mount)
artifact_location: str = field(
default_factory=lambda: os.environ.get(
"MLFLOW_ARTIFACT_LOCATION",
"/mlflow/artifacts"
)
default_factory=lambda: os.environ.get("MLFLOW_ARTIFACT_LOCATION", "/mlflow/artifacts")
)
# Default experiment settings
default_experiment: str = field(
default_factory=lambda: os.environ.get(
"MLFLOW_DEFAULT_EXPERIMENT",
"llm-workflows"
)
default_factory=lambda: os.environ.get("MLFLOW_DEFAULT_EXPERIMENT", "llm-workflows")
)
# Service identification
service_name: str = field(
default_factory=lambda: os.environ.get(
"OTEL_SERVICE_NAME",
"unknown-service"
)
)
service_name: str = field(default_factory=lambda: os.environ.get("OTEL_SERVICE_NAME", "unknown-service"))
# Additional tags to add to all runs
default_tags: Dict[str, str] = field(default_factory=dict)
@@ -85,10 +68,7 @@ def get_tracking_uri(external: bool = False) -> str:
return config.external_uri if external else config.tracking_uri
def get_mlflow_client(
tracking_uri: Optional[str] = None,
configure_global: bool = True
) -> MlflowClient:
def get_mlflow_client(tracking_uri: Optional[str] = None, configure_global: bool = True) -> MlflowClient:
"""
Get a configured MLflow client.
@@ -110,9 +90,7 @@ def get_mlflow_client(
def ensure_experiment(
experiment_name: str,
artifact_location: Optional[str] = None,
tags: Optional[Dict[str, str]] = None
experiment_name: str, artifact_location: Optional[str] = None, tags: Optional[Dict[str, str]] = None
) -> str:
"""
Ensure an experiment exists, creating it if necessary.
@@ -134,11 +112,7 @@ def ensure_experiment(
if experiment is None:
# Create the experiment
artifact_loc = artifact_location or f"{config.artifact_location}/{experiment_name}"
experiment_id = client.create_experiment(
name=experiment_name,
artifact_location=artifact_loc,
tags=tags or {}
)
experiment_id = client.create_experiment(name=experiment_name, artifact_location=artifact_loc, tags=tags or {})
logger.info(f"Created experiment '{experiment_name}' with ID: {experiment_id}")
else:
experiment_id = experiment.experiment_id
@@ -148,9 +122,7 @@ def ensure_experiment(
def get_or_create_registered_model(
model_name: str,
description: Optional[str] = None,
tags: Optional[Dict[str, str]] = None
model_name: str, description: Optional[str] = None, tags: Optional[Dict[str, str]] = None
) -> str:
"""
Get or create a registered model in the Model Registry.
@@ -172,9 +144,7 @@ def get_or_create_registered_model(
except mlflow.exceptions.MlflowException:
# Create the model
client.create_registered_model(
name=model_name,
description=description or f"Model for {model_name}",
tags=tags or {}
name=model_name, description=description or f"Model for {model_name}", tags=tags or {}
)
logger.info(f"Created registered model: {model_name}")

View File

@@ -51,6 +51,7 @@ logger = logging.getLogger(__name__)
@dataclass
class RunComparison:
"""Comparison result for multiple MLflow runs."""
run_ids: List[str]
experiment_name: str
@@ -111,6 +112,7 @@ class RunComparison:
@dataclass
class PromotionRecommendation:
"""Recommendation for model promotion."""
model_name: str
version: Optional[int]
recommended: bool
@@ -262,13 +264,9 @@ class ExperimentAnalyzer:
# Metadata
comparison.run_names[run_id] = run.info.run_name or run_id[:8]
comparison.start_times[run_id] = datetime.fromtimestamp(
run.info.start_time / 1000
)
comparison.start_times[run_id] = datetime.fromtimestamp(run.info.start_time / 1000)
if run.info.end_time:
comparison.durations[run_id] = (
run.info.end_time - run.info.start_time
) / 1000
comparison.durations[run_id] = (run.info.end_time - run.info.start_time) / 1000
# Metrics
for key, value in run.data.metrics.items():
@@ -288,10 +286,7 @@ class ExperimentAnalyzer:
continue
# Determine if lower is better based on metric name
minimize = any(
term in metric_name.lower()
for term in ["latency", "error", "loss", "time"]
)
minimize = any(term in metric_name.lower() for term in ["latency", "error", "loss", "time"])
if minimize:
best_id = min(values.keys(), key=lambda k: values[k])
@@ -330,10 +325,7 @@ class ExperimentAnalyzer:
)
# Filter to only runs that have the metric
runs_with_metric = [
r for r in runs
if metric in r.data.metrics
]
runs_with_metric = [r for r in runs if metric in r.data.metrics]
return runs_with_metric[0] if runs_with_metric else None

View File

@@ -29,6 +29,7 @@ logger = logging.getLogger(__name__)
@dataclass
class InferenceMetrics:
"""Metrics collected during an inference request."""
request_id: str
user_id: Optional[str] = None
session_id: Optional[str] = None
@@ -190,31 +191,22 @@ class InferenceMetricsTracker:
# Initialize MLflow in thread pool to avoid blocking
loop = asyncio.get_event_loop()
await loop.run_in_executor(
self._executor,
self._init_mlflow
)
await loop.run_in_executor(self._executor, self._init_mlflow)
if self.enable_batching:
self._flush_task = asyncio.create_task(self._periodic_flush())
logger.info(
f"InferenceMetricsTracker started for {self.service_name} "
f"(experiment: {self.experiment_name})"
)
logger.info(f"InferenceMetricsTracker started for {self.service_name} (experiment: {self.experiment_name})")
def _init_mlflow(self) -> None:
"""Initialize MLflow client and experiment (runs in thread pool)."""
self._client = get_mlflow_client(
tracking_uri=self.tracking_uri,
configure_global=True
)
self._client = get_mlflow_client(tracking_uri=self.tracking_uri, configure_global=True)
self._experiment_id = ensure_experiment(
self.experiment_name,
tags={
"service": self.service_name,
"type": "inference-metrics",
}
},
)
async def stop(self) -> None:
@@ -265,10 +257,7 @@ class InferenceMetricsTracker:
else:
# Immediate logging in thread pool
loop = asyncio.get_event_loop()
await loop.run_in_executor(
self._executor,
partial(self._log_single_inference, metrics)
)
await loop.run_in_executor(self._executor, partial(self._log_single_inference, metrics))
async def _periodic_flush(self) -> None:
"""Periodically flush batched metrics."""
@@ -287,10 +276,7 @@ class InferenceMetricsTracker:
# Log in thread pool
loop = asyncio.get_event_loop()
await loop.run_in_executor(
self._executor,
partial(self._log_batch, batch)
)
await loop.run_in_executor(self._executor, partial(self._log_batch, batch))
def _log_single_inference(self, metrics: InferenceMetrics) -> None:
"""Log a single inference request to MLflow (runs in thread pool)."""
@@ -302,7 +288,7 @@ class InferenceMetricsTracker:
"service": self.service_name,
"request_id": metrics.request_id,
"type": "single-inference",
}
},
):
mlflow.log_params(metrics.as_params_dict())
mlflow.log_metrics(metrics.as_metrics_dict())
@@ -336,7 +322,7 @@ class InferenceMetricsTracker:
"service": self.service_name,
"type": "batch-inference",
"batch_size": str(len(batch)),
}
},
):
# Log aggregate metrics
mlflow.log_metrics(aggregates)
@@ -352,12 +338,14 @@ class InferenceMetricsTracker:
premium_count = sum(1 for m in batch if m.is_premium)
error_count = sum(1 for m in batch if m.has_error)
mlflow.log_metrics({
mlflow.log_metrics(
{
"rag_enabled_pct": rag_enabled_count / len(batch) * 100,
"streaming_pct": streaming_count / len(batch) * 100,
"premium_pct": premium_count / len(batch) * 100,
"error_rate": error_count / len(batch) * 100,
})
}
)
# Log model distribution
model_counts: Dict[str, int] = defaultdict(int)
@@ -366,20 +354,14 @@ class InferenceMetricsTracker:
model_counts[m.model_name] += 1
if model_counts:
mlflow.log_dict(
{"models": dict(model_counts)},
"model_distribution.json"
)
mlflow.log_dict({"models": dict(model_counts)}, "model_distribution.json")
logger.info(f"Logged batch of {len(batch)} inference metrics")
except Exception as e:
logger.error(f"Failed to log batch metrics: {e}")
def _calculate_aggregates(
self,
batch: List[InferenceMetrics]
) -> Dict[str, float]:
def _calculate_aggregates(self, batch: List[InferenceMetrics]) -> Dict[str, float]:
"""Calculate aggregate statistics from a batch of metrics."""
import statistics

View File

@@ -47,17 +47,14 @@ MLFLOW_PACKAGES = [
]
@dsl.component(
base_image=MLFLOW_IMAGE,
packages_to_install=MLFLOW_PACKAGES
)
@dsl.component(base_image=MLFLOW_IMAGE, packages_to_install=MLFLOW_PACKAGES)
def create_mlflow_run(
experiment_name: str,
run_name: str,
mlflow_tracking_uri: str = "http://mlflow.mlflow.svc.cluster.local:80",
tags: Dict[str, str] = None,
params: Dict[str, str] = None,
) -> NamedTuple('RunInfo', [('run_id', str), ('experiment_id', str), ('artifact_uri', str)]):
) -> NamedTuple("RunInfo", [("run_id", str), ("experiment_id", str), ("artifact_uri", str)]):
"""
Create a new MLflow run for the pipeline.
@@ -90,8 +87,7 @@ def create_mlflow_run(
experiment = client.get_experiment_by_name(experiment_name)
if experiment is None:
experiment_id = client.create_experiment(
name=experiment_name,
artifact_location=f"/mlflow/artifacts/{experiment_name}"
name=experiment_name, artifact_location=f"/mlflow/artifacts/{experiment_name}"
)
else:
experiment_id = experiment.experiment_id
@@ -122,14 +118,11 @@ def create_mlflow_run(
# End run (KFP components are isolated, we'll resume in other components)
mlflow.end_run()
RunInfo = namedtuple('RunInfo', ['run_id', 'experiment_id', 'artifact_uri'])
RunInfo = namedtuple("RunInfo", ["run_id", "experiment_id", "artifact_uri"])
return RunInfo(run_id, experiment_id, artifact_uri)
@dsl.component(
base_image=MLFLOW_IMAGE,
packages_to_install=MLFLOW_PACKAGES
)
@dsl.component(base_image=MLFLOW_IMAGE, packages_to_install=MLFLOW_PACKAGES)
def log_params_component(
run_id: str,
params: Dict[str, str],
@@ -158,10 +151,7 @@ def log_params_component(
return run_id
@dsl.component(
base_image=MLFLOW_IMAGE,
packages_to_install=MLFLOW_PACKAGES
)
@dsl.component(base_image=MLFLOW_IMAGE, packages_to_install=MLFLOW_PACKAGES)
def log_metrics_component(
run_id: str,
metrics: Dict[str, float],
@@ -192,10 +182,7 @@ def log_metrics_component(
return run_id
@dsl.component(
base_image=MLFLOW_IMAGE,
packages_to_install=MLFLOW_PACKAGES
)
@dsl.component(base_image=MLFLOW_IMAGE, packages_to_install=MLFLOW_PACKAGES)
def log_artifact_component(
run_id: str,
artifact_path: str,
@@ -225,10 +212,7 @@ def log_artifact_component(
return run_id
@dsl.component(
base_image=MLFLOW_IMAGE,
packages_to_install=MLFLOW_PACKAGES
)
@dsl.component(base_image=MLFLOW_IMAGE, packages_to_install=MLFLOW_PACKAGES)
def log_dict_artifact(
run_id: str,
data: Dict[str, Any],
@@ -258,23 +242,20 @@ def log_dict_artifact(
client = MlflowClient()
# Ensure .json extension
if not filename.endswith('.json'):
filename += '.json'
if not filename.endswith(".json"):
filename += ".json"
# Write to temp file and log
with tempfile.TemporaryDirectory() as tmpdir:
filepath = Path(tmpdir) / filename
with open(filepath, 'w') as f:
with open(filepath, "w") as f:
json.dump(data, f, indent=2)
client.log_artifact(run_id, str(filepath))
return run_id
@dsl.component(
base_image=MLFLOW_IMAGE,
packages_to_install=MLFLOW_PACKAGES
)
@dsl.component(base_image=MLFLOW_IMAGE, packages_to_install=MLFLOW_PACKAGES)
def end_mlflow_run(
run_id: str,
status: str = "FINISHED",
@@ -310,10 +291,7 @@ def end_mlflow_run(
return run_id
@dsl.component(
base_image=MLFLOW_IMAGE,
packages_to_install=MLFLOW_PACKAGES + ["httpx"]
)
@dsl.component(base_image=MLFLOW_IMAGE, packages_to_install=MLFLOW_PACKAGES + ["httpx"])
def log_training_metrics(
run_id: str,
model_type: str,
@@ -370,7 +348,7 @@ def log_training_metrics(
# Log full config as artifact
with tempfile.TemporaryDirectory() as tmpdir:
config_path = Path(tmpdir) / "training_config.json"
with open(config_path, 'w') as f:
with open(config_path, "w") as f:
json.dump(training_config, f, indent=2)
client.log_artifact(run_id, str(config_path))
@@ -382,10 +360,7 @@ def log_training_metrics(
return run_id
@dsl.component(
base_image=MLFLOW_IMAGE,
packages_to_install=MLFLOW_PACKAGES
)
@dsl.component(base_image=MLFLOW_IMAGE, packages_to_install=MLFLOW_PACKAGES)
def log_document_ingestion_metrics(
run_id: str,
source_url: str,
@@ -452,10 +427,7 @@ def log_document_ingestion_metrics(
return run_id
@dsl.component(
base_image=MLFLOW_IMAGE,
packages_to_install=MLFLOW_PACKAGES
)
@dsl.component(base_image=MLFLOW_IMAGE, packages_to_install=MLFLOW_PACKAGES)
def log_evaluation_results(
run_id: str,
model_name: str,
@@ -502,7 +474,7 @@ def log_evaluation_results(
if sample_results:
with tempfile.TemporaryDirectory() as tmpdir:
results_path = Path(tmpdir) / "evaluation_results.json"
with open(results_path, 'w') as f:
with open(results_path, "w") as f:
json.dump(sample_results, f, indent=2)
client.log_artifact(run_id, str(results_path))

View File

@@ -192,7 +192,7 @@ def register_model_for_kserve(
tags={
"model.type": model_type,
"deployment.target": "kserve",
}
},
)
logger.info(f"Created registered model: {model_name}")
@@ -206,13 +206,10 @@ def register_model_for_kserve(
**(tags or {}),
"model.type": model_type,
**kserve_config.as_dict(),
}
},
)
logger.info(
f"Registered model version {model_version.version} "
f"for {model_name} (type: {model_type})"
)
logger.info(f"Registered model version {model_version.version} for {model_name} (type: {model_type})")
return model_version
@@ -412,7 +409,8 @@ def generate_kserve_manifest(
if config.container_image:
# Custom container
predictor["containers"] = [{
predictor["containers"] = [
{
"name": "predictor",
"image": config.container_image,
"ports": [{"containerPort": config.container_port, "protocol": "TCP"}],
@@ -426,11 +424,9 @@ def generate_kserve_manifest(
"memory": config.memory_limit,
},
},
"env": [
{"name": k, "value": v}
for k, v in config.env_vars.items()
],
}]
"env": [{"name": k, "value": v} for k, v in config.env_vars.items()],
}
]
# Add GPU if needed
if config.gpu_count > 0:
@@ -494,7 +490,7 @@ def generate_kserve_yaml(
yaml_str = yaml.dump(manifest, default_flow_style=False, sort_keys=False)
if output_path:
with open(output_path, 'w') as f:
with open(output_path, "w") as f:
f.write(yaml_str)
logger.info(f"Wrote KServe manifest to {output_path}")

View File

@@ -23,6 +23,7 @@ logger = logging.getLogger(__name__)
@dataclass
class PipelineMetadata:
"""Metadata about the Kubeflow Pipeline run."""
pipeline_name: str
run_id: str
run_name: Optional[str] = None
@@ -30,12 +31,8 @@ class PipelineMetadata:
namespace: str = "ai-ml"
# KFP-specific metadata (populated from environment if available)
kfp_run_id: Optional[str] = field(
default_factory=lambda: os.environ.get("KFP_RUN_ID")
)
kfp_pod_name: Optional[str] = field(
default_factory=lambda: os.environ.get("KFP_POD_NAME")
)
kfp_run_id: Optional[str] = field(default_factory=lambda: os.environ.get("KFP_RUN_ID"))
kfp_pod_name: Optional[str] = field(default_factory=lambda: os.environ.get("KFP_POD_NAME"))
def as_tags(self) -> Dict[str, str]:
"""Convert metadata to MLflow tags."""
@@ -142,10 +139,7 @@ class MLflowTracker:
Yields:
The MLflow run object
"""
self.client = get_mlflow_client(
tracking_uri=self.tracking_uri,
configure_global=True
)
self.client = get_mlflow_client(tracking_uri=self.tracking_uri, configure_global=True)
# Ensure experiment exists
experiment_id = ensure_experiment(self.experiment_name)
@@ -163,8 +157,7 @@ class MLflowTracker:
self.run_id = self.run.info.run_id
logger.info(
f"Started MLflow run '{self.run_name}' "
f"(ID: {self.run_id}) in experiment '{self.experiment_name}'"
f"Started MLflow run '{self.run_name}' (ID: {self.run_id}) in experiment '{self.experiment_name}'"
)
yield self.run
@@ -214,11 +207,7 @@ class MLflowTracker:
"""Log a single parameter."""
self.log_params({key: value})
def log_metrics(
self,
metrics: Dict[str, Union[float, int]],
step: Optional[int] = None
) -> None:
def log_metrics(self, metrics: Dict[str, Union[float, int]], step: Optional[int] = None) -> None:
"""
Log metrics to the current run.
@@ -233,20 +222,11 @@ class MLflowTracker:
mlflow.log_metrics(metrics, step=step)
logger.debug(f"Logged {len(metrics)} metrics")
def log_metric(
self,
key: str,
value: Union[float, int],
step: Optional[int] = None
) -> None:
def log_metric(self, key: str, value: Union[float, int], step: Optional[int] = None) -> None:
"""Log a single metric."""
self.log_metrics({key: value}, step=step)
def log_artifact(
self,
local_path: str,
artifact_path: Optional[str] = None
) -> None:
def log_artifact(self, local_path: str, artifact_path: Optional[str] = None) -> None:
"""
Log an artifact file to the current run.
@@ -261,11 +241,7 @@ class MLflowTracker:
mlflow.log_artifact(local_path, artifact_path)
logger.info(f"Logged artifact: {local_path}")
def log_artifacts(
self,
local_dir: str,
artifact_path: Optional[str] = None
) -> None:
def log_artifacts(self, local_dir: str, artifact_path: Optional[str] = None) -> None:
"""
Log all files in a directory as artifacts.
@@ -280,12 +256,7 @@ class MLflowTracker:
mlflow.log_artifacts(local_dir, artifact_path)
logger.info(f"Logged artifacts from: {local_dir}")
def log_dict(
self,
data: Dict[str, Any],
filename: str,
artifact_path: Optional[str] = None
) -> None:
def log_dict(self, data: Dict[str, Any], filename: str, artifact_path: Optional[str] = None) -> None:
"""
Log a dictionary as a JSON artifact.
@@ -311,7 +282,7 @@ class MLflowTracker:
model_name: str,
model_path: Optional[str] = None,
framework: str = "pytorch",
extra_info: Optional[Dict[str, Any]] = None
extra_info: Optional[Dict[str, Any]] = None,
) -> None:
"""
Log model information as parameters and tags.
@@ -341,11 +312,7 @@ class MLflowTracker:
mlflow.set_tag("model.name", model_name)
def log_dataset_info(
self,
name: str,
source: str,
size: Optional[int] = None,
extra_info: Optional[Dict[str, Any]] = None
self, name: str, source: str, size: Optional[int] = None, extra_info: Optional[Dict[str, Any]] = None
) -> None:
"""
Log dataset information.