pipelines go to gravenhollow now.
All checks were successful
Compile and Upload Pipelines / Compile & Upload (push) Successful in 15s
Compile and Upload Pipelines / Notify (push) Successful in 1s

This commit is contained in:
2026-02-18 07:14:12 -05:00
parent 7f2b011c95
commit d4eb54d92b
3 changed files with 14 additions and 14 deletions

View File

@@ -7,7 +7,7 @@ distributed across multiple cluster nodes via KubeRay RayJob.
GPUs remain 100 % dedicated to inference serving.
Architecture:
1. Fetch PDFs from Quobjects S3
1. Fetch PDFs from RustFS S3
2. Prepare instruction-tuning dataset
3. Upload prepared data to S3 (shared storage for Ray workers)
4. Submit a KubeRay RayJob that runs Ray Train TorchTrainer
@@ -41,7 +41,7 @@ from typing import NamedTuple
# ──────────────────────────────────────────────────────────────
# 1. Fetch PDFs from Quobjects S3
# 1. Fetch PDFs from RustFS S3
# ──────────────────────────────────────────────────────────────
@dsl.component(
base_image="python:3.13-slim",
@@ -54,7 +54,7 @@ def fetch_pdfs_from_s3(
aws_access_key_id: str,
aws_secret_access_key: str,
) -> NamedTuple("PDFOutput", [("pdf_dir", str), ("num_files", int)]):
"""Download all PDFs from a Quobjects S3 bucket."""
"""Download all PDFs from an S3 bucket."""
import os
import boto3
@@ -994,7 +994,7 @@ def log_training_metrics(
"num_epochs": num_epochs,
"num_pdfs": num_pdfs,
"backend": "ray-train-gloo",
"data_source": "quobjects/training-data",
"data_source": "rustfs/training-data",
}
)
mlflow.log_metrics(
@@ -1023,7 +1023,7 @@ def log_training_metrics(
),
)
def cpu_training_pipeline(
# ── S3 / Quobjects ──
# ── S3 / RustFS ──
s3_endpoint: str = "https://gravenhollow.lab.daviestechlabs.io:30292",
s3_bucket: str = "training-data",
s3_prefix: str = "",