Upload a Dataset

# Requires Python 3.9 or later and the requests package.
import os
import time

import requests

API = f"https://forge.coreweave.com/api/distillation/v1/projects/ticket-classifier/datasets/{os.environ['DATASET_ID']}"
HEADERS = {
    "Authorization": f"Bearer {os.environ['WANDB_API_KEY']}",
    "Wandb-Entity": "your-team",
}
FILE = "tickets.jsonl"

# 1. Open the upload.
response = requests.post(
    f"{API}/uploads",
    headers=HEADERS,
    json={
        "idempotency_key": "support-tickets-v1",
        "file": {"name": FILE, "size_bytes": os.path.getsize(FILE)},
        "mode": "append",
    },
)
response.raise_for_status()
upload = response.json()

# 2. Send the file. The signed URL carries its own authorization.
with open(FILE, "rb") as handle:
    requests.put(upload["upload"]["url"], data=handle).raise_for_status()

# 3. Complete the upload. This queues validation and the commit.
requests.post(f"{API}/uploads/{upload['id']}/complete", headers=HEADERS).raise_for_status()

# 4. Poll until the upload reaches a final state.
while True:
    status = requests.get(f"{API}/uploads/{upload['id']}", headers=HEADERS).json()
    if status["state"] in ("committed", "failed", "cancelled", "expired"):
        break
    time.sleep(10)

print(status["state"], status["counters"], status["error"])

// Requires Node.js 18 or later. No packages to install.
import { readFile } from "node:fs/promises";

const API = `https://forge.coreweave.com/api/distillation/v1/projects/ticket-classifier/datasets/${process.env.DATASET_ID}`;
const HEADERS = {
  Authorization: `Bearer ${process.env.WANDB_API_KEY}`,
  "Wandb-Entity": "your-team",
};
const FILE = "tickets.jsonl";

async function call(path, init = {}) {
  const response = await fetch(`${API}${path}`, { ...init, headers: { ...HEADERS, ...init.headers } });
  if (!response.ok) throw new Error(`${init.method ?? "GET"} ${path} failed: ${await response.text()}`);
  return response.json();
}

// 1. Open the upload.
const body = await readFile(FILE);
const upload = await call("/uploads", {
  method: "POST",
  headers: { "Content-Type": "application/json" },
  body: JSON.stringify({
    idempotency_key: "support-tickets-v1",
    file: { name: FILE, size_bytes: body.byteLength },
    mode: "append",
  }),
});

// 2. Send the file. The signed URL carries its own authorization.
const put = await fetch(upload.upload.url, { method: "PUT", body });
if (!put.ok) throw new Error(`Upload failed with ${put.status}`);

// 3. Complete the upload. This queues validation and the commit.
await call(`/uploads/${upload.id}/complete`, { method: "POST" });

// 4. Poll until the upload reaches a final state.
let status;
while (true) {
  status = await call(`/uploads/${upload.id}`);
  if (["committed", "failed", "cancelled", "expired"].includes(status.state)) break;
  await new Promise((resolve) => setTimeout(resolve, 10_000));
}
console.log(status.state, status.counters, status.error);