Pre-stage the LOTA cache

import boto3
from botocore.client import Config

s3 = boto3.client(
    "s3",
    endpoint_url="http://cwlota.com",
    config=Config(
        s3={"addressing_style": "virtual"},
    ),
)

response = s3.head_object(
    Bucket="[BUCKET-NAME]",
    Key="[OBJECT-KEY]",
    Range="bytes=0-0",
)

print(f"Object size: {response['ContentLength']:,} bytes")
print(f"ETag: {response['ETag']}")

s3cmd info s3://[BUCKET-NAME]/[OBJECT-KEY] \
  --host=cwlota.com \
  --host-bucket='%(bucket)s.cwlota.com' \
  --no-ssl \
  --add-header='Range: bytes=0-0'

import boto3
import threading
import time
from botocore.client import Config
from concurrent.futures import ThreadPoolExecutor, as_completed

BUCKET = "[BUCKET-NAME]"
PREFIX = "[PREFIX]"
MAX_WORKERS = 16
REQUESTS_PER_SECOND = 100

s3 = boto3.client(
    "s3",
    endpoint_url="http://cwlota.com",
    config=Config(
        max_pool_connections=MAX_WORKERS,
        retries={"max_attempts": 5, "mode": "standard"},
        s3={"addressing_style": "virtual"},
    ),
)

_rate_lock = threading.Lock()
_next_slot = time.monotonic()

def throttle():
    """Hold the pre-stage pass at REQUESTS_PER_SECOND across all threads."""
    global _next_slot
    interval = 1.0 / REQUESTS_PER_SECOND
    with _rate_lock:
        now = time.monotonic()
        wait = max(0.0, _next_slot - now)
        _next_slot = max(now, _next_slot) + interval
    if wait:
        time.sleep(wait)

def prestage_object(key):
    """Issue a HeadObject call (range 0-0) to pre-stage a single object."""
    throttle()
    s3.head_object(
        Bucket=BUCKET,
        Key=key,
        Range="bytes=0-0",
    )
    return key

paginator = s3.get_paginator("list_objects_v2")
keys = []
for page in paginator.paginate(Bucket=BUCKET, Prefix=PREFIX):
    for obj in page.get("Contents", []):
        keys.append(obj["Key"])

print(f"Pre-staging {len(keys)} objects...")

succeeded = 0
failed = 0

with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
    futures = {executor.submit(prestage_object, k): k for k in keys}
    for future in as_completed(futures):
        key = futures[future]
        try:
            future.result()
            succeeded += 1
        except Exception as exc:
            failed += 1
            print(f"  Failed to pre-stage {key}: {exc}")
        if (succeeded + failed) % 100 == 0:
            print(f"  {succeeded + failed}/{len(keys)} objects processed")

print(f"Done. {succeeded} succeeded, {failed} failed out of {len(keys)} objects.")

s3cmd ls s3://[BUCKET-NAME]/[PREFIX] \
  --host=cwlota.com \
  --host-bucket='%(bucket)s.cwlota.com' \
  --no-ssl | \
grep -v ' DIR ' | grep -o 's3://.*' | tr '\n' '\0' | \
xargs -0 -P [CONCURRENCY] -I {} \
  s3cmd info {} \
    --host=cwlota.com \
    --host-bucket='%(bucket)s.cwlota.com' \
    --no-ssl \
    --add-header='Range: bytes=0-0'

echo "Pre-staging complete."