Pre-stage the LOTA cache

import boto3
from botocore.client import Config

s3 = boto3.client(
    "s3",
    endpoint_url="http://cwlota.com",
    config=Config(
        s3={"addressing_style": "virtual"},
    ),
)

response = s3.head_object(
    Bucket="[BUCKET-NAME]",
    Key="[OBJECT-KEY]",
    Range="bytes=0-0",
)

print(f"Object size: {response['ContentLength']:,} bytes")
print(f"ETag: {response['ETag']}")

s3cmd info s3://[BUCKET-NAME]/[OBJECT-KEY] \
  --host=cwlota.com \
  --host-bucket='%(bucket)s.cwlota.com' \
  --no-ssl \
  --add-header='Range: bytes=0-0'

import boto3
from botocore.client import Config
from concurrent.futures import ThreadPoolExecutor, as_completed

BUCKET = "[BUCKET-NAME]"
PREFIX = "[PREFIX]"
MAX_WORKERS = 32

s3 = boto3.client(
    "s3",
    endpoint_url="http://cwlota.com",
    config=Config(
        max_pool_connections=MAX_WORKERS,
        s3={"addressing_style": "virtual"},
    ),
)

def prestage_object(key):
    """Issue a HeadObject call (range 0-0) to pre-stage a single object."""
    s3.head_object(
        Bucket=BUCKET,
        Key=key,
        Range="bytes=0-0",
    )
    return key

paginator = s3.get_paginator("list_objects_v2")
keys = []
for page in paginator.paginate(Bucket=BUCKET, Prefix=PREFIX):
    for obj in page.get("Contents", []):
        keys.append(obj["Key"])

print(f"Pre-staging {len(keys)} objects...")

succeeded = 0
failed = 0

with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
    futures = {executor.submit(prestage_object, k): k for k in keys}
    for future in as_completed(futures):
        key = futures[future]
        try:
            future.result()
            succeeded += 1
        except Exception as exc:
            failed += 1
            print(f"  Failed to pre-stage {key}: {exc}")
        if (succeeded + failed) % 100 == 0:
            print(f"  {succeeded + failed}/{len(keys)} objects processed")

print(f"Done. {succeeded} succeeded, {failed} failed out of {len(keys)} objects.")

s3cmd ls s3://[BUCKET-NAME]/[PREFIX] \
  --host=cwlota.com \
  --host-bucket='%(bucket)s.cwlota.com' \
  --no-ssl | \
awk '{print $4}' | \
while read uri; do
  s3cmd info "$uri" \
    --host=cwlota.com \
    --host-bucket='%(bucket)s.cwlota.com' \
    --no-ssl \
    --add-header='Range: bytes=0-0' &
done

wait
echo "Pre-staging complete."