Price the same basket in 20 countries

import { writeFile } from 'node:fs/promises';
import {
  ConcurrencyLimitError,
  NoCapacityError,
  Outcrawl,
  QuotaExceededError,
} from '@outcrawl/sdk';

const oc = new Outcrawl();

// Every key on your account shares one browser slot count (Free 2, Builder 5,
// Pro 25, Scale 100). Keep LANES below yours so other work still gets a slot.
const LANES = 4;
const RUN_BUDGET = '100.00'; // per country: about 6.5 minutes of a typical run
const BATCH_CEILING = 1_500; // credits, across the whole batch

const BASKET = [
  { id: 'netflix-standard', ask: 'Netflix Standard plan, billed monthly, on netflix.com' },
  { id: 'spotify-premium', ask: 'Spotify Premium Individual, billed monthly, on spotify.com' },
  { id: 'iphone-16-128gb', ask: 'iPhone 16 with 128 GB, bought outright, on apple.com' },
  { id: 'm365-personal', ask: 'Microsoft 365 Personal, billed yearly, on microsoft.com' },
];

const COUNTRIES: [code: string, name: string][] = [
  ['us', 'United States'], ['ca', 'Canada'], ['mx', 'Mexico'], ['br', 'Brazil'],
  ['gb', 'United Kingdom'], ['de', 'Germany'], ['fr', 'France'], ['es', 'Spain'],
  ['se', 'Sweden'], ['no', 'Norway'], ['pl', 'Poland'], ['ch', 'Switzerland'],
  ['tr', 'Türkiye'], ['za', 'South Africa'], ['in', 'India'], ['jp', 'Japan'],
  ['kr', 'South Korea'], ['sg', 'Singapore'], ['au', 'Australia'], ['nz', 'New Zealand'],
];

const SCHEMA = {
  type: 'object',
  properties: {
    items: {
      type: 'array',
      items: {
        type: 'object',
        properties: {
          item: { type: 'string', enum: BASKET.map((b) => b.id) },
          available: { type: 'boolean' },
          price: { type: 'number', description: 'The number shown, tax as the page shows it' },
          currency: { type: 'string', description: 'ISO 4217 code, e.g. JPY' },
          period: { type: 'string', enum: ['month', 'year', 'one-off'] },
          url: { type: 'string' },
        },
        required: ['item', 'available'],
      },
    },
  },
  required: ['items'],
};

interface Price {
  item: string;
  available: boolean;
  price?: number;
  currency?: string;
  period?: string;
  url?: string;
}
interface Row {
  country: string;
  name: string;
  prices: Price[];
  credits: number;
  problem?: string;
}

const sleep = (ms: number) => new Promise((wake) => setTimeout(wake, ms));

// A refused call is retried; anything else is the caller's to see.
async function withSlot<T>(call: () => Promise<T>): Promise<T> {
  for (let attempt = 1; ; attempt++) {
    try {
      return await call();
    } catch (error) {
      if (attempt === 8) throw error;
      if (error instanceof ConcurrencyLimitError || error instanceof NoCapacityError) {
        // Refused, not queued, and no Retry-After: a slot frees when one of your runs ends.
        await sleep(20_000);
      } else if (error instanceof QuotaExceededError && error.ceiling === 'requests_per_minute') {
        await sleep(error.retryAfterSeconds * 1_000);
      } else {
        throw error; // out of monthly credits, or a bad request: retrying cannot help
      }
    }
  }
}

// At most LANES items in flight: each lane takes the next item when it is free.
async function inLanes<T, R>(items: readonly T[], work: (item: T) => Promise<R>): Promise<R[]> {
  const results: R[] = [];
  let next = 0;
  const lane = async () => {
    while (next < items.length) {
      const index = next++;
      results[index] = await work(items[index] as T);
    }
  };
  await Promise.all(Array.from({ length: Math.min(LANES, items.length) }, lane));
  return results;
}

// Reserve a run's cap before it starts; settle to what it really cost.
let spent = 0;
let reserved = 0;

// 1. One profile per country, created once and reused on every run.
const profiles = new Map((await oc.profiles.list({ label: 'basket-*', limit: 100 })).map((p) => [p.label, p.id]));
for (const [code] of COUNTRIES) {
  const label = `basket-${code}`;
  if (!profiles.has(label)) profiles.set(label, (await oc.profiles.create({ label, exit: { country: code } })).id);
}

// 2. One agent per country.
async function priceCountry([code, name]: [string, string]): Promise<Row> {
  const cap = Number(RUN_BUDGET);
  if (spent + reserved + cap > BATCH_CEILING) {
    return { country: code, name, prices: [], credits: 0, problem: 'skipped: batch ceiling reached' };
  }
  reserved += cap;
  let credits = 0;
  try {
    // Only the submit is retried: a refusal arrives before any run exists.
    const { job } = await withSlot(async () => {
      const job = oc.agent({
        task:
          `You are a new customer in ${name}. For each item below, open the official site as it ` +
          `is served in ${name} and find today's price for this country, in the local currency, ` +
          `as the page shows it. If an item is not sold in ${name}, set available to false.\n` +
          BASKET.map((b) => `- ${b.id}: ${b.ask}`).join('\n'),
        caps: { budget: RUN_BUDGET, duration: '8m' },
        schema: SCHEMA,
        profile: profiles.get(`basket-${code}`),
        rules: ['Never sign in, create an account, or start a checkout.'],
      });
      await job.submitted; // a refused submit throws here
      return { job }; // wrapped, because a job is awaitable and returning it bare waits for the run
    });
    const run = await job; // the lane holds its slot until the run ends
    credits = Number(run.credits);
    const data = run.data as { items?: Price[] } | undefined;
    return {
      country: code,
      name,
      prices: data?.items ?? [],
      credits,
      problem: run.status === 'completed' ? undefined : `${run.status} (${run.stopReason ?? run.error ?? 'no reason'})`,
    };
  } catch (error) {
    return { country: code, name, prices: [], credits, problem: String(error) };
  } finally {
    reserved -= cap;
    spent += credits;
  }
}

const rows = await inLanes(COUNTRIES, priceCountry);

// 3. One table, in US dollars at today's ECB reference rate.
const fx = (await (await fetch('https://api.frankfurter.dev/v1/latest?base=USD')).json()) as {
  date: string;
  rates: Record<string, number>;
};
const toUsd = (p: Price): number | undefined => {
  if (!p.available || p.price === undefined || p.currency === undefined) return undefined;
  const rate = p.currency === 'USD' ? 1 : fx.rates[p.currency];
  return rate === undefined ? undefined : p.price / rate;
};

console.log(`| Country | ${BASKET.map((b) => b.id).join(' | ')} |`);
console.log(`| --- |${BASKET.map(() => ' --- |').join('')}`);
for (const row of rows) {
  const cells = BASKET.map((b) => {
    const p = row.prices.find((price) => price.item === b.id);
    const usd = p === undefined ? undefined : toUsd(p);
    if (p === undefined || !p.available) return 'not sold';
    return usd === undefined ? `${p.price} ${p.currency}` : `$${usd.toFixed(2)} (${p.price} ${p.currency})`;
  });
  console.log(`| ${row.name} | ${row.problem ?? cells.join(' | ')} |`);
}
console.log(`\n${rows.length} countries, ${spent.toFixed(2)} credits, rates of ${fx.date}`);

await writeFile('basket-prices.json', JSON.stringify({ rates: fx.date, rows }, null, 2));

import asyncio
import json

import httpx
from outcrawl import ConcurrencyLimitError, NoCapacityError, Outcrawl, QuotaExceededError

# Every key on your account shares one browser slot count (Free 2, Builder 5,
# Pro 25, Scale 100). Keep LANES below yours so other work still gets a slot.
LANES = 4
RUN_BUDGET = "100.00"  # per country: about 6.5 minutes of a typical run
BATCH_CEILING = 1_500  # credits, across the whole batch

BASKET = [
    ("netflix-standard", "Netflix Standard plan, billed monthly, on netflix.com"),
    ("spotify-premium", "Spotify Premium Individual, billed monthly, on spotify.com"),
    ("iphone-16-128gb", "iPhone 16 with 128 GB, bought outright, on apple.com"),
    ("m365-personal", "Microsoft 365 Personal, billed yearly, on microsoft.com"),
]

COUNTRIES = [
    ("us", "United States"), ("ca", "Canada"), ("mx", "Mexico"), ("br", "Brazil"),
    ("gb", "United Kingdom"), ("de", "Germany"), ("fr", "France"), ("es", "Spain"),
    ("se", "Sweden"), ("no", "Norway"), ("pl", "Poland"), ("ch", "Switzerland"),
    ("tr", "Türkiye"), ("za", "South Africa"), ("in", "India"), ("jp", "Japan"),
    ("kr", "South Korea"), ("sg", "Singapore"), ("au", "Australia"), ("nz", "New Zealand"),
]

SCHEMA = {
    "type": "object",
    "properties": {
        "items": {
            "type": "array",
            "items": {
                "type": "object",
                "properties": {
                    "item": {"type": "string", "enum": [item for item, _ in BASKET]},
                    "available": {"type": "boolean"},
                    "price": {"type": "number", "description": "The number shown, tax as the page shows it"},
                    "currency": {"type": "string", "description": "ISO 4217 code, e.g. JPY"},
                    "period": {"type": "string", "enum": ["month", "year", "one-off"]},
                    "url": {"type": "string"},
                },
                "required": ["item", "available"],
            },
        }
    },
    "required": ["items"],
}


async def with_slot(call):
    """A refused call is retried; anything else is the caller's to see."""
    for attempt in range(1, 9):
        try:
            return await call()
        except (ConcurrencyLimitError, NoCapacityError):
            # Refused, not queued, and no Retry-After: a slot frees when one of your runs ends.
            if attempt == 8:
                raise
            await asyncio.sleep(20)
        except QuotaExceededError as error:
            # Out of monthly credits: retrying cannot help. Per-minute: wait as told.
            if error.ceiling != "requests_per_minute" or attempt == 8:
                raise
            await asyncio.sleep(error.retry_after_seconds)


async def main() -> None:
    async with Outcrawl() as oc:
        # 1. One profile per country, created once and reused on every run.
        profiles = {p["label"]: p["id"] for p in await oc.profiles.list(label="basket-*", limit=100)}
        for code, _ in COUNTRIES:
            label = f"basket-{code}"
            if label not in profiles:
                profiles[label] = (await oc.profiles.create(label=label, exit={"country": code}))["id"]

        lanes = asyncio.Semaphore(LANES)
        tally = {"spent": 0.0, "reserved": 0.0}  # reserve a run's cap; settle to its real cost

        # 2. One agent per country.
        async def price_country(code: str, name: str) -> dict:
            async with lanes:
                cap = float(RUN_BUDGET)
                if tally["spent"] + tally["reserved"] + cap > BATCH_CEILING:
                    return {"country": code, "name": name, "prices": [], "credits": 0,
                            "problem": "skipped: batch ceiling reached"}
                tally["reserved"] += cap
                credits = 0.0
                try:
                    async def submit():
                        job = oc.agent(
                            task=(
                                f"You are a new customer in {name}. For each item below, open the official "
                                f"site as it is served in {name} and find today's price for this country, "
                                f"in the local currency, as the page shows it. If an item is not sold in "
                                f"{name}, set available to false.\n"
                                + "\n".join(f"- {item}: {ask}" for item, ask in BASKET)
                            ),
                            caps={"budget": RUN_BUDGET, "duration": "8m"},
                            schema=SCHEMA,
                            profile=profiles[f"basket-{code}"],
                            rules=["Never sign in, create an account, or start a checkout."],
                        )
                        await job.submitted  # a refused submit raises here
                        return job

                    # Only the submit is retried: a refusal arrives before any run exists.
                    job = await with_slot(submit)
                    run = await job  # the lane holds its slot until the run ends
                    credits = float(run["credits"])
                    data = run.get("data") or {}
                    problem = None
                    if run["status"] != "completed":
                        problem = f"{run['status']} ({run.get('stopReason') or run.get('error') or 'no reason'})"
                    return {"country": code, "name": name, "prices": data.get("items", []),
                            "credits": credits, "problem": problem}
                except Exception as error:
                    return {"country": code, "name": name, "prices": [], "credits": credits, "problem": str(error)}
                finally:
                    tally["reserved"] -= cap
                    tally["spent"] += credits

        rows = await asyncio.gather(*(price_country(code, name) for code, name in COUNTRIES))

    # 3. One table, in US dollars at today's ECB reference rate.
    async with httpx.AsyncClient() as http:
        fx = (await http.get("https://api.frankfurter.dev/v1/latest", params={"base": "USD"})).json()

    def cell(price: dict | None) -> str:
        if price is None or not price.get("available"):
            return "not sold"
        amount, currency = price.get("price"), price.get("currency")
        rate = 1 if currency == "USD" else fx["rates"].get(currency)
        if amount is None or rate is None:
            return f"{amount} {currency}"
        return f"${amount / rate:.2f} ({amount} {currency})"

    print("| Country | " + " | ".join(item for item, _ in BASKET) + " |")
    print("| --- |" + " --- |" * len(BASKET))
    for row in rows:
        by_item = {p["item"]: p for p in row["prices"]}
        cells = " | ".join(cell(by_item.get(item)) for item, _ in BASKET)
        print(f"| {row['name']} | {row['problem'] or cells} |")
    print(f"\n{len(rows)} countries, {tally['spent']:.2f} credits, rates of {fx['date']}")

    with open("basket-prices.json", "w") as out:
        json.dump({"rates": fx["date"], "rows": rows}, out, indent=2, ensure_ascii=False)


asyncio.run(main())