Turn a website into an API

import { serve } from '@hono/node-server';
import { Hono } from 'hono';
import { Outcrawl } from '@outcrawl/sdk';

const oc = new Outcrawl();
const PORTAL = 'https://dealers.harbor-supply.example';
const PROFILE_ID = process.env.PORTAL_PROFILE_ID!; // the profile that is signed in to the portal
const FRESH_FOR_MS = 10 * 60_000; // how long an answer is served from cache

interface Order {
  found: boolean;
  po: string;
  status?: string;
  orderedOn?: string;
  shipBy?: string;
  carrier?: string;
  tracking?: string;
  lines?: { sku: string; description: string; quantity: number; unitPrice: string }[];
}

const orderSchema = {
  type: 'object',
  properties: {
    found: { type: 'boolean', description: 'false when the portal has no order with this number' },
    po: { type: 'string' },
    status: { type: 'string', description: 'the status exactly as the portal shows it' },
    orderedOn: { type: 'string', description: 'YYYY-MM-DD' },
    shipBy: { type: 'string', description: 'YYYY-MM-DD' },
    carrier: { type: 'string' },
    tracking: { type: 'string' },
    lines: {
      type: 'array',
      items: {
        type: 'object',
        properties: {
          sku: { type: 'string' },
          description: { type: 'string' },
          quantity: { type: 'integer' },
          unitPrice: { type: 'string', description: 'with currency, e.g. $14.20' },
        },
        required: ['sku', 'quantity'],
      },
    },
  },
  required: ['found', 'po'],
};

const cache = new Map<string, { at: number; order: Order }>();
const inFlight = new Map<string, Promise<Order>>();
// A profile holds one session at a time, so runs on it go one after another.
let lane: Promise<unknown> = Promise.resolve();

function lookUp(po: string): Promise<Order> {
  const next = lane.then(async () => {
    const run = await oc.agent({
      task:
        `On ${PORTAL}, open Orders, search for purchase order ${po} and open it. ` +
        'Report its status, order date, ship-by date, carrier, tracking number and every line item. ' +
        'If no order has that number, set found to false.',
      profile: PROFILE_ID,
      schema: orderSchema,
      caps: { budget: '30.00' },
      rules: ['Never place, change, cancel or reorder an order.'],
    });
    if (run.data === undefined) {
      throw new Error(`run ${run.id} ended ${run.status} (${run.stopReason ?? 'no reason given'})`);
    }
    return run.data as Order;
  });
  lane = next.catch(() => undefined);
  return next;
}

const app = new Hono();

app.get('/orders/:po', async (c) => {
  const po = c.req.param('po').toUpperCase();
  if (!/^[A-Z0-9-]{4,20}$/.test(po)) {
    return c.json({ error: 'A purchase order number is 4 to 20 letters, digits or dashes.' }, 400);
  }

  const hit = cache.get(po);
  if (hit !== undefined && Date.now() - hit.at < FRESH_FOR_MS) {
    return c.json({ ...hit.order, cachedAt: new Date(hit.at).toISOString() }, hit.order.found ? 200 : 404);
  }

  // Two callers asking for the same order at once share one run.
  let pending = inFlight.get(po);
  if (pending === undefined) {
    pending = lookUp(po).finally(() => inFlight.delete(po));
    inFlight.set(po, pending);
  }

  try {
    const order = await pending;
    cache.set(po, { at: Date.now(), order });
    return c.json(order, order.found ? 200 : 404);
  } catch (err) {
    return c.json({ error: String(err) }, 502);
  }
});

serve({ fetch: app.fetch, port: 8787 });
console.log('GET http://localhost:8787/orders/PO-20931');

import asyncio
import os
import re
import time

from fastapi import FastAPI, HTTPException
from outcrawl import Outcrawl

PORTAL = "https://dealers.harbor-supply.example"
PROFILE_ID = os.environ["PORTAL_PROFILE_ID"]  # the profile that is signed in to the portal
FRESH_FOR = 10 * 60  # seconds an answer is served from cache

ORDER_SCHEMA = {
    "type": "object",
    "properties": {
        "found": {"type": "boolean", "description": "false when the portal has no order with this number"},
        "po": {"type": "string"},
        "status": {"type": "string", "description": "the status exactly as the portal shows it"},
        "orderedOn": {"type": "string", "description": "YYYY-MM-DD"},
        "shipBy": {"type": "string", "description": "YYYY-MM-DD"},
        "carrier": {"type": "string"},
        "tracking": {"type": "string"},
        "lines": {
            "type": "array",
            "items": {
                "type": "object",
                "properties": {
                    "sku": {"type": "string"},
                    "description": {"type": "string"},
                    "quantity": {"type": "integer"},
                    "unitPrice": {"type": "string", "description": "with currency, e.g. $14.20"},
                },
                "required": ["sku", "quantity"],
            },
        },
    },
    "required": ["found", "po"],
}

oc = Outcrawl()  # one long-lived client for the server
app = FastAPI()
cache: dict[str, tuple[float, dict]] = {}
in_flight: dict[str, asyncio.Task] = {}
lane = asyncio.Lock()  # a profile holds one session at a time, so runs on it go one after another


async def look_up(po: str) -> dict:
    async with lane:
        run = await oc.agent(
            task=f"On {PORTAL}, open Orders, search for purchase order {po} and open it. "
            "Report its status, order date, ship-by date, carrier, tracking number and every line item. "
            "If no order has that number, set found to false.",
            profile=PROFILE_ID,
            schema=ORDER_SCHEMA,
            caps={"budget": "30.00"},
            rules=["Never place, change, cancel or reorder an order."],
        )
    data = run.get("data")
    if data is None:
        raise RuntimeError(f"run {run['id']} ended {run['status']} ({run.get('stopReason', 'no reason given')})")
    return data


@app.get("/orders/{po}")
async def get_order(po: str) -> dict:
    po = po.upper()
    if not re.fullmatch(r"[A-Z0-9-]{4,20}", po):
        raise HTTPException(400, "A purchase order number is 4 to 20 letters, digits or dashes.")

    hit = cache.get(po)
    if hit is None or time.monotonic() - hit[0] >= FRESH_FOR:
        # Two callers asking for the same order at once share one run.
        task = in_flight.get(po)
        if task is None:
            task = asyncio.create_task(look_up(po))
            in_flight[po] = task
            task.add_done_callback(lambda _: in_flight.pop(po, None))
        try:
            order = await asyncio.shield(task)  # a caller hanging up does not cancel the shared run
        except RuntimeError as err:
            raise HTTPException(502, str(err)) from err
        hit = (time.monotonic(), order)
        cache[po] = hit

    order = hit[1]
    if not order["found"]:
        raise HTTPException(404, f"No order {po} on the portal.")
    return order


# Run it: uvicorn orders:app --port 8787