{
 "id": "asi-pilot-2026-11-02",
 "benchmark": "ARC Agent Shopping Index",
 "edition": "pilot",
 "title": "Can AI agents check out on their own? First pilot: 35 stores, 3 models",
 "run_date": "2026-11-02",
 "run_window_utc": "08:49-09:24",
 "source_run": "ARC-100 run 3",
 "method_version": "task-v1",
 "harness_commit": "9f3d75e",
 "license": "CC-BY-4.0",
 "attribution": "ARC (arcreport.ai)",
 "method": {
  "task": "task-v1 (product page): open one fixed product URL, add that product at qty 1, reach the first checkout step.",
  "policy": "no-assist: the agent gets the goal text only, no store-specific hints",
  "stop": "stop before payment: no payment control is pressed, no order is placed",
  "accounts": "no accounts: no sign-in, no account created, no personal data entered",
  "limits": "16 steps, 180 s, $0.50 per attempt",
  "checkout_rule": "A checkout step loaded (contact, address or e-mail step) with the right product at qty 1 visible in the summary or in the cart just before it. A sign-in or guest-choice page with no form does not count.",
  "denominator": "n leaves out not_run cells and includes real store blocks at the door.",
  "ci": "Wilson score interval, 95%",
  "models_via": "OpenRouter API, benchmark personas, not the consumer apps"
 },
 "store_set": {
  "name": "ARC-100 v1",
  "frozen": "2026-10-07",
  "stores_run": 35,
  "skipped_shopify": 64,
  "skipped_opted_out": 1,
  "note": "Shopify stores are behind a switch and were not run. This pilot covers the 35 non-Shopify stores of ARC-100 only."
 },
 "review": {
  "protocol": "Every harness checkout and every refused row (38), plus a seeded random sample of 30 other executed rows (random.Random(2026).sample over a pool of 57). One reviewer. Strict pass rule as in method.checkout_rule.",
  "reviewers": 1,
  "seed": 2026,
  "agreement": {
   "k": 67,
   "n": 68,
   "set": "required set (mandatory + random)"
  },
  "agreement_all_reviewed": {
   "k": 70,
   "n": 72
  },
  "false_pass": 0,
  "false_fail": 1,
  "cart_level_underscore": 1,
  "false_block": 0
 },
 "spend_usd": 3.972,
 "outcomes_legend": {
  "checkout": "reached a checkout step with the right product",
  "cart_only": "product in cart, no checkout step",
  "failed": "did not get the product into the cart",
  "blocked": "store blocked the browser at the door",
  "not_run": "store not reachable (maintenance page)"
 },
 "models": [
  {
   "key": "claude-sonnet-5.5",
   "openrouter_id": "anthropic/claude-sonnet-5.5",
   "label": "Claude Sonnet 5.5",
   "checkout": 16,
   "n": 34,
   "ci95": [
    0.315,
    0.633
   ],
   "cart_or_better": 22,
   "blocked": 3,
   "not_run": 1,
   "harness_checkout": 16,
   "sessions": 33,
   "median_steps": 5,
   "median_seconds": 49.0,
   "cost_usd": 2.7743,
   "cost_per_checkout_usd": 0.173
  },
  {
   "key": "gemini-3.8-flash",
   "openrouter_id": "google/gemini-3.8-flash",
   "label": "Gemini 3.8 Flash",
   "checkout": 12,
   "n": 34,
   "ci95": [
    0.215,
    0.521
   ],
   "cart_or_better": 21,
   "blocked": 3,
   "not_run": 1,
   "harness_checkout": 12,
   "sessions": 33,
   "median_steps": 5,
   "median_seconds": 63.9,
   "cost_usd": 1.0517,
   "cost_per_checkout_usd": 0.088
  },
  {
   "key": "qwen3-vl-235b",
   "openrouter_id": "qwen/qwen3-vl-235b-a22b-instruct",
   "label": "Qwen3-VL 235B",
   "checkout": 9,
   "n": 34,
   "ci95": [
    0.146,
    0.431
   ],
   "cart_or_better": 19,
   "blocked": 3,
   "not_run": 1,
   "harness_checkout": 8,
   "sessions": 33,
   "median_steps": 6,
   "median_seconds": 55.6,
   "cost_usd": 0.2812,
   "cost_per_checkout_usd": 0.031
  }
 ],
 "stores": [
  {
   "store": "vuoriclothing.com",
   "category": "apparel",
   "platform": "not_detected",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 7,
     "seconds": 79.7
    },
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 1,
     "seconds": 30.8
    },
    "qwen3-vl-235b": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 7,
     "seconds": 86.4
    }
   }
  },
  {
   "store": "www.crewclothing.co.uk",
   "category": "apparel",
   "platform": "not_detected",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 1,
     "seconds": 11.7
    },
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 1,
     "seconds": 31.2
    },
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 2,
     "seconds": 19.5
    }
   }
  },
  {
   "store": "www.patagonia.com",
   "category": "apparel",
   "platform": "not_detected",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "not_run",
     "harness": "not_run",
     "hand_reviewed": false,
     "steps": 1,
     "seconds": 13.7
    },
    "gemini-3.8-flash": {
     "outcome": "not_run",
     "harness": "not_run",
     "hand_reviewed": false,
     "steps": 1,
     "seconds": 13.7
    },
    "qwen3-vl-235b": {
     "outcome": "not_run",
     "harness": "not_run",
     "hand_reviewed": false,
     "steps": 1,
     "seconds": 11.7
    }
   }
  },
  {
   "store": "www.phase-eight.com",
   "category": "apparel",
   "platform": "salesforce",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 13,
     "seconds": 184.1
    },
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 12,
     "seconds": 184.3
    },
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 6,
     "seconds": 143.1
    }
   }
  },
  {
   "store": "theordinary.com",
   "category": "beauty",
   "platform": "not_detected",
   "outcomes": {
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 14,
     "seconds": 184.9
    },
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 107.5
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 7,
     "seconds": 69.6
    }
   }
  },
  {
   "store": "www.drunkelephant.com",
   "category": "beauty",
   "platform": "not_detected",
   "outcomes": {
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 13,
     "seconds": 188.2
    },
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 6,
     "seconds": 78.3
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 49.9
    }
   }
  },
  {
   "store": "north-america.beyerdynamic.com",
   "category": "electronics",
   "platform": "not_detected",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 6,
     "seconds": 49.0
    },
    "gemini-3.8-flash": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 5,
     "seconds": 100.1
    },
    "qwen3-vl-235b": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 3,
     "seconds": 32.2
    }
   }
  },
  {
   "store": "powercity.ie",
   "category": "electronics",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 10,
     "seconds": 118.8
    },
    "claude-sonnet-5.5": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 9,
     "seconds": 83.1
    },
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 12,
     "seconds": 181.4
    }
   }
  },
  {
   "store": "www.alternate.de",
   "category": "electronics",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 31.7
    },
    "qwen3-vl-235b": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 36.0
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 29.1
    }
   }
  },
  {
   "store": "www.marshall.com",
   "category": "electronics",
   "platform": "not_detected",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 3,
     "seconds": 33.0
    },
    "qwen3-vl-235b": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 3,
     "seconds": 30.0
    },
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 3,
     "seconds": 29.8
    }
   }
  },
  {
   "store": "www.dovesfarm.co.uk",
   "category": "food-drink",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 5,
     "seconds": 51.8
    },
    "claude-sonnet-5.5": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 5,
     "seconds": 38.0
    },
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 5,
     "seconds": 43.3
    }
   }
  },
  {
   "store": "www.palaisdesthes.com",
   "category": "food-drink",
   "platform": "magento",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 39.6
    },
    "gemini-3.8-flash": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 4,
     "seconds": 48.2
    },
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 45.7
    }
   }
  },
  {
   "store": "www.cheaney.co.uk",
   "category": "footwear",
   "platform": "not_detected",
   "outcomes": {
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 16,
     "seconds": 132.2
    },
    "claude-sonnet-5.5": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 13,
     "seconds": 144.0
    },
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 10,
     "seconds": 147.4
    }
   }
  },
  {
   "store": "www.office.co.uk",
   "category": "footwear",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "blocked",
     "harness": "blocked",
     "hand_reviewed": true,
     "steps": 1,
     "seconds": 4.0
    },
    "claude-sonnet-5.5": {
     "outcome": "blocked",
     "harness": "blocked (skipped)",
     "hand_reviewed": false,
     "skipped_after_block": true,
     "steps": null,
     "seconds": null
    },
    "qwen3-vl-235b": {
     "outcome": "blocked",
     "harness": "blocked (skipped)",
     "hand_reviewed": false,
     "skipped_after_block": true,
     "steps": null,
     "seconds": null
    }
   }
  },
  {
   "store": "www.rmwilliams.com",
   "category": "footwear",
   "platform": "salesforce",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 9,
     "seconds": 79.3
    },
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 12,
     "seconds": 190.4
    },
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 3,
     "seconds": 34.6
    }
   }
  },
  {
   "store": "www.dunelm.com",
   "category": "home",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 6,
     "seconds": 83.2
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 7,
     "seconds": 57.2
    },
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 12,
     "seconds": 187.0
    }
   }
  },
  {
   "store": "www.angara.com",
   "category": "jewelry",
   "platform": "not_detected",
   "outcomes": {
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 11,
     "seconds": 111.4
    },
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 2,
     "seconds": 50.4
    },
    "claude-sonnet-5.5": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 2,
     "seconds": 26.6
    }
   }
  },
  {
   "store": "www.georgjensen.com",
   "category": "jewelry",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 7,
     "seconds": 96.1
    },
    "claude-sonnet-5.5": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 6,
     "seconds": 59.0
    },
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 6,
     "seconds": 60.6
    }
   }
  },
  {
   "store": "www.jared.com",
   "category": "jewelry",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 1,
     "seconds": 21.4
    },
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 1,
     "seconds": 27.6
    },
    "claude-sonnet-5.5": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 1,
     "seconds": 18.2
    }
   }
  },
  {
   "store": "www.queensmith.co.uk",
   "category": "jewelry",
   "platform": "not_detected",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 2,
     "seconds": 21.8
    },
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 2,
     "seconds": 31.0
    },
    "qwen3-vl-235b": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 2,
     "seconds": 21.2
    }
   }
  },
  {
   "store": "www.bugaboo.com",
   "category": "kids",
   "platform": "not_detected",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 6,
     "seconds": 40.7
    },
    "qwen3-vl-235b": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 7,
     "seconds": 70.5
    },
    "gemini-3.8-flash": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 6,
     "seconds": 73.8
    }
   }
  },
  {
   "store": "www.childrensplace.com",
   "category": "kids",
   "platform": "not_detected",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "blocked",
     "harness": "blocked",
     "hand_reviewed": true,
     "steps": 1,
     "seconds": 1.7
    },
    "gemini-3.8-flash": {
     "outcome": "blocked",
     "harness": "blocked (skipped)",
     "hand_reviewed": false,
     "skipped_after_block": true,
     "steps": null,
     "seconds": null
    },
    "qwen3-vl-235b": {
     "outcome": "blocked",
     "harness": "blocked (skipped)",
     "hand_reviewed": false,
     "skipped_after_block": true,
     "steps": null,
     "seconds": null
    }
   }
  },
  {
   "store": "www.cybex-online.com",
   "category": "kids",
   "platform": "not_detected",
   "outcomes": {
    "qwen3-vl-235b": {
     "outcome": "checkout",
     "harness": "unverified",
     "hand_reviewed": true,
     "steps": 6,
     "seconds": 55.6
    },
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 5,
     "seconds": 54.6
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 5,
     "seconds": 52.1
    }
   }
  },
  {
   "store": "www.nameit.com",
   "category": "kids",
   "platform": "not_detected",
   "outcomes": {
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 12,
     "seconds": 126.4
    },
    "claude-sonnet-5.5": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 6,
     "seconds": 60.5
    },
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 7,
     "seconds": 93.3
    }
   }
  },
  {
   "store": "paramo-clothing.com",
   "category": "outdoor",
   "platform": "woocommerce",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 1,
     "seconds": 11.9
    },
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 1,
     "seconds": 15.1
    },
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 3,
     "seconds": 17.6
    }
   }
  },
  {
   "store": "www.bergans.com",
   "category": "outdoor",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 2,
     "seconds": 19.1
    },
    "claude-sonnet-5.5": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": false,
     "steps": 2,
     "seconds": 15.6
    },
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 2,
     "seconds": 15.9
    }
   }
  },
  {
   "store": "www.cotswoldoutdoor.com",
   "category": "outdoor",
   "platform": "not_detected",
   "outcomes": {
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 13,
     "seconds": 186.3
    },
    "gemini-3.8-flash": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 7,
     "seconds": 99.8
    },
    "claude-sonnet-5.5": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 7,
     "seconds": 76.2
    }
   }
  },
  {
   "store": "www.hardloop.co.uk",
   "category": "outdoor",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 4,
     "seconds": 63.9
    },
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 4,
     "seconds": 51.9
    },
    "claude-sonnet-5.5": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 47.9
    }
   }
  },
  {
   "store": "www.fressnapf.de",
   "category": "pets",
   "platform": "not_detected",
   "outcomes": {
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 11,
     "seconds": 221.9
    },
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 10,
     "seconds": 95.6
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 9,
     "seconds": 61.0
    }
   }
  },
  {
   "store": "www.maxizoo.ie",
   "category": "pets",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 6,
     "seconds": 80.7
    },
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 8,
     "seconds": 183.2
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 6,
     "seconds": 51.3
    }
   }
  },
  {
   "store": "www.poochandmutt.co.uk",
   "category": "pets",
   "platform": "not_detected",
   "outcomes": {
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 5,
     "seconds": 44.8
    },
    "qwen3-vl-235b": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 5,
     "seconds": 49.4
    },
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 5,
     "seconds": 53.9
    }
   }
  },
  {
   "store": "drinklmnt.com",
   "category": "supplements",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 52.3
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 41.9
    },
    "qwen3-vl-235b": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 12,
     "seconds": 189.2
    }
   }
  },
  {
   "store": "huel.com",
   "category": "supplements",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "cart_only",
     "harness": "cart_only",
     "hand_reviewed": false,
     "steps": 10,
     "seconds": 225.5
    },
    "qwen3-vl-235b": {
     "outcome": "failed",
     "harness": "failed",
     "hand_reviewed": true,
     "steps": 4,
     "seconds": 43.2
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 10,
     "seconds": 126.4
    }
   }
  },
  {
   "store": "seed.com",
   "category": "supplements",
   "platform": "not_detected",
   "outcomes": {
    "qwen3-vl-235b": {
     "outcome": "blocked",
     "harness": "blocked",
     "hand_reviewed": true,
     "steps": 1,
     "seconds": 7.1
    },
    "claude-sonnet-5.5": {
     "outcome": "blocked",
     "harness": "blocked (skipped)",
     "hand_reviewed": false,
     "skipped_after_block": true,
     "steps": null,
     "seconds": null
    },
    "gemini-3.8-flash": {
     "outcome": "blocked",
     "harness": "blocked (skipped)",
     "hand_reviewed": false,
     "skipped_after_block": true,
     "steps": null,
     "seconds": null
    }
   }
  },
  {
   "store": "www.bulknutrients.com.au",
   "category": "supplements",
   "platform": "not_detected",
   "outcomes": {
    "gemini-3.8-flash": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 5,
     "seconds": 60.9
    },
    "qwen3-vl-235b": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 5,
     "seconds": 50.7
    },
    "claude-sonnet-5.5": {
     "outcome": "checkout",
     "harness": "checkout",
     "hand_reviewed": true,
     "steps": 5,
     "seconds": 48.1
    }
   }
  }
 ],
 "known_issues": [
  "ARC safety filter refused two harmless guest-checkout buttons for Gemini (Dunelm, Bugaboo); at least one checkout lost to ARC, not the model.",
  "Run-to-run variance is about 8% of cells between runs on the same stores."
 ]
}