{
  "_note": "Community/third-party tests compiled by BuildOff. Scores and rankings are copied from sources we opened. Empty ranking = no declared result we could verify. includeInTallies=false means excluded from home aggregates.",
  "compiledAt": "2026-10-16",
  "ourAutomatedTests": "coming later — not run yet; no fabricated BuildOff scores",
  "tests": [
    {
      "id": "smf-prompt-to-app-2026-06",
      "title": "Prompt-to-App Benchmark (task tracker with auth + DB)",
      "sourceUrl": "https://www.smfclearinghouse.com/tests/prompt-to-app-benchmark/",
      "opened": true,
      "author": "SMF Clearinghouse",
      "date": "2026-06-10",
      "format": "article",
      "toolsTested": [
        "lovable",
        "bolt",
        "replit",
        "v0"
      ],
      "task": "Build a task tracker app. Users sign up, sign in, add tasks, mark them complete, and delete them. Store tasks in a real database. Deploy it. (single prompt, no follow-ups)",
      "method": "hand",
      "methodDetail": "Same one-paragraph prompt; manual testing of signup/login/CRUD/persistence; scored rubric (deployed URL 30%, auth 25%, DB 20%, UI 15%, cost/speed 10%).",
      "resultAsStated": "Scores: Lovable 88 (14m, ~$8, Pass); Bolt.new 82 (18m, ~$12, Pass); Replit Agent 74 (25m, ~$5.50, Pass); v0 65 (35m, $0, Fail). Winner: Lovable.",
      "ranking": [
        {
          "builderId": "lovable",
          "place": 1,
          "scoreLabel": "88"
        },
        {
          "builderId": "bolt",
          "place": 2,
          "scoreLabel": "82"
        },
        {
          "builderId": "replit",
          "place": 3,
          "scoreLabel": "74"
        },
        {
          "builderId": "v0",
          "place": 4,
          "scoreLabel": "65 (Fail)"
        }
      ],
      "conflictOfInterest": "None stated. Site says 'Not sponsored. Not cherry-picked.'",
      "notes": "Strongest independent same-prompt full-stack scorecard found.",
      "includeInTallies": true,
      "slug": "smf-prompt-to-app-2026-06",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "smf-agent-ui-2026-06",
      "title": "Agent UI Generation Benchmark (React + Tailwind dashboard)",
      "sourceUrl": "https://www.smfclearinghouse.com/tests/agent-ui-generation-benchmark/",
      "opened": true,
      "author": "SMF Clearinghouse",
      "date": "2026-06-16",
      "format": "article",
      "toolsTested": [
        "v0",
        "lovable",
        "bolt",
        "claude-code"
      ],
      "task": "Responsive React dashboard: sidebar, metric cards, line chart placeholder, recent activity table, mobile responsiveness, Tailwind.",
      "method": "hand",
      "methodDetail": "Same prompt, no design references; rendered in clean Next.js; desktop + mobile evaluation.",
      "resultAsStated": "Scores: v0 91 (8m, ~$0.75, Pass); Lovable 86 (11m, ~$0.95, Pass); Bolt.new 82 (14m, ~$1.10, Pass); Claude Code 74 (26m, ~$2.10, Pass). Winner: v0.",
      "ranking": [
        {
          "builderId": "v0",
          "place": 1,
          "scoreLabel": "91"
        },
        {
          "builderId": "lovable",
          "place": 2,
          "scoreLabel": "86"
        },
        {
          "builderId": "bolt",
          "place": 3,
          "scoreLabel": "82"
        }
      ],
      "conflictOfInterest": "None stated.",
      "notes": "UI-only task; Claude Code included but not an AI app builder in our roster.",
      "includeInTallies": true,
      "slug": "smf-agent-ui-2026-06",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "technically-vandelay-2026-08",
      "title": "2026 vibe coding tool comparison (Vandelay Industries ops dashboard)",
      "sourceUrl": "https://technically.dev/posts/vibe-coding-tool-comparison",
      "opened": true,
      "author": "Justin Gage (technically.dev)",
      "date": "2026-08-25",
      "format": "article",
      "toolsTested": [
        "replit",
        "v0",
        "lovable",
        "bolt"
      ],
      "task": "Internal ops dashboard for Vandelay Industries: inventory/CRM table of potato-chip shipments, analytics/charts, map of shipments. Iterated with bug-fix prompts.",
      "method": "hand",
      "methodDetail": "Same starting prompt across tools; author inspected apps and prompted fixes one by one. Paid for 2 of 4 tools.",
      "resultAsStated": "TL;DR as written: 'Replit was the most feature rich, well thought out, and powerful'; 'v0 was the best if you’re already a developer and want a technical interface'; 'Lovable and Bolt were pretty much objectively worse than the other two.' Final pick: 'If I were starting a new project from scratch I’d use Replit or v0.'",
      "ranking": [
        {
          "builderId": "replit",
          "place": 1,
          "scoreLabel": "most feature rich / best finished app (qualitative)"
        },
        {
          "builderId": "v0",
          "place": 2,
          "scoreLabel": "best for developers (qualitative)"
        },
        {
          "builderId": "lovable",
          "place": 3,
          "scoreLabel": "objectively worse (tied qualitative)"
        },
        {
          "builderId": "bolt",
          "place": 3,
          "scoreLabel": "objectively worse (tied qualitative)"
        }
      ],
      "conflictOfInterest": "None stated for any vendor.",
      "notes": "No numeric scores. Places 3 tied for Lovable and Bolt per author's wording.",
      "includeInTallies": true,
      "slug": "technically-vandelay-2026-08",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "converge-slack-2026-01",
      "title": "Vibe Coding Benchmark: Building a Slack Clone",
      "sourceUrl": "https://converge.run/blog/vibe-coding-slack-clone-test",
      "opened": true,
      "author": "Dan Cleary (Converge)",
      "date": "2026-01-23",
      "format": "article",
      "toolsTested": [
        "converge",
        "lovable",
        "bolt",
        "replit",
        "v0"
      ],
      "task": "Slack-like app with 13 checklist features (auth, channels, messages, profile photo storage, search, typing indicators, private channels, unread badges, pins, realtime, etc.).",
      "method": "hand",
      "methodDetail": "Exact same prompt; first generation only; no fixing; one point per working checklist item (13 max).",
      "resultAsStated": "Final scores (highest → lowest): Converge — 13/13; Replit — 5/13; v0 — 3/13; Bolt — 2/13; Lovable — 0/13.",
      "ranking": [
        {
          "builderId": "converge",
          "place": 1,
          "scoreLabel": "13/13"
        },
        {
          "builderId": "replit",
          "place": 2,
          "scoreLabel": "5/13"
        },
        {
          "builderId": "v0",
          "place": 3,
          "scoreLabel": "3/13"
        },
        {
          "builderId": "bolt",
          "place": 4,
          "scoreLabel": "2/13"
        },
        {
          "builderId": "lovable",
          "place": 5,
          "scoreLabel": "0/13"
        }
      ],
      "conflictOfInterest": "Strong — Converge is the author's product and finished 13/13 while competitors scored ≤5. Treat as vendor-adjacent.",
      "notes": "Useful checklist detail; tallies still counted but COI flagged on the site.",
      "includeInTallies": true,
      "slug": "converge-slack-2026-01",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "arxiv-prompt-to-product-2025",
      "title": "From Prompt to Product: Human-Centered Benchmark (Replit, Bolt, Firebase Studio)",
      "sourceUrl": "https://arxiv.org/html/2512.18080",
      "opened": true,
      "author": "Marcos Ortiz, Justin Hill, Collin Overbay, Ingrida Semenec, Frederic Sauve-Hoover, Jim Schwoebel, Joel Shor",
      "date": "2025-12",
      "format": "preprint",
      "toolsTested": [
        "replit",
        "bolt",
        "firebase-studio"
      ],
      "task": "96 prompts spanning domains/difficulty; 288 generated apps; 205 human raters; 1,071 pairwise comparisons after quality filtering.",
      "method": "mixed",
      "methodDetail": "Single-shot generation per system+prompt; deployed; human isolated ratings + side-by-side preferences (ease, trust, visual appeal, visual appropriateness). Bradley-Terry / LMM analysis.",
      "resultAsStated": "Pairwise win rates (Table 2): Firebase highest on all four metrics (Ease 42.5%, Trust 41.2%, Visual Appeal 45.5%, Visual Appropriateness 45.1%); Bolt second; Replit third. Authors' hierarchy: 'Firebase > Bolt > Replit'. Composite scores stated: Firebase 1.933, Bolt 1.561, Replit 1.302.",
      "ranking": [
        {
          "builderId": "firebase-studio",
          "place": 1,
          "scoreLabel": "composite 1.933 / top win rates"
        },
        {
          "builderId": "bolt",
          "place": 2,
          "scoreLabel": "composite 1.561"
        },
        {
          "builderId": "replit",
          "place": 3,
          "scoreLabel": "composite 1.302"
        }
      ],
      "conflictOfInterest": "Study funded by Quome Inc. (acknowledgements). Quome Cloud used for deployment.",
      "notes": "Largest academic human study found. Does not include Lovable/v0/Base44.",
      "includeInTallies": true,
      "slug": "arxiv-prompt-to-product-2025",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "hostadvice-base44-lovable-bolt-2026-07",
      "title": "Base44 vs Lovable vs Bolt — same client management app",
      "sourceUrl": "https://nz.hostadvice.com/ai-app-builders/base44-vs-lovable-vs-bolt/",
      "opened": true,
      "author": "Walter Akolo (HostAdvice)",
      "date": "2026-07-01",
      "format": "article",
      "toolsTested": [
        "base44",
        "lovable",
        "bolt"
      ],
      "task": "Complex client project management system: multi-user auth, RBAC, dashboard analytics, projects, time logging, invoices/PDF, Stripe, client portals.",
      "method": "hand",
      "methodDetail": "Same detailed prompt on all three; timed builds; inspected code/architecture/deployment.",
      "resultAsStated": "Overall winner: Lovable. Stated metrics include build time Base44 6 min / Lovable 10 min / Bolt 4–5 min*; first-build success rate Base44 100% / Lovable 90% / Bolt 30%. Author: 'Avoid Bolt for now' due to preview/deploy failures. Category winners: Pricing Base44; AI capabilities Base44; Speed/quality split (Base44 faster, Lovable better quality); Ease Base44; Integrations Lovable.",
      "ranking": [
        {
          "builderId": "lovable",
          "place": 1,
          "scoreLabel": "overall winner"
        },
        {
          "builderId": "base44",
          "place": 2,
          "scoreLabel": "speed / ease / pricing winner"
        },
        {
          "builderId": "bolt",
          "place": 3,
          "scoreLabel": "30% first-build success; avoid"
        }
      ],
      "conflictOfInterest": "Affiliate/review site; HostAdvice often monetizes tool reviews (exact affiliate status for these three UNVERIFIED on the opened page).",
      "notes": "Detailed timed metrics. Overall ranking uses author's stated overall winner.",
      "includeInTallies": true,
      "slug": "hostadvice-base44-lovable-bolt-2026-07",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "emergent-bolt-base44-2026-08",
      "title": "Bolt vs Base44 vs Emergent — same waitlist app + follow-up",
      "sourceUrl": "https://emergent.sh/learn/bolt-vs-base44",
      "opened": true,
      "author": "Anupam (Emergent), reviewed by Anmol",
      "date": "2026-08-25",
      "format": "article",
      "toolsTested": [
        "bolt",
        "base44",
        "emergent"
      ],
      "task": "Waitlist landing page with email capture, DB, admin signup count; then follow-up: add company name field to form + admin.",
      "method": "hand",
      "methodDetail": "Same two-prompt workflow; default models/stacks; timed; noted credit burn.",
      "resultAsStated": "Author final: 'Emergent is still where I'd start for a real production app'; Base44 'genuinely the fastest to a demo'; Bolt 'honesty when things go wrong' but free-tier tokens burned on fixes. Explicit winners by category: Speed → Base44; Complex logic / testing evidence → Emergent; Code ownership → Bolt; Mobile → Emergent.",
      "ranking": [
        {
          "builderId": "emergent",
          "place": 1,
          "scoreLabel": "author's production pick"
        },
        {
          "builderId": "base44",
          "place": 2,
          "scoreLabel": "fastest demo"
        },
        {
          "builderId": "bolt",
          "place": 3,
          "scoreLabel": "code ownership; preview issues"
        }
      ],
      "conflictOfInterest": "Strong — published on emergent.sh (vendor content).",
      "notes": "Useful timings (Base44 ~2–5 min publish; Emergent ~16 min; Bolt ~19 min with fixes).",
      "includeInTallies": true,
      "slug": "emergent-bolt-base44-2026-08",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "deccan-webapp-study",
      "title": "WebApp Study: Evaluating AI-Native Web Application Builders",
      "sourceUrl": "https://www.deccan.ai/research/webapp-study-evaluating-ai-native-web-application-builders",
      "opened": true,
      "author": "Deccan.ai research team (multiple annotators)",
      "date": null,
      "dateNote": "Page does not state a clear publication date in the fetched text.",
      "format": "research",
      "toolsTested": [
        "lovable",
        "bolt",
        "v0",
        "firebase-studio",
        "replit"
      ],
      "task": "53 curated real-world tasks across Easy/Medium/Hard; seven rubrics (functionality, visual appeal, error handling, responsiveness, content quality, navigation, data consistency).",
      "method": "hand",
      "methodDetail": "Blind independent human expert scoring; multi-annotator.",
      "resultAsStated": "No numeric leaderboard table was readable in the fetched HTML (Table 1.1 / charts are images). Qualitative claims on the page: 'Lovable stands out for its reliability, functional consistency, and polished design'; 'v0 maintains the highest functional testing and navigation scores across all complexities'; 'Replit underperforms across most metrics'; 'Lovable and Bolt excel visually but lack backend robustness'; 'Firebase GenAI Studio's full-stack support' highlighted. 'No single tool demonstrates comprehensive reliability.'",
      "ranking": [],
      "conflictOfInterest": "Independent research firm; no vendor sponsorship stated on the opened page.",
      "notes": "Included for methodology/quotes only — NOT counted in 1st/top-3/last tallies because numeric/place rankings were not extractable from opened text.",
      "includeInTallies": false,
      "slug": "deccan-webapp-study",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "x-alexhalescoder-14-ranked-2026-10",
      "title": "X: 'I ranked 14 AI app builders by strength'",
      "sourceUrl": "https://x.com/alexhalescoder/status/2110719570726265145",
      "opened": true,
      "author": "Roxo (@alexhalescoder)",
      "date": "2026-10-15",
      "format": "x-post",
      "toolsTested": [
        "replit",
        "v0",
        "bolt",
        "emergent",
        "lindy",
        "nxcode",
        "taskade",
        "openai-agent-builder",
        "same-new",
        "grok-build",
        "framer",
        "google-stitch",
        "antigravity",
        "sticklight"
      ],
      "task": "Opinion ranking 'by strength' — not a single shared build prompt.",
      "method": "opinion",
      "methodDetail": "Author list ranked 1–14 with short tags; image + long note confirmed.",
      "resultAsStated": "Ranked by strength: 1 Replit Agent 3; 2 v0 by Vercel; 3 Lindy; 4 NxCode; 5 Taskade Genesis; 6 OpenAI Agent Builder; 7 Bolt.new; 8 Same.new; 9 Emergent AI; 10 Grok Build Mode; 11 Framer AI Agents; 12 Google Stitch; 13 Antigravity; 14 Sticklight. Quick takes: backend-heavy → Replit; beautiful UI → v0 or Google Stitch; landing page → Framer; clone a site → Same.new.",
      "ranking": [
        {
          "builderId": "replit",
          "place": 1,
          "scoreLabel": "#1 of 14"
        },
        {
          "builderId": "v0",
          "place": 2,
          "scoreLabel": "#2 of 14"
        },
        {
          "builderId": "bolt",
          "place": 7,
          "scoreLabel": "#7 of 14"
        },
        {
          "builderId": "emergent",
          "place": 9,
          "scoreLabel": "#9 of 14"
        }
      ],
      "conflictOfInterest": "Author bio: 'AI Educator… DM Open for collaboration'. Not a controlled same-prompt test.",
      "notes": "Counted in tallies with method=opinion so readers can filter. Lovable/Base44 not in the 14.",
      "includeInTallies": true,
      "slug": "x-alexhalescoder-14-ranked-2026-10",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "x-promptleague-founder-os-2026-08",
      "title": "X: same frontend prompt → Founder OS dashboard (5 builders)",
      "sourceUrl": "https://x.com/Promptleague13/status/2089657904387555584",
      "opened": true,
      "author": "Evan Cole (@Promptleague13)",
      "date": "2026-08-18",
      "format": "x-post",
      "toolsTested": [
        "lovable",
        "bolt",
        "v0",
        "replit",
        "base44"
      ],
      "task": "Exact same frontend prompt; default settings; one attempt; each builds a Founder OS dashboard. Post asks 'Which one wins?'",
      "method": "hand",
      "methodDetail": "Same prompt, one attempt, default settings (per post text). Attached image is a teaser listing the five tools — no ranking table in the image text we could read.",
      "resultAsStated": "NO DECLARED RANKING in the opened post text or readable image content. Author asks the audience which wins. Follow-up search from:Promptleague13 for a winner post returned 0 results.",
      "ranking": [],
      "conflictOfInterest": "Independent tester bio; no vendor link stated.",
      "notes": "Included as a verified same-prompt attempt; excluded from tallies until a declared result is found.",
      "includeInTallies": false,
      "slug": "x-promptleague-founder-os-2026-08",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "x-theoreed-northform-2026-09",
      "title": "X: same e-commerce prompt → Northform product page (5 builders)",
      "sourceUrl": "https://x.com/TheoReednp/status/2095469372785758397",
      "opened": true,
      "author": "Theo Reed (@TheoReednp)",
      "date": "2026-09-03",
      "format": "x-post",
      "toolsTested": [
        "lovable",
        "bolt",
        "replit",
        "base44",
        "emergent"
      ],
      "task": "Exact same e-commerce prompt; default settings; complete product page for fictional brand Northform.",
      "method": "hand",
      "methodDetail": "Same prompt, default settings (per post). Image is a comparison teaser with a Northform backpack product-page screenshot — no ranking numbers readable.",
      "resultAsStated": "NO DECLARED RANKING in the opened post text or readable image content. Author asks 'Which one wins?'. Follow-up search from:TheoReednp for a winner post returned 0 results.",
      "ranking": [],
      "conflictOfInterest": "Independent tester bio; no vendor link stated.",
      "notes": "Verified attempt including Base44 + Emergent; excluded from tallies until a declared result is found.",
      "includeInTallies": false,
      "slug": "x-theoreed-northform-2026-09",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    },
    {
      "id": "reddit-vibecoding-booking-blocked",
      "title": "Reddit r/vibecoding: same booking app prompt across 6 builders",
      "sourceUrl": "https://www.reddit.com/r/vibecoding/comments/1vjrpsm/i_tested_emergent_lovable_bolt_replit_v0_and/",
      "opened": false,
      "author": "unknown (page blocked)",
      "date": null,
      "format": "reddit",
      "toolsTested": [
        "emergent",
        "lovable",
        "bolt",
        "replit",
        "v0",
        "base44"
      ],
      "task": "Appointment booking app with customer accounts, staff page, DB, email API that must fail visibly, public deploy (per search-snippet summary — NOT verified from page body).",
      "method": "hand",
      "methodDetail": "UNVERIFIED — Reddit returned a network security block page; body not read.",
      "resultAsStated": "NOT VERIFIED. Search snippets suggest no single clear winner and situational guidance; do not treat as scored.",
      "ranking": [],
      "conflictOfInterest": "Unknown.",
      "notes": "Listed so we don't pretend it was read. Excluded from tallies.",
      "includeInTallies": false,
      "slug": "reddit-vibecoding-booking-blocked",
      "creator": {
        "claimed": false,
        "handle": null,
        "displayName": null,
        "payoutShare": 0.3,
        "payoutEmail": null,
        "claimCode": null,
        "verifiedAt": null,
        "verificationMethod": null
      }
    }
  ],
  "_creatorProgram": {
    "name": "BuildOff Creator Program",
    "status": "launching",
    "payoutShare": 0.3,
    "note": "Creators who claim or submit a test earn 30% of affiliate commission attributed to that test page via sub-ID. Program launching — no earnings promised. See /creator-program/."
  }
}