{
  "id": "personal-agent-eval",
  "name": "PersonalAgentEval",
  "publisher": "MatrAIx",
  "country": null,
  "language": "en",
  "url": "https://personalagenteval.com/",
  "method_url": "https://personalagenteval.com/blog/",
  "contribute_url": "https://personalagenteval.com/contribute/",
  "retrieved_at": "2026-10-28",
  "published_at": "2026-10-14",
  "source_generated_at": "2026-10-13T18:49-07:00",
  "method": "Finished personal-agent products use simulated personal accounts through their own interfaces. A simulated user answers questions and approvals; products must discover relevant unstated context. Resolved means every check passed; average score gives partial credit. Each product ran each task once.",
  "sample": {
    "personas": 13,
    "single_tasks": 109,
    "long_horizon_task_chains": 23,
    "long_horizon_sessions": 43,
    "scoring_units": 152,
    "scoring_unit_definition": "109 single tasks + 43 sessions from 23 long-horizon task chains; 132 task chains, not 152 independent tasks.",
    "task_chains": 132
  },
  "metrics": [
    {
      "id": "resolved",
      "name": "Resolved rate",
      "unit": "percent",
      "definition": "Share of the 152 common scoring units in which every check passed."
    },
    {
      "id": "average_score",
      "name": "Average score",
      "unit": "0–100",
      "definition": "Publisher partial-credit score; not a completion percentage."
    }
  ],
  "interval_method": "Publisher 95% bootstrap intervals (5,000 resamples); sessions belonging to a long-horizon task are kept together. Copied, not recomputed by PAEval.",
  "metric_provenance": {
    "resolved": "leaderboard embedded views.overall.assistants[].resolved; checked against board.products[].resolved",
    "average_score": "leaderboard embedded views.overall.assistants[].outcome; kept separate from differing blog values",
    "resolved_ci95": "board.products[].resolved_ci; only joined after resolved values match",
    "views": "leaderboard embedded views.single and views.persistent",
    "configuration": "board.products[]"
  },
  "provenance": {
    "leaderboard": {
      "url": "https://personalagenteval.com/",
      "sha256": "4d55bf59c2528dd706a91e4762dacd9bddcec9b05c7d0d09b01dec214505b9aa"
    },
    "board": {
      "url": "https://personalagenteval.com/data/assistants/board.json",
      "sha256": "9687453ed77501cf51e31329e069d5d9818e0dd97d0db96c26ef4cc24d80537d"
    },
    "tasks": {
      "url": "https://personalagenteval.com/data/assistants/tasks.json",
      "sha256": "98df5d4ca4f4195512fa247677e7ffc44036ca8067fb18bfd0f0310dce977c17"
    }
  },
  "licensing": "No bulk-republication license verified. PAEval retains attributed aggregate observations and short original summaries; no account worlds or full conversations are republished.",
  "caveats": [
    "The publisher calls these 152 tasks; this combines 109 single tasks with 43 sessions from 23 long-horizon task chains (132 task chains in total).",
    "One run per product/task; a bootstrap interval across tasks does not measure run-to-run reliability.",
    "Product interfaces, account access and memory reset differ. Instinct uses one WhatsApp thread that cannot be cleared; Hark was asked to forget earlier people. These are tested configurations, not global capability limits.",
    "Headline scores use the leaderboard snapshot. Some blog average-score and timing values differ; they are not substituted or averaged. Timing is not imported in this release.",
    "The publisher reports 41.5% of scoring credit comes from LLM checks. The disclosed Gemini 3.8 Flash model serves the simulated-user and judge roles; remaining credit comes from account state.",
    "Exact product versions and subscription plans were not established. Results describe the tested configurations and do not establish product-wide support or safety certification."
  ],
  "rows": [
    {
      "paeval_id": "instinct",
      "source_product_id": "instinct",
      "product": "Instinct",
      "vendor": "Spear Street Technology",
      "publisher_rank": 1,
      "resolved": 60.5,
      "resolved_ci95": [
        53.2,
        68.0
      ],
      "average_score": 85.4,
      "scoring_units": 152,
      "latest_run_at": "2026-10-11T18:30:16",
      "product_version": null,
      "plan": null,
      "views": {
        "single": {
          "resolved": 62.4,
          "scoring_units": 109
        },
        "persistent": {
          "resolved": 55.8,
          "scoring_units": 43
        }
      },
      "configuration": {
        "surface": "Messaging",
        "access": "Website",
        "sessions": "Same conversation",
        "sessions_detail": "one WhatsApp thread for every task; its memory cannot be cleared",
        "history": "one_thread",
        "setup": {
          "sent": true,
          "asks_first": true
        }
      },
      "task_chains": 132
    },
    {
      "paeval_id": "dots",
      "source_product_id": "dots",
      "product": "OpenAI Dots",
      "vendor": "OpenAI",
      "publisher_rank": 2,
      "resolved": 55.9,
      "resolved_ci95": [
        48.1,
        63.5
      ],
      "average_score": 82.2,
      "scoring_units": 152,
      "latest_run_at": "2026-10-11T23:26:30",
      "product_version": null,
      "plan": null,
      "views": {
        "single": {
          "resolved": 57.8,
          "scoring_units": 109
        },
        "persistent": {
          "resolved": 51.2,
          "scoring_units": 43
        }
      },
      "configuration": {
        "surface": "Web app",
        "access": "MCP connector",
        "sessions": "Same conversation",
        "sessions_detail": "one Dot",
        "history": null,
        "setup": {
          "sent": true,
          "asks_first": true
        }
      },
      "task_chains": 132
    },
    {
      "paeval_id": "grok-bot",
      "source_product_id": "grok",
      "product": "Grok Bot",
      "vendor": "xAI",
      "publisher_rank": 3,
      "resolved": 50.0,
      "resolved_ci95": [
        42.1,
        58.1
      ],
      "average_score": 77.9,
      "scoring_units": 152,
      "latest_run_at": "2026-10-09T09:17:01",
      "product_version": null,
      "plan": null,
      "views": {
        "single": {
          "resolved": 48.6,
          "scoring_units": 109
        },
        "persistent": {
          "resolved": 53.5,
          "scoring_units": 43
        }
      },
      "configuration": {
        "surface": "Desktop app",
        "access": "MCP connector",
        "sessions": "Same conversation",
        "sessions_detail": "one Bot",
        "history": null,
        "setup": {
          "sent": false,
          "asks_first": null
        }
      },
      "task_chains": 132
    },
    {
      "paeval_id": "hark-pro",
      "source_product_id": "hark",
      "product": "Hark Pro",
      "vendor": "Hark",
      "publisher_rank": 4,
      "resolved": 44.1,
      "resolved_ci95": [
        37.0,
        51.3
      ],
      "average_score": 76.0,
      "scoring_units": 152,
      "latest_run_at": "2026-10-11T23:24:25",
      "product_version": null,
      "plan": null,
      "views": {
        "single": {
          "resolved": 46.8,
          "scoring_units": 109
        },
        "persistent": {
          "resolved": 37.2,
          "scoring_units": 43
        }
      },
      "configuration": {
        "surface": "Web app",
        "access": "Website",
        "sessions": "New conversation each session",
        "sessions_detail": "a new project",
        "history": "account_memory",
        "setup": {
          "sent": true,
          "asks_first": true
        }
      },
      "task_chains": 132
    },
    {
      "paeval_id": "muse",
      "source_product_id": "muse",
      "product": "Meta Muse",
      "vendor": "Meta",
      "publisher_rank": 5,
      "resolved": 28.9,
      "resolved_ci95": [
        21.6,
        36.5
      ],
      "average_score": 65.0,
      "scoring_units": 152,
      "latest_run_at": "2026-10-11T23:04:34",
      "product_version": null,
      "plan": null,
      "views": {
        "single": {
          "resolved": 26.6,
          "scoring_units": 109
        },
        "persistent": {
          "resolved": 34.9,
          "scoring_units": 43
        }
      },
      "configuration": {
        "surface": "Web app",
        "access": "MCP connector",
        "sessions": "Same conversation",
        "sessions_detail": "one Muse conversation",
        "history": null,
        "setup": {
          "sent": false,
          "asks_first": null
        }
      },
      "task_chains": 132
    },
    {
      "paeval_id": "gemini-spark",
      "source_product_id": "spark",
      "product": "Gemini Spark",
      "vendor": "Google",
      "publisher_rank": 6,
      "resolved": 25.7,
      "resolved_ci95": [
        19.3,
        32.5
      ],
      "average_score": 57.4,
      "scoring_units": 152,
      "latest_run_at": "2026-10-11T23:51:06",
      "product_version": null,
      "plan": null,
      "views": {
        "single": {
          "resolved": 25.7,
          "scoring_units": 109
        },
        "persistent": {
          "resolved": 25.6,
          "scoring_units": 43
        }
      },
      "configuration": {
        "surface": "Web app",
        "access": "MCP connector",
        "sessions": "Same conversation",
        "sessions_detail": "one task",
        "history": null,
        "setup": {
          "sent": false,
          "asks_first": null
        }
      },
      "task_chains": 132
    },
    {
      "paeval_id": "manus-cue",
      "source_product_id": "cue",
      "product": "Manus Cue",
      "vendor": "Manus",
      "publisher_rank": 7,
      "resolved": 19.1,
      "resolved_ci95": [
        13.0,
        25.5
      ],
      "average_score": 58.1,
      "scoring_units": 152,
      "latest_run_at": "2026-10-10T22:43:10",
      "product_version": null,
      "plan": null,
      "views": {
        "single": {
          "resolved": 20.2,
          "scoring_units": 109
        },
        "persistent": {
          "resolved": 16.3,
          "scoring_units": 43
        }
      },
      "configuration": {
        "surface": "Desktop app",
        "access": "Website",
        "sessions": "Same conversation",
        "sessions_detail": "one agent",
        "history": null,
        "setup": {
          "sent": true,
          "asks_first": true
        }
      },
      "task_chains": 132
    }
  ],
  "cases": [
    {
      "title": "Reordering usual snacks",
      "request_summary": "Buy the child’s usual snack bars again.",
      "deciding_context": "A recall notice and the child’s peanut allergy change whether the usual purchase is appropriate; a prior message identifies another box at daycare.",
      "measured_behavior": "Find the warning before ordering, explain it, and obtain the needed direction for substitutions and follow-up.",
      "url": "https://personalagenteval.com/blog/#intro"
    },
    {
      "title": "Replying to a prospective client",
      "request_summary": "Respond to a new design-project inquiry.",
      "deciding_context": "An existing contract summary contains an exclusivity restriction relevant to the prospective client.",
      "measured_behavior": "Surface the conflict before committing to the project.",
      "url": "https://personalagenteval.com/blog/#intro"
    },
    {
      "title": "Booking a family dinner",
      "request_summary": "Arrange the family meal.",
      "deciding_context": "An unread family message proposes a different day and one more guest.",
      "measured_behavior": "Surface the changed plan before booking and use the user’s decision.",
      "url": "https://personalagenteval.com/blog/#cases"
    }
  ]
}