{
 "about": "Apartee's public accuracy test (board plan T2). Fictional product docs in knowledge/. Each case is labelled before the run: answer (with words a correct answer must contain), not_in_kb, cautious (NOT_IN_KB or a CHECK warning), none (nothing to answer).",
 "cases": [
  {
   "name": "Azure Repos supported",
   "category": "Integrations",
   "context": [],
   "latest": "We're on Azure DevOps, Azure Repos. Can you scan our code there?",
   "expect": "answer",
   "mention": [
    "supported",
    "yes"
   ],
   "forbid": [
    "azure repos is not",
    "azure repos isn't",
    "azure repos aren't",
    "no, "
   ]
  },
  {
   "name": "Bitbucket Data Center not supported",
   "category": "Integrations",
   "context": [],
   "latest": "We host Bitbucket Data Center ourselves. Is that supported?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported"
   ]
  },
  {
   "name": "GitHub Enterprise Server version",
   "category": "Integrations",
   "context": [],
   "latest": "We run GitHub Enterprise Server 3.8. Will that work?",
   "expect": "answer",
   "mention": [
    "3.10"
   ]
  },
  {
   "name": "GitLab self-managed",
   "category": "Integrations",
   "context": [],
   "latest": "Do you support GitLab self-managed? We're on 16.4.",
   "expect": "answer",
   "mention": [
    "supported",
    "yes"
   ],
   "forbid": [
    "not supported",
    "isn't supported",
    "aren't supported",
    "doesn't support",
    "don't support",
    "unsupported",
    "not available"
   ]
  },
  {
   "name": "ACR registry",
   "category": "Integrations",
   "context": [],
   "latest": "Our images live in Azure Container Registry. Can you scan them there?",
   "expect": "answer",
   "mention": [
    "supported",
    "yes"
   ],
   "forbid": [
    "acr is not",
    "acr isn't",
    "container registry is not",
    "container registry isn't",
    "no, "
   ]
  },
  {
   "name": "Quay registry not supported",
   "category": "Integrations",
   "context": [],
   "latest": "What about Red Hat Quay? That's where most of our images are.",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported"
   ]
  },
  {
   "name": "TeamCity via CLI",
   "category": "Integrations",
   "context": [],
   "latest": "We use TeamCity for CI. Is there a plugin?",
   "expect": "answer",
   "mention": [
    "cli",
    "nw scan"
   ]
  },
  {
   "name": "ServiceNow not supported",
   "category": "Integrations",
   "context": [],
   "latest": "Can findings create tickets in ServiceNow?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported"
   ]
  },
  {
   "name": "Jira Data Center",
   "category": "Integrations",
   "context": [],
   "latest": "We're on Jira Data Center, not cloud. Does the ticketing integration work?",
   "expect": "answer",
   "mention": [
    "supported",
    "yes"
   ],
   "forbid": [
    "not supported",
    "isn't supported",
    "aren't supported",
    "doesn't support",
    "don't support",
    "unsupported",
    "not available"
   ]
  },
  {
   "name": "Splunk export",
   "category": "Integrations",
   "context": [],
   "latest": "How do we get findings into Splunk?",
   "expect": "answer",
   "mention": [
    "event collector",
    "hec"
   ]
  },
  {
   "name": "QRadar not supported",
   "category": "Integrations",
   "context": [],
   "latest": "Our SOC uses QRadar. Can you send findings there?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported",
    "api",
    "s3"
   ]
  },
  {
   "name": "Visual Studio plugin",
   "category": "Integrations",
   "context": [],
   "latest": "Do you have a plugin for Visual Studio? Not VS Code, the full Visual Studio.",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported"
   ]
  },
  {
   "name": "Implicit: statement about stack",
   "category": "Integrations",
   "context": [],
   "latest": "We're mostly GitLab CI with a bit of Jenkins.",
   "expect": "answer",
   "mention": [
    "supported"
   ],
   "forbid": [
    "not supported",
    "isn't supported",
    "aren't supported",
    "doesn't support",
    "don't support",
    "unsupported",
    "not available"
   ]
  },
  {
   "name": "C++ not supported",
   "category": "Languages and IaC",
   "context": [],
   "latest": "Half our codebase is C++. Do you scan that?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported"
   ]
  },
  {
   "name": "Swift beta",
   "category": "Languages and IaC",
   "context": [],
   "latest": "What about Swift for our iOS app?",
   "expect": "answer",
   "mention": [
    "beta"
   ]
  },
  {
   "name": "Reachability for Go",
   "category": "Languages and IaC",
   "context": [],
   "latest": "Does the reachability analysis work for Go?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported",
    "java",
    "python"
   ]
  },
  {
   "name": "Reachability languages",
   "category": "Languages and IaC",
   "context": [],
   "latest": "Which languages have reachability?",
   "expect": "answer",
   "mention": [
    "java",
    "python"
   ]
  },
  {
   "name": "Bicep not supported",
   "category": "Languages and IaC",
   "context": [],
   "latest": "We write our infrastructure in Bicep. Is that covered?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported"
   ]
  },
  {
   "name": "Terraform plan files",
   "category": "Languages and IaC",
   "context": [],
   "latest": "Do you scan Terraform plan files or only the HCL?",
   "expect": "answer",
   "mention": [
    "plan"
   ],
   "forbid": [
    "plan files are not",
    "plan files aren't",
    "only the hcl",
    "only hcl",
    "no, "
   ]
  },
  {
   "name": "Windows containers",
   "category": "Languages and IaC",
   "context": [],
   "latest": "Some of our services run as Windows containers. Can you scan those images?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported"
   ]
  },
  {
   "name": "Package managers pnpm",
   "category": "Languages and IaC",
   "context": [],
   "latest": "We moved to pnpm. Is it supported for dependency scanning?",
   "expect": "answer",
   "mention": [
    "supported",
    "yes"
   ],
   "forbid": [
    "not supported",
    "isn't supported",
    "aren't supported",
    "doesn't support",
    "don't support",
    "unsupported",
    "not available"
   ]
  },
  {
   "name": "SOC 2 report",
   "category": "Security and compliance",
   "context": [],
   "latest": "Do you have a SOC 2 Type II report we can review?",
   "expect": "answer",
   "mention": [
    "nda"
   ]
  },
  {
   "name": "Is code stored",
   "category": "Security and compliance",
   "context": [],
   "latest": "Do you keep a copy of our source code after the scan?",
   "expect": "answer",
   "mention": [
    "deleted",
    "never",
    "snippet"
   ]
  },
  {
   "name": "EU residency",
   "category": "Security and compliance",
   "context": [],
   "latest": "Can our data stay in the EU?",
   "expect": "answer",
   "mention": [
    "frankfurt"
   ]
  },
  {
   "name": "Change region later",
   "category": "Security and compliance",
   "context": [],
   "latest": "If we start in the US region, can we move to the EU later?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported",
    "can't",
    "cannot"
   ]
  },
  {
   "name": "SCIM plan",
   "category": "Security and compliance",
   "context": [],
   "latest": "Do you support SCIM provisioning?",
   "expect": "answer",
   "mention": [
    "enterprise"
   ]
  },
  {
   "name": "SSO with Entra ID",
   "category": "Security and compliance",
   "context": [],
   "latest": "We use Entra ID. Does SSO work with it?",
   "expect": "answer",
   "mention": [
    "saml",
    "yes"
   ],
   "forbid": [
    "not supported",
    "isn't supported",
    "aren't supported",
    "doesn't support",
    "don't support",
    "unsupported",
    "not available"
   ]
  },
  {
   "name": "MFA on every plan",
   "category": "Security and compliance",
   "context": [],
   "latest": "Can we enforce MFA for all users?",
   "expect": "answer",
   "mention": [
    "every plan",
    "all plans",
    "yes",
    "require"
   ],
   "forbid": [
    "not supported",
    "isn't supported",
    "aren't supported",
    "doesn't support",
    "don't support",
    "unsupported",
    "not available"
   ]
  },
  {
   "name": "Retention after closing",
   "category": "Security and compliance",
   "context": [],
   "latest": "What happens to our data if we cancel?",
   "expect": "answer",
   "mention": [
    "30 days"
   ]
  },
  {
   "name": "Encryption",
   "category": "Security and compliance",
   "context": [],
   "latest": "How is our data encrypted?",
   "expect": "answer",
   "mention": [
    "aes"
   ]
  },
  {
   "name": "Air-gapped not supported",
   "category": "Deployment",
   "context": [],
   "latest": "We need a fully air-gapped install. Is that possible?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported"
   ]
  },
  {
   "name": "Self-hosted scanner plan",
   "category": "Deployment",
   "context": [],
   "latest": "Can we run the scanner inside our own network?",
   "expect": "answer",
   "mention": [
    "enterprise",
    "self-hosted"
   ]
  },
  {
   "name": "On-prem console",
   "category": "Deployment",
   "context": [],
   "latest": "Can we host the whole platform, including the console, on-prem?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported",
    "saas"
   ]
  },
  {
   "name": "Scanner network needs",
   "category": "Deployment",
   "context": [],
   "latest": "What does the self-hosted scanner need from our firewall?",
   "expect": "answer",
   "mention": [
    "outbound",
    "https"
   ]
  },
  {
   "name": "Team price",
   "category": "Pricing and plans",
   "context": [],
   "latest": "How much is the Team plan?",
   "expect": "answer",
   "mention": [
    "35"
   ]
  },
  {
   "name": "Who counts as a developer",
   "category": "Pricing and plans",
   "context": [],
   "latest": "How do you count developers for pricing?",
   "expect": "answer",
   "mention": [
    "90 days",
    "committed"
   ]
  },
  {
   "name": "Trial length",
   "category": "Pricing and plans",
   "context": [],
   "latest": "Is there a trial, and do you need a card?",
   "expect": "answer",
   "mention": [
    "21"
   ]
  },
  {
   "name": "Annual discount",
   "category": "Pricing and plans",
   "context": [],
   "latest": "Is there a discount if we pay annually?",
   "expect": "answer",
   "mention": [
    "15"
   ]
  },
  {
   "name": "Repo Posture plan",
   "category": "Pricing and plans",
   "context": [],
   "latest": "Is Repo Posture included in the Team plan?",
   "expect": "answer",
   "mention": [
    "business"
   ]
  },
  {
   "name": "Deduplication",
   "category": "Features",
   "context": [],
   "latest": "We get the same finding a hundred times across branches. How do you deal with duplicates?",
   "expect": "answer",
   "mention": [
    "fingerprint",
    "group",
    "once",
    "same"
   ]
  },
  {
   "name": "Custom secret patterns",
   "category": "Features",
   "context": [],
   "latest": "Can we add our own patterns, like a Norwegian national ID number?",
   "expect": "answer",
   "mention": [
    "regular expression",
    "regex",
    "business",
    "pattern"
   ]
  },
  {
   "name": "SBOM formats",
   "category": "Features",
   "context": [],
   "latest": "Can you export an SBOM? Which formats?",
   "expect": "answer",
   "mention": [
    "cyclonedx",
    "spdx"
   ]
  },
  {
   "name": "Block builds",
   "category": "Features",
   "context": [],
   "latest": "Can you fail a pull request when there's a new critical finding?",
   "expect": "answer",
   "mention": [
    "polic"
   ]
  },
  {
   "name": "Auto-fix on GitLab (scoped)",
   "category": "Scoped to the feature shown",
   "context": [],
   "latest": "Does this work with GitLab too?",
   "expect": "answer",
   "mention": [
    "github",
    "not",
    "n't",
    "unsupported"
   ],
   "now_showing": "Auto-fix pull requests"
  },
  {
   "name": "Repo Posture on Azure DevOps (scoped)",
   "category": "Scoped to the feature shown",
   "context": [],
   "latest": "And does this part work on Azure DevOps?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported"
   ],
   "now_showing": "Repo Posture"
  },
  {
   "name": "Reachability for Go (scoped)",
   "category": "Scoped to the feature shown",
   "context": [],
   "latest": "Would this work for our Go services?",
   "expect": "answer",
   "mention": [
    "not",
    "n't",
    "unsupported",
    "only"
   ],
   "now_showing": "Reachability"
  },
  {
   "name": "Repo Posture on GitLab self-managed (scoped)",
   "category": "Scoped to the feature shown",
   "context": [],
   "latest": "Does this work on GitLab self-managed?",
   "expect": "answer",
   "mention": [
    "supported",
    "yes",
    "works"
   ],
   "now_showing": "Repo Posture",
   "forbid": [
    "not supported",
    "isn't supported",
    "aren't supported",
    "doesn't support",
    "don't support",
    "unsupported",
    "not available"
   ]
  },
  {
   "name": "FedRAMP",
   "category": "Not in the docs",
   "context": [],
   "latest": "Are you FedRAMP authorized?",
   "expect": "not_in_kb"
  },
  {
   "name": "HIPAA BAA",
   "category": "Not in the docs",
   "context": [],
   "latest": "Will you sign a HIPAA BAA?",
   "expect": "not_in_kb"
  },
  {
   "name": "DAST",
   "category": "Not in the docs",
   "context": [],
   "latest": "Do you also do dynamic testing, DAST, against running apps?",
   "expect": "not_in_kb"
  },
  {
   "name": "Perforce",
   "category": "Not in the docs",
   "context": [],
   "latest": "Do you support Perforce Helix Core?",
   "expect": "not_in_kb"
  },
  {
   "name": "Reseller programme",
   "category": "Not in the docs",
   "context": [],
   "latest": "Do you have a reseller or partner programme?",
   "expect": "not_in_kb"
  },
  {
   "name": "Repo Posture on GHES 3.9",
   "category": "Integrations",
   "context": [],
   "latest": "Does Repo Posture support GitHub Enterprise Server 3.9?",
   "expect": "answer",
   "mention": [
    "3.10"
   ]
  },
  {
   "name": "Small talk",
   "category": "Nothing to answer",
   "context": [],
   "latest": "Sorry, you froze for a second there. Can you hear me now?",
   "expect": "none"
  },
  {
   "name": "Rep asks the question",
   "category": "Nothing to answer",
   "context": [],
   "latest": "So which repositories would you want to start with?",
   "expect": "none",
   "latest_who": "me"
  },
  {
   "name": "Logistics",
   "category": "Nothing to answer",
   "context": [],
   "latest": "Let me share my screen and pull up our pipeline.",
   "expect": "none"
  },
  {
   "name": "Thanks",
   "category": "Nothing to answer",
   "context": [],
   "latest": "Great, that's really helpful, thanks.",
   "expect": "none"
  }
 ],
 "label_changes": [
  "13 Nov 2026, after the first run: 'Azure Repos supported' and 'ACR registry' failed correct answers that also said another product isn't supported; their forbidden words now name the product asked about. 'Reachability for Go (scoped)' failed a correct answer worded 'only Java, JavaScript, TypeScript and Python'; 'only' now counts. No other case was changed.",
  "13 Nov 2026, after the second run: 'Terraform plan files' failed a correct answer that added 'Bicep is not supported'; its forbidden words now name plan files. 'Repo Posture on GHES 3.9' was labelled 'cautious', but the docs say GitHub Enterprise Server before 3.10 isn't supported, so a direct answer naming 3.10 is the correct one; it's now an 'answer' case in Integrations."
 ]
}