{"id":"robots-longest-prefix-match","text":"`RobotsParser.is_allowed` resolves conflicting allow/disallow rules by selecting the rule with the longest matching path prefix, falling back from agent-specific rules to `*`.","truth_value":"IN","source":"entries/2026/06/05/web-crawler-web_crawler.md","source_url":"","source_hash":"","justifications":[],"dependents":[],"metadata":{},"created_at":"","updated_at":"","reviewed_at":"","verified_at":"","retracted_at":"","explanation":{"steps":[{"node":"robots-longest-prefix-match","truth_value":"IN","reason":"premise"}]}}