{
  "openapi": "3.1.0",
  "info": {
    "title": "robots.txt evaluation API",
    "version": "1.0.0",
    "summary": "Tests one path against a pasted robots.txt for each user agent, under RFC 9309.",
    "description": "Send a robots.txt as text, a path and an optional list of user agents. The answer says, for each agent, whether the rules allow the path, which group applied and which line decided. It runs the reader and the matcher of answer-crawler-check 0.1.2, the code of the robots.txt sandbox. It never fetches a URL. No key is needed."
  },
  "externalDocs": {
    "description": "Docs, limits and terms",
    "url": "https://synapsereality.io/tools/robots-sandbox/api/"
  },
  "servers": [
    {
      "url": "https://synapsereality.io"
    }
  ],
  "paths": {
    "/api/robots/evaluate": {
      "post": {
        "operationId": "evaluateRobotsTxt",
        "summary": "Test a path against a pasted robots.txt",
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/Request"
              }
            }
          }
        },
        "responses": {
          "200": {
            "description": "One result for each user agent, in the order sent.",
            "headers": {
              "X-RateLimit-Limit": {
                "description": "The hourly limit in force for one address.",
                "schema": {
                  "type": "integer"
                }
              },
              "X-RateLimit-Remaining": {
                "description": "The requests left to this address in the current hour.",
                "schema": {
                  "type": "integer"
                }
              }
            },
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Answer"
                }
              }
            }
          },
          "400": {
            "description": "The body is not UTF-8 JSON, a field is missing, has the wrong type or is past its limit, or an unknown field was sent.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Error"
                }
              }
            }
          },
          "405": {
            "description": "Any method but POST and OPTIONS.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Error"
                }
              }
            }
          },
          "413": {
            "description": "The body is over 1000000 bytes, robots_txt is over 512000 bytes, or the file and path are past the work or time budget. A body over 1 MiB is refused by the web server in front of the API, with an HTML page.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Error"
                }
              }
            }
          },
          "429": {
            "description": "More than the hourly limit from one address, which for IPv6 means one /64. The limit is 60, and the X-RateLimit-Limit header gives the one in force. Retry-After gives the seconds to wait. More than ten requests a minute after a burst of ten gets an HTML 429 from the web server in front of the API.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Error"
                }
              }
            }
          },
          "500": {
            "description": "Something failed at our end.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Error"
                }
              }
            }
          },
          "503": {
            "description": "The evaluator or the request counter is busy or not available. Retry-After may give the seconds to wait.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Error"
                }
              }
            }
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "Request": {
        "type": "object",
        "additionalProperties": false,
        "required": [
          "robots_txt",
          "path"
        ],
        "properties": {
          "robots_txt": {
            "type": "string",
            "description": "The file as text. At most 512000 bytes of UTF-8."
          },
          "path": {
            "type": "string",
            "maxLength": 2048,
            "pattern": "^[^\\u0000-\\u001f\\u007f]*$",
            "description": "A path such as /blog/post, or a full URL whose path and query are tested. No control characters."
          },
          "user_agents": {
            "type": "array",
            "minItems": 1,
            "maxItems": 50,
            "items": {
              "type": "string",
              "minLength": 1,
              "maxLength": 200,
              "pattern": "^[^\\u0000-\\u001f\\u007f]*[^\\s\\u0000-\\u001f\\u007f][^\\u0000-\\u001f\\u007f]*$",
              "description": "Not blank, and no control characters."
            },
            "description": "Names to test. Left out, the API tests the 9 AI crawler names the sandbox tests.",
            "default": [
              "ClaudeBot",
              "Claude-SearchBot",
              "Claude-User",
              "GPTBot",
              "OAI-SearchBot",
              "ChatGPT-User",
              "Google-Extended",
              "PerplexityBot",
              "Perplexity-User"
            ]
          }
        }
      },
      "Answer": {
        "type": "object",
        "required": [
          "ok",
          "path",
          "groups",
          "truncated",
          "results",
          "matcher"
        ],
        "properties": {
          "ok": {
            "const": true
          },
          "path": {
            "type": "string",
            "description": "The path that was tested, written as a browser would request it."
          },
          "groups": {
            "type": "integer",
            "description": "How many User-agent names the file holds."
          },
          "truncated": {
            "type": "boolean",
            "description": "True when the file holds more than 20,000 rules. Only the first 20,000 were stored. A name seen before that point keeps the rules stored under it, as RFC 9309 merges the groups of one agent. A name first seen after it still counts as a group with no rules."
          },
          "results": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/Result"
            }
          },
          "matcher": {
            "type": "object",
            "properties": {
              "package": {
                "type": "string"
              },
              "version": {
                "type": "string"
              }
            }
          }
        }
      },
      "Result": {
        "type": "object",
        "required": [
          "user_agent",
          "token",
          "allowed",
          "group",
          "rule"
        ],
        "properties": {
          "user_agent": {
            "type": "string",
            "description": "The name as sent."
          },
          "token": {
            "type": "string",
            "description": "Its product token in lower case, which groups are matched on."
          },
          "allowed": {
            "type": "boolean"
          },
          "group": {
            "type": [
              "string",
              "null"
            ],
            "description": "The token of the group that applied, \"*\", or null when no group applies."
          },
          "rule": {
            "oneOf": [
              {
                "type": "null"
              },
              {
                "type": "object",
                "required": [
                  "line",
                  "text",
                  "allow",
                  "pattern",
                  "truncated"
                ],
                "properties": {
                  "line": {
                    "type": "integer",
                    "description": "The line number in robots_txt, counting from 1."
                  },
                  "text": {
                    "type": "string",
                    "description": "The rule as the sandbox prints it, such as \"Disallow: /private/\"."
                  },
                  "allow": {
                    "type": "boolean"
                  },
                  "pattern": {
                    "type": "string",
                    "description": "The pattern with non-ASCII characters percent-encoded. Cut at 1024 characters."
                  },
                  "truncated": {
                    "type": "boolean",
                    "description": "True when the pattern was cut."
                  }
                }
              }
            ]
          }
        }
      },
      "Error": {
        "type": "object",
        "required": [
          "ok",
          "code",
          "error"
        ],
        "properties": {
          "ok": {
            "const": false
          },
          "code": {
            "type": "string",
            "enum": [
              "bad_json",
              "bad_input",
              "too_large",
              "too_much_work",
              "rate_limited",
              "busy",
              "unavailable",
              "method_not_allowed",
              "internal"
            ]
          },
          "error": {
            "type": "string",
            "description": "One sentence that says what went wrong."
          }
        }
      }
    }
  }
}
