{
  "apiVersion": 1,
  "generatedAt": "2026-09-14T00:00:00+00:00",
  "slug": "link-crawler",
  "botId": "link-crawler",
  "configEncoding": "plain",
  "configDigest": "dbb0b3bf63d1504615d183b9db980ede7ab1fdeb950807136cfa9c3d83065084",
  "configBytes": 2358,
  "project": {
    "name": "Link Crawler",
    "settings": {
      "urls": [
        "${target_url}"
      ],
      "headless": false,
      "humanoidEnabled": false,
      "proxyEnabled": false,
      "captchaEnabled": false,
      "matchTimezoneAndLocale": false,
      "block_resources": false,
      "get_urls_from_sitemap": false,
      "respect_robots_txt": false,
      "skip_initial_navigation": false,
      "variables": [
        {
          "name": "target_url",
          "value": "https://rtila.com",
          "required": true,
          "is_list": false
        }
      ]
    },
    "datasets": {
      "page_metadata": {
        "item_selector": "css=html",
        "properties": [
          {
            "name": "meta_title",
            "type": "text",
            "selector": "css=head title",
            "transformations": [
              {
                "type": "trim"
              }
            ]
          },
          {
            "name": "url",
            "type": "text",
            "selector": "variable=last_navigation_url"
          },
          {
            "name": "status",
            "type": "text",
            "selector": "variable=last_http_status"
          },
          {
            "name": "content_type",
            "type": "text",
            "selector": "variable=last_content_type"
          },
          {
            "name": "charset",
            "type": "text",
            "selector": "variable=last_charset"
          },
          {
            "name": "content_length",
            "type": "text",
            "selector": "variable=last_content_length"
          },
          {
            "name": "server",
            "type": "text",
            "selector": "variable=last_server"
          },
          {
            "name": "cache_control",
            "type": "text",
            "selector": "variable=last_cache_control"
          },
          {
            "name": "x_robots_tag",
            "type": "text",
            "selector": "variable=last_x_robots_tag"
          }
        ],
        "required_fields": [
          "meta_title"
        ]
      }
    },
    "commands": [
      {
        "command": "crawl_links",
        "params": {
          "concurrency": 7,
          "delay": 1000,
          "link_type": "internal",
          "max_depth": 3,
          "max_urls": 999,
          "do": [
            {
              "command": "extract_data",
              "params": {
                "dataset": "page_metadata"
              }
            }
          ]
        }
      }
    ],
    "authors": []
  }
}