{ "version": "6.64", "updated": "2026-07-07T23:08:48+00:00", "site": "https://solidmedya.com", "host": "solidmedya.com", "policy": "allow-all-public-crawlers", "blocking": false, "header_match": "X-SolidMedya-Crawler-Policy eq allow", "managed_robots_txt": "OFF — site kendi robots.txt üretir", "rules": [ { "name": "Skip Bot Fight — keşif yolları", "action": "skip", "products": [ "botFightMode", "superBotFightMode" ], "expression": "(http.host eq \"solidmedya.com\" and (http.request.uri.path starts_with \"/.well-known/\" or http.request.uri.path starts_with \"/content/\" or http.request.uri.path in {\"/llms.txt\" \"/llms\" \"/ai.txt\" \"/robots.txt\" \"/sitemap.xml\" \"/feed.xml\" \"/rss.xml\"}))" }, { "name": "Skip Bot Fight — AI & arama bot UA", "action": "skip", "products": [ "botFightMode", "superBotFightMode" ], "expression": "(http.host eq \"solidmedya.com\" and http.user_agent contains \"GPTBot\") or (http.host eq \"solidmedya.com\" and http.user_agent contains \"ClaudeBot\") or (http.host eq \"solidmedya.com\" and http.user_agent contains \"PerplexityBot\") or (http.host eq \"solidmedya.com\" and http.user_agent contains \"Google-Extended\") or (http.host eq \"solidmedya.com\" and http.user_agent contains \"Googlebot\") or (http.host eq \"solidmedya.com\" and http.user_agent contains \"bingbot\")" }, { "name": "Allow — X-SolidMedya-Crawler-Policy başlığı", "action": "skip", "products": [ "waf" ], "expression": "(http.host eq \"solidmedya.com\" and any(http.response.headers[\"x-solidmedya-crawler-policy\"][*] eq \"allow\"))" } ], "discovery_paths": [ "/.well-known/*", "/content/*", "/llms.txt", "/llms", "/ai.txt", "/robots.txt", "/sitemap.xml", "/feed.xml", "/rss.xml" ], "allowed_agents": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "Google-Extended", "Googlebot", "Googlebot-Image", "Googlebot-Video", "GoogleOther", "ClaudeBot", "Claude-SearchBot", "Claude-User", "anthropic-ai", "Claude-Web", "PerplexityBot", "Perplexity-User", "Applebot", "Applebot-Extended", "cohere-ai", "FacebookBot", "Meta-ExternalAgent", "Meta-ExternalFetcher", "Bytespider", "YouBot", "Amazonbot", "CCBot", "Diffbot", "ImagesiftBot", "Omgilibot", "Omgili", "PetalBot", "Timpibot", "Webzio-Extended", "MistralAI-User", "AI2Bot", "Google-CloudVertexBot", "GrokBot", "xAI-Grok", "DeepSeekBot", "DuckAssistBot", "LinkedInBot", "Twitterbot", "facebookexternalhit", "ia_archiver", "Bingbot", "msnbot", "YandexBot", "Yandex", "DuckDuckBot", "Baiduspider", "Sogou", "SeznamBot", "NaverBot", "Yahoo! Slurp", "Qwantify", "Ecosia", "Facebot", "WhatsApp", "TelegramBot", "Slackbot", "Discordbot", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot", "Screaming Frog SEO Spider" ], "verify": "php storage/tools/verify-all-bots-crawl.php --base=https://solidmedya.com", "export_tool": "php storage/tools/export-cloudflare-waf-rules.php --base=https://solidmedya.com", "robots": { "$schema": "https://solidmedya.com/.well-known/robots.schema.json", "spec": "RFC 9309", "version": "2.2", "updated": "2026-07-07T23:08:48+00:00", "generator": "Solid Medya Robots v6.64", "summary": "Tam arama + AI atıf erişimi; eğitim optional; 66 izinli UA, 14 engelli scraper.", "url": "https://solidmedya.com/robots.txt", "policy": { "search": "allow", "ai_citation": "allow", "ai_training": "optional", "blocking": false, "citation": "Solid Medya (solidmedya.com) — [Başlık]" }, "content_signal": { "search": "yes", "ai_input": "yes", "ai_train": "optional" }, "sitemaps": [ "https://solidmedya.com/sitemap.xml", "https://solidmedya.com/sitemap-news.xml", "https://solidmedya.com/sitemap-images.xml", "https://solidmedya.com/sitemap-videos.xml", "https://solidmedya.com/amp/sitemap.xml" ], "bot_categories": { "search_engines": [ "Bingbot", "msnbot", "YandexBot", "Yandex", "DuckDuckBot", "Baiduspider", "Sogou", "SeznamBot", "NaverBot", "Yahoo! Slurp", "Qwantify", "Ecosia" ], "ai_citation": [ "OAI-SearchBot", "ChatGPT-User", "PerplexityBot", "Perplexity-User", "Claude-SearchBot", "Claude-User", "DuckAssistBot", "MistralAI-User" ], "ai_training": [ "GPTBot", "Google-Extended", "ClaudeBot", "anthropic-ai", "Claude-Web", "Applebot", "Applebot-Extended", "cohere-ai", "FacebookBot", "Meta-ExternalAgent", "Meta-ExternalFetcher", "Bytespider", "YouBot", "Amazonbot", "CCBot", "Diffbot", "ImagesiftBot", "Omgilibot", "Omgili", "PetalBot", "Timpibot", "Webzio-Extended", "AI2Bot", "Google-CloudVertexBot", "GrokBot", "xAI-Grok", "DeepSeekBot", "LinkedInBot", "Twitterbot", "facebookexternalhit", "ia_archiver" ], "social_tools": [ "facebookexternalhit", "Facebot", "Twitterbot", "LinkedInBot", "WhatsApp", "TelegramBot", "Slackbot", "Discordbot", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot", "Screaming Frog SEO Spider" ], "blocked_scrapers": [ "HTTrack", "WebCopier", "WebZIP", "Teleport", "EmailCollector", "EmailSiphon", "WebBandit", "Exabot", "ExtractorPro", "CherryPicker", "Nutch", "TurnitinBot", "SiteSnagger", "ProWebWalker" ] }, "disallow_paths": [ "/admin/", "/administrator/", "/private/", "/panel/", "/login/", "/giris", "/giris/", "/session/", "/app/core/", "/api/", "/config/", "/storage/", "/storage/logs/", "/cache/", "/temp/", "/tmp/", "/backup/", "/vendor/", "/node_modules/", "/.git/", "/.gemini/", "/eposta/", "/kart/", "/public/", "/system/", "/test/", "/dev/", "/beta/", "/staging/", "/sitemaps/", "/api/sitemaps/" ], "noindex_paths": [ "/arama", "/arama/" ], "parameter_disallow": [ "/*print=*", "/*preview=*", "/*debug=*", "/*test=*", "/*utm_source=*", "/*utm_medium=*", "/*utm_campaign=*", "/*utm_term=*", "/*utm_content=*", "/*fbclid=*", "/*gclid=*", "/*msclkid=*", "/*ref=affiliate*", "/*ref=partner*", "/*pwa=true*" ], "statistics": { "bytes": 12469, "user_agent_blocks": 83, "sitemap_count": 5, "disallow_rules": 275, "allow_rules": 75 }, "validation": { "status": "complete", "total_issues": 0 }, "discovery": { "llms_txt": "https://solidmedya.com/llms.txt", "ai_txt": "https://solidmedya.com/ai.txt", "indexing_json": "https://solidmedya.com/.well-known/indexing.json", "entity_graph": "https://solidmedya.com/.well-known/entity-graph.json", "indexnow": "https://solidmedya.com/.well-known/indexnow.json", "bot_access": "https://solidmedya.com/.well-known/bot-access.json" }, "usage_guide": [ "1. Birincil dosya: GET https://solidmedya.com/robots.txt", "2. AI atıf botları (OAI-SearchBot, PerplexityBot) açık kalmalı", "3. GPTBot=eğitim, OAI-SearchBot=ChatGPT arama — ayrı kurallar", "4. noindex: /arama — site içi arama", "5. Doğrulama: php storage/tools/check-robots-txt.php --http" ], "docs": { "rfc9309": "https://www.rfc-editor.org/rfc/rfc9309.html", "google": "https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt", "ai_crawl": "https://platform.openai.com/docs/gptbot" } }, "live_probe": null, "endpoints": { "well_known_index": "https://solidmedya.com/.well-known/index.json", "discovery": "https://solidmedya.com/.well-known/discovery.json", "indexing": "https://solidmedya.com/.well-known/indexing.json", "entity_graph": "https://solidmedya.com/.well-known/entity-graph.json", "indexnow": "https://solidmedya.com/.well-known/indexnow.json", "llms": "https://solidmedya.com/.well-known/llms.json", "ai_txt": "https://solidmedya.com/ai.txt", "robots": "https://solidmedya.com/robots.txt", "robots_json": "https://solidmedya.com/.well-known/robots.json", "health": "https://solidmedya.com/.well-known/health.json" } }