From b6e61caa298ea1ff50cb508b6b739ca7b50a25e7 Mon Sep 17 00:00:00 2001 From: Alexander Bocken Date: Tue, 30 Jul 2024 14:08:41 +0200 Subject: [PATCH] no crawling of /static/ --- static/robots.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/static/robots.txt b/static/robots.txt index f4625e6..3319f20 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,2 +1,5 @@ User-agent: GPTBot Disallow: / + +User-agent: * +Disallow: /static/