Update robots.txt (#458)

MagicLex · web-flow · commit 39eddfd5502a · 2025-04-01T14:24:02.000+02:00
changed the principle to disallow everything BUT latest release. Will need to be updated every release.
diff --git a/docs/robots.txt b/docs/robots.txt
@@ -1,4 +1,10 @@
 User-agent: *
-Disallow:
+Disallow: /
+Allow: /4.1/
+Allow: /latest/
 
-Sitemap: https://docs.hopsworks.ai/3.2/sitemap.xml
+Allow: /hopsworks-api/4.1/
+Allow: /hopsworks-api/latest/
+
+Allow: /hopsworks-api/4.1/javadoc/
+Allow: /hopsworks-api/latest/javadoc/
