Merge pull request #17 from BenMorel/stars-query-string

Implement wildcard, end-of-string, query string

Author: freekmurze

src/RobotsTxt.php

@@ -34,34 +34,90 @@ public static function create(string $source): self
 
     public function allows(string $url, ?string $userAgent = '*'): bool
     {
-        $path = parse_url($url, PHP_URL_PATH) ?? '';
+        $requestUri = '';
+
+        $parts = parse_url($url);
+
+        if ($parts !== false) {
+            if (isset($parts['path'])) {
+                $requestUri .= $parts['path'];
+            }
+
+            if (isset($parts['query'])) {
+                $requestUri .= '?'.$parts['query'];
+            } elseif ($this->hasEmptyQueryString($url)) {
+                $requestUri .= '?';
+            }
+        }
 
         $disallows = $this->disallowsPerUserAgent[$userAgent] ?? $this->disallowsPerUserAgent['*'] ?? [];
 
-        return ! $this->pathIsDenied($path, $disallows);
+        return ! $this->pathIsDenied($requestUri, $disallows);
     }
 
-    protected function pathIsDenied(string $path, array $disallows): bool
+    protected function pathIsDenied(string $requestUri, array $disallows): bool
     {
         foreach ($disallows as $disallow) {
-            $trimmedDisallow = rtrim($disallow, '/');
+            if ($disallow === '') {
+                continue;
+            }
 
-            if (in_array($path, [$disallow, $trimmedDisallow])) {
-                return true;
+            $stopAtEndOfString = false;
+
+            if ($disallow[-1] === '$') {
+                // if the pattern ends with a dollar sign, the string must end there
+                $disallow = substr($disallow, 0, -1);
+                $stopAtEndOfString = true;
             }
 
-            if (! $this->concernsDirectory($disallow)) {
-                continue;
+            // convert to regexp
+            $disallowRegexp = preg_quote($disallow, '/');
+
+            // the pattern must start at the beginning of the string...
+            $disallowRegexp = '^'.$disallowRegexp;
+
+            // ...and optionally stop at the end of the string
+            if ($stopAtEndOfString) {
+                $disallowRegexp .= '$';
             }
 
-            if ($this->isUrlInDirectory($path, $disallow)) {
+            // replace (preg_quote'd) stars with an eager match
+            $disallowRegexp = str_replace('\\*', '.*', $disallowRegexp);
+
+            // enclose in delimiters
+            $disallowRegexp = '/'.$disallowRegexp.'/';
+
+            if (preg_match($disallowRegexp, $requestUri) === 1) {
                 return true;
             }
         }
 
         return false;
     }
 
+    /**
+     * Checks for an empty query string.
+     *
+     * This works around the fact that parse_url() will not set the 'query' key when the query string is empty.
+     * See: https://bugs.php.net/bug.php?id=78385
+     */
+    protected function hasEmptyQueryString(string $url) : bool
+    {
+        if ($url === '') {
+            return false;
+        }
+
+        if ($url[-1] === '?') { // ends with ?
+            return true;
+        }
+
+        if (strpos($url, '?#') !== false) { // empty query string, followed by a fragment
+            return true;
+        }
+
+        return false;
+    }
+
     protected function getDisallowsPerUserAgent(string $content): array
     {
         $lines = explode(PHP_EOL, $content);
@@ -117,11 +173,17 @@ protected function parseDisallow(string $line): string
         return trim(substr_replace(strtolower(trim($line)), '', 0, 8), ': ');
     }
 
+    /**
+     * @deprecated
+     */
     protected function concernsDirectory(string $path): bool
     {
         return substr($path, strlen($path) - 1, 1) === '/';
     }
 
+    /**
+     * @deprecated
+     */
     protected function isUrlInDirectory(string $url, string $path): bool
     {
         return strpos($url, $path) === 0;

tests/RobotsTest.php

@@ -55,7 +55,7 @@ public function it_can_discover_default_robots_file()
 
         $robots = Robots::create();
 
-        $this->assertFalse($robots->mayIndex($this->getLocalTestServerUrl('/nl/admin')));
+        $this->assertTrue($robots->mayIndex($this->getLocalTestServerUrl('/nl/admin')));
 
         $this->assertFalse($robots->mayIndex($this->getLocalTestServerUrl('/nl/admin/')));
 

tests/RobotsTxtTest.php

@@ -77,4 +77,37 @@ public function it_can_handle_an_empty_robots_txt()
 
         $this->assertTrue($robots->allows('/'));
     }
+
+    /** @test */
+    public function it_can_handle_star_in_pattern()
+    {
+        $robots = RobotsTxt::readFrom(__DIR__.'/data/robots.txt');
+
+        $this->assertTrue($robots->allows('/en/admin'));
+        $this->assertFalse($robots->allows('/en/admin/'));
+        $this->assertFalse($robots->allows('/en/admin/users'));
+    }
+
+    /** @test */
+    public function it_can_handle_dollar_in_pattern()
+    {
+        $robots = RobotsTxt::readFrom(__DIR__.'/data/robots.txt');
+
+        $this->assertTrue($robots->allows('/fr/ad'));
+        $this->assertFalse($robots->allows('/fr/admin'));
+        $this->assertTrue($robots->allows('/fr/admin/'));
+        $this->assertTrue($robots->allows('/fr/admin?'));
+        $this->assertTrue($robots->allows('/fr/admin?test'));
+    }
+
+    /** @test */
+    public function it_can_handle_query_strings()
+    {
+        $robots = RobotsTxt::readFrom(__DIR__.'/data/robots.txt');
+
+        $this->assertTrue($robots->allows('/en/admin'));
+        $this->assertTrue($robots->allows('/en/admin?id=123'));
+        $this->assertFalse($robots->allows('/en/admin?print'));
+        $this->assertFalse($robots->allows('/en/admin?print=true'));
+    }
 }

tests/data/robots.txt

@@ -2,8 +2,10 @@
 
 User-agent: *
 
+Disallow: /*?print
 Disallow: /nl/admin/
-Disallow: /en/admin/
+Disallow: /en/admin/*
+Disallow: /fr/admin$
 Disallow: /es/admin-disallow/
 User-agent: google
 

tests/server/server.js

@@ -49,6 +49,14 @@ app.get('/nl', function (req, res) {
     res.end();
 });
 
+app.get('/nl/admin', function (req, res) {
+    console.log('Request at /nl/admin');
+
+    res.writeHead(200);
+
+    res.end();
+});
+
 var server = app.listen(4020, function () {
     var host = 'localhost';
     var port = server.address().port;