diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 1fa7fc13baa539..3e84f86d4dd383 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -24,11 +24,18 @@ structure of :file:`robots.txt` files, see :rfc:`9309`. .. class:: RobotFileParser(url='') This class provides methods to read, parse and answer questions about the - :file:`robots.txt` file at *url*. + :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object. + + .. versionchanged:: next + *url* parameter can be a :class:`urllib.request.Request` object. .. method:: set_url(url) - Sets the URL referring to a :file:`robots.txt` file. + Sets the URL referring to a :file:`robots.txt` file or a + :class:`urllib.request.Request` object. + + .. versionchanged:: next + *url* parameter can be a :class:`urllib.request.Request` object. .. method:: read() @@ -102,3 +109,17 @@ class:: True >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/") False + + +The following example demonstrates use of a :class:`urllib.request.Request` +object with additional user-agent headers populated:: + + >>> import urllib.robotparser + >>> import urllib.request + >>> rp = urllib.robotparser.RobotFileParser() + >>> rp.set_url(urllib.request.Request("http://www.pythontest.net/robots.txt", headers={"User-Agent": "IsraBot"})) + >>> rp.read() + >>> rp.can_fetch("*", "http://www.pythontest.net/") + True + >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/") + False diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index cd1477037e94b7..725c6e3b09e1d0 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -773,6 +773,37 @@ def testServiceUnavailable(self): self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) +class UserAgentSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + + class RobotHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.headers.get('User-Agent').startswith('Python-urllib'): + self.send_error(403, "Forbidden access") + else: + self.send_response(200) + self.end_headers() + self.wfile.write(b"User-agent: *\nDisallow:") + + def log_message(self, format, *args): + pass + + def testUserAgentFilteringSite(self): + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + file_url = url + "/document" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(robots_url) + parser.read() + self.assertTrue(parser.disallow_all) + self.assertFalse(parser.can_fetch("*", file_url)) + parser = urllib.robotparser.RobotFileParser() + parser.set_url(urllib.request.Request(robots_url, headers={'User-Agent': 'cybermapper'})) + parser.read() + self.assertFalse(parser.disallow_all) + self.assertTrue(parser.can_fetch("*", file_url)) + + @support.requires_working_socket() class NetworkTestCase(unittest.TestCase): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 0c3e5d92890935..61772d90e2d53b 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -55,8 +55,13 @@ def modified(self): self.last_checked = time.time() def set_url(self, url): - """Sets the URL referring to a robots.txt file.""" + """Sets the URL referring to a robots.txt file. + can be a string or a Request object. + """ self.url = url + + if isinstance(url, urllib.request.Request): + url = url.full_url self.host, self.path = urllib.parse.urlsplit(url)[1:3] def read(self): diff --git a/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst new file mode 100644 index 00000000000000..58fface7caded5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst @@ -0,0 +1 @@ +Let ``urllib.robotparser.RobotFileParser`` accept a ``urllib.request.Request`` object as well as a url string when setting a robots.txt url.