From 52d12ea82342fb1a5697cf48b3723f7f2d96f29f Mon Sep 17 00:00:00 2001 From: ephphatha Date: Mon, 24 Apr 2023 20:53:16 +1000 Subject: [PATCH 1/8] Allow passing a Request instance for the url parameter --- Lib/test/test_robotparser.py | 59 ++++++++++++++++++++++++++++++++++++ Lib/urllib/robotparser.py | 7 ++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index cd1477037e94b7..6c016e05b5f7d8 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -773,6 +773,65 @@ def testServiceUnavailable(self): self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) +class UserAgentRobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + if self.headers.get('User-Agent').startswith('Python-urllib'): + self.send_error(403, "Forbidden access") + else: + self.send_response(200) + self.end_headers() + self.wfile.write(b"User-agent: *\nDisallow:") + + def log_message(self, format, *args): + pass + + +@unittest.skipUnless( + support.has_socket_support, + "Socket server requires working socket." +) +class UserAgentSiteTestCase(unittest.TestCase): + + def setUp(self): + # clear _opener global variable + self.addCleanup(urllib.request.urlcleanup) + + self.server = HTTPServer((socket_helper.HOST, 0), UserAgentRobotHandler) + + self.t = threading.Thread( + name='HTTPServer serving', + target=self.server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval':0.01}) + self.t.daemon = True # In case this function raises. + self.t.start() + + def tearDown(self): + self.server.shutdown() + self.t.join() + self.server.server_close() + + @threading_helper.reap_threads + def testUserAgentFilteringSite(self): + addr = self.server.server_address + url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) + robots_url = url + "/robots.txt" + file_url = url + "/document" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(robots_url) + parser.read() + self.assertTrue(parser.disallow_all) + self.assertFalse(parser.can_fetch("*", file_url)) + parser = urllib.robotparser.RobotFileParser() + parser.set_url(urllib.request.Request(robots_url, headers={'User-Agent': 'cybermapper'})) + parser.read() + self.assertFalse(parser.disallow_all) + self.assertTrue(parser.can_fetch("*", file_url)) + + @support.requires_working_socket() class NetworkTestCase(unittest.TestCase): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 0c3e5d92890935..9af9db33dddb7e 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -57,7 +57,12 @@ def modified(self): def set_url(self, url): """Sets the URL referring to a robots.txt file.""" self.url = url - self.host, self.path = urllib.parse.urlsplit(url)[1:3] + + if isinstance(url, urllib.request.Request): + self.host = url.host + self.path = url.selector + else: + self.host, self.path = urllib.parse.urlparse(url)[1:3] def read(self): """Reads the robots.txt URL and feeds it to the parser.""" From dd2275410524713c05abd44544f19064091e5766 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 24 Apr 2023 11:12:01 +0000 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst diff --git a/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst new file mode 100644 index 00000000000000..1619832cf27cd5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst @@ -0,0 +1 @@ +Let ``urllib.robotparser.RobotParser`` accept a ``urllib.request.Request`` object as well as a url string when setting a robots.txt url. From 814d27f32c96ccbc321749c98daf41f1e57d26ae Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 4 Jul 2026 15:33:33 -0700 Subject: [PATCH 3/8] gh-60055: rebase onto main and document Request support --- Doc/library/urllib.robotparser.rst | 24 ++++++++++++++++++++---- Lib/urllib/robotparser.py | 20 +++++++++++--------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 1fa7fc13baa539..a726af30310db5 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -21,14 +21,17 @@ website that published the :file:`robots.txt` file. For more details on the structure of :file:`robots.txt` files, see :rfc:`9309`. -.. class:: RobotFileParser(url='') +.. class:: RobotFileParser(url_or_request='') This class provides methods to read, parse and answer questions about the - :file:`robots.txt` file at *url*. + :file:`robots.txt` file at *url* or a :class:`Request` object with additional + user-agent headers populated. - .. method:: set_url(url) - Sets the URL referring to a :file:`robots.txt` file. + .. method:: set_url(url_or_request) + + Sets the URL referring to a :file:`robots.txt` file or a :class:`Request` + object with additional user-agent headers populated. .. method:: read() @@ -102,3 +105,16 @@ class:: True >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/") False + + +The following example demonstrates use of a :class:`Request` object with additional user-agent headers populated:: + + >>> import urllib.robotparser + >>> import urllib.request + >>> rp = urllib.robotparser.RobotFileParser() + >>> rp.set_url(urllib.request.Request("http://en.wikipedia.org/robots.txt", headers={"User-Agent": "IsraBot"})) + >>> rp.read() + >>> rp.can_fetch("*", "https://en.wikipedia.org/") + True + >>> rp.can_fetch("*", "https://en.wikipedia.org/trap/") + False \ No newline at end of file diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 9af9db33dddb7e..b083785271bdb2 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -27,14 +27,14 @@ class RobotFileParser: """ - def __init__(self, url=''): + def __init__(self, url_or_request=''): self.entries = [] self.groups = {} self.sitemaps = [] self.default_entry = None self.disallow_all = False self.allow_all = False - self.set_url(url) + self.set_url(url_or_request) self.last_checked = 0 def mtime(self): @@ -54,15 +54,17 @@ def modified(self): import time self.last_checked = time.time() - def set_url(self, url): - """Sets the URL referring to a robots.txt file.""" - self.url = url + def set_url(self, url_or_request): + """Sets the URL referring to a robots.txt file. + url_or_request can be a string or a Request object. + """ + self.url = url_or_request - if isinstance(url, urllib.request.Request): - self.host = url.host - self.path = url.selector + if isinstance(url_or_request, urllib.request.Request): + self.host = url_or_request.host + self.path = url_or_request.selector else: - self.host, self.path = urllib.parse.urlparse(url)[1:3] + self.host, self.path = urllib.parse.urlparse(url_or_request)[1:3] def read(self): """Reads the robots.txt URL and feeds it to the parser.""" From 1959b0296b3f4518224ddfb0e4747218322426d1 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 4 Jul 2026 17:14:12 -0700 Subject: [PATCH 4/8] Fix the lint warnings and CI Errors. --- Doc/library/urllib.robotparser.rst | 14 ++++++++------ Lib/urllib/robotparser.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index a726af30310db5..be50b28bd2d1a8 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -24,14 +24,15 @@ structure of :file:`robots.txt` files, see :rfc:`9309`. .. class:: RobotFileParser(url_or_request='') This class provides methods to read, parse and answer questions about the - :file:`robots.txt` file at *url* or a :class:`Request` object with additional - user-agent headers populated. + :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object + with additional user-agent headers populated. .. method:: set_url(url_or_request) - Sets the URL referring to a :file:`robots.txt` file or a :class:`Request` - object with additional user-agent headers populated. + Sets the URL referring to a :file:`robots.txt` file or a + :class:`urllib.request.Request` object with additional user-agent headers + populated. .. method:: read() @@ -107,7 +108,8 @@ class:: False -The following example demonstrates use of a :class:`Request` object with additional user-agent headers populated:: +The following example demonstrates use of a :class:`urllib.request.Request` +object with additional user-agent headers populated:: >>> import urllib.robotparser >>> import urllib.request @@ -117,4 +119,4 @@ The following example demonstrates use of a :class:`Request` object with additio >>> rp.can_fetch("*", "https://en.wikipedia.org/") True >>> rp.can_fetch("*", "https://en.wikipedia.org/trap/") - False \ No newline at end of file + False diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index b083785271bdb2..d16130c91546de 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -55,7 +55,7 @@ def modified(self): self.last_checked = time.time() def set_url(self, url_or_request): - """Sets the URL referring to a robots.txt file. + """Sets the URL referring to a robots.txt file. url_or_request can be a string or a Request object. """ self.url = url_or_request From 6f9e3f559758aae0208645ee0dd94bd63e3931da Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 5 Jul 2026 06:25:09 -0700 Subject: [PATCH 5/8] Addressing RobotFileParser review comments. --- Doc/library/urllib.robotparser.rst | 21 ++++---- Lib/test/test_robotparser.py | 52 +++++-------------- Lib/urllib/robotparser.py | 17 +++--- ...3-04-24-11-12-00.gh-issue-60055.UjP6aX.rst | 2 +- 4 files changed, 33 insertions(+), 59 deletions(-) diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index be50b28bd2d1a8..1b187a874aed8c 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -21,18 +21,21 @@ website that published the :file:`robots.txt` file. For more details on the structure of :file:`robots.txt` files, see :rfc:`9309`. -.. class:: RobotFileParser(url_or_request='') +.. class:: RobotFileParser(url='') This class provides methods to read, parse and answer questions about the - :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object - with additional user-agent headers populated. + :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object. + .. versionchanged:: 3.16 + *url* parameter can be a :class:`urllib.request.Request` object. - .. method:: set_url(url_or_request) + .. method:: set_url(url) Sets the URL referring to a :file:`robots.txt` file or a - :class:`urllib.request.Request` object with additional user-agent headers - populated. + :class:`urllib.request.Request` object. + + .. versionchanged:: 3.16 + *url* parameter can be a :class:`urllib.request.Request` object. .. method:: read() @@ -114,9 +117,9 @@ object with additional user-agent headers populated:: >>> import urllib.robotparser >>> import urllib.request >>> rp = urllib.robotparser.RobotFileParser() - >>> rp.set_url(urllib.request.Request("http://en.wikipedia.org/robots.txt", headers={"User-Agent": "IsraBot"})) + >>> rp.set_url(urllib.request.Request("http://www.pythontest.net/robots.txt", headers={"User-Agent": "IsraBot"})) >>> rp.read() - >>> rp.can_fetch("*", "https://en.wikipedia.org/") + >>> rp.can_fetch("*", "http://www.pythontest.net/") True - >>> rp.can_fetch("*", "https://en.wikipedia.org/trap/") + >>> rp.can_fetch("*", "https://www.pythontest.net/no-robots-here/") False diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 6c016e05b5f7d8..725c6e3b09e1d0 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -773,51 +773,23 @@ def testServiceUnavailable(self): self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) -class UserAgentRobotHandler(BaseHTTPRequestHandler): +class UserAgentSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): - def do_GET(self): - if self.headers.get('User-Agent').startswith('Python-urllib'): - self.send_error(403, "Forbidden access") - else: - self.send_response(200) - self.end_headers() - self.wfile.write(b"User-agent: *\nDisallow:") - - def log_message(self, format, *args): - pass - - -@unittest.skipUnless( - support.has_socket_support, - "Socket server requires working socket." -) -class UserAgentSiteTestCase(unittest.TestCase): - - def setUp(self): - # clear _opener global variable - self.addCleanup(urllib.request.urlcleanup) - - self.server = HTTPServer((socket_helper.HOST, 0), UserAgentRobotHandler) - - self.t = threading.Thread( - name='HTTPServer serving', - target=self.server.serve_forever, - # Short poll interval to make the test finish quickly. - # Time between requests is short enough that we won't wake - # up spuriously too many times. - kwargs={'poll_interval':0.01}) - self.t.daemon = True # In case this function raises. - self.t.start() + class RobotHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.headers.get('User-Agent').startswith('Python-urllib'): + self.send_error(403, "Forbidden access") + else: + self.send_response(200) + self.end_headers() + self.wfile.write(b"User-agent: *\nDisallow:") - def tearDown(self): - self.server.shutdown() - self.t.join() - self.server.server_close() + def log_message(self, format, *args): + pass - @threading_helper.reap_threads def testUserAgentFilteringSite(self): addr = self.server.server_address - url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) + url = f'http://{socket_helper.HOST}:{addr[1]}' robots_url = url + "/robots.txt" file_url = url + "/document" parser = urllib.robotparser.RobotFileParser() diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index d16130c91546de..6696e10eccd329 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -27,14 +27,14 @@ class RobotFileParser: """ - def __init__(self, url_or_request=''): + def __init__(self, url=''): self.entries = [] self.groups = {} self.sitemaps = [] self.default_entry = None self.disallow_all = False self.allow_all = False - self.set_url(url_or_request) + self.set_url(url) self.last_checked = 0 def mtime(self): @@ -54,17 +54,16 @@ def modified(self): import time self.last_checked = time.time() - def set_url(self, url_or_request): + def set_url(self, url): """Sets the URL referring to a robots.txt file. - url_or_request can be a string or a Request object. + can be a string or a Request object. """ - self.url = url_or_request + self.url = url - if isinstance(url_or_request, urllib.request.Request): - self.host = url_or_request.host - self.path = url_or_request.selector + if isinstance(url, urllib.request.Request): + self.host, self.path = urllib.parse.urlsplit(url.full_url)[1:3] else: - self.host, self.path = urllib.parse.urlparse(url_or_request)[1:3] + self.host, self.path = urllib.parse.urlsplit(url)[1:3] def read(self): """Reads the robots.txt URL and feeds it to the parser.""" diff --git a/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst index 1619832cf27cd5..58fface7caded5 100644 --- a/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst +++ b/Misc/NEWS.d/next/Library/2023-04-24-11-12-00.gh-issue-60055.UjP6aX.rst @@ -1 +1 @@ -Let ``urllib.robotparser.RobotParser`` accept a ``urllib.request.Request`` object as well as a url string when setting a robots.txt url. +Let ``urllib.robotparser.RobotFileParser`` accept a ``urllib.request.Request`` object as well as a url string when setting a robots.txt url. From 320b83725cbf79f7d559c28133eeed42a63dc8e2 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 5 Jul 2026 07:15:04 -0700 Subject: [PATCH 6/8] Update Doc/library/urllib.robotparser.rst Co-authored-by: Serhiy Storchaka --- Doc/library/urllib.robotparser.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 1b187a874aed8c..fcc2211eadb7db 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -26,7 +26,7 @@ structure of :file:`robots.txt` files, see :rfc:`9309`. This class provides methods to read, parse and answer questions about the :file:`robots.txt` file at *url* or a :class:`urllib.request.Request` object. - .. versionchanged:: 3.16 + .. versionchanged:: next *url* parameter can be a :class:`urllib.request.Request` object. .. method:: set_url(url) From 6fe0acd12e47c629517d0ac3582589230451c8aa Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 5 Jul 2026 07:20:21 -0700 Subject: [PATCH 7/8] Address additional Review Comments. --- Doc/library/urllib.robotparser.rst | 2 +- Lib/urllib/robotparser.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index fcc2211eadb7db..ee932c18a3b9d4 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -121,5 +121,5 @@ object with additional user-agent headers populated:: >>> rp.read() >>> rp.can_fetch("*", "http://www.pythontest.net/") True - >>> rp.can_fetch("*", "https://www.pythontest.net/no-robots-here/") + >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/") False diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 6696e10eccd329..61772d90e2d53b 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -61,9 +61,8 @@ def set_url(self, url): self.url = url if isinstance(url, urllib.request.Request): - self.host, self.path = urllib.parse.urlsplit(url.full_url)[1:3] - else: - self.host, self.path = urllib.parse.urlsplit(url)[1:3] + url = url.full_url + self.host, self.path = urllib.parse.urlsplit(url)[1:3] def read(self): """Reads the robots.txt URL and feeds it to the parser.""" From e231aecd65ddc3219fda9f411880030d4ef83768 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 5 Jul 2026 07:21:58 -0700 Subject: [PATCH 8/8] Changed versionchanged to next. --- Doc/library/urllib.robotparser.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index ee932c18a3b9d4..3e84f86d4dd383 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -34,7 +34,7 @@ structure of :file:`robots.txt` files, see :rfc:`9309`. Sets the URL referring to a :file:`robots.txt` file or a :class:`urllib.request.Request` object. - .. versionchanged:: 3.16 + .. versionchanged:: next *url* parameter can be a :class:`urllib.request.Request` object. .. method:: read()