Provide User-Agent and Accept headers in URLFile requests (#2011)

It turns out some providers require these headers to be present
otherwise they'll give an error response. This issue is not present
in the requests version of the codebase because the requests library
provides default headers on our behalf.

python/cog/types.py

@@ -278,7 +278,13 @@ def __wrapped__(self) -> Any:
             # is that the book keeping for closing the response needs to be
             # handled elsewhere. There's probably a better design for this
             # in the long term.
-            res = urllib.request.urlopen(url)  # noqa: S310
+            from . import __version__ as cog_version
+
+            req = urllib.request.Request(  # noqa: S310
+                url,
+                headers={"User-agent": f"cog/{cog_version}", "Accept": "*/*"},
+            )
+            res = urllib.request.urlopen(req)  # noqa: S310
             object.__setattr__(self, "__target__", res)
 
             return res

python/tests/server/test_clients.py

@@ -138,7 +138,9 @@ async def test_upload_files_with_url_file(urlopen_mock, respx_mock):
 
     assert uploader.call_count == 1
     assert urlopen_mock.call_count == 1
-    assert urlopen_mock.call_args[0][0] == "https://example.com/cdn/my_file.txt"
+    assert (
+        urlopen_mock.call_args[0][0].full_url == "https://example.com/cdn/my_file.txt"
+    )
 
 
 @pytest.mark.asyncio
@@ -164,4 +166,6 @@ async def test_upload_files_with_url_file_with_retry(urlopen_mock, respx_mock):
 
     assert uploader.call_count == 3
     assert urlopen_mock.call_count == 1
-    assert urlopen_mock.call_args[0][0] == "https://example.com/cdn/my_file.txt"
+    assert (
+        urlopen_mock.call_args[0][0].full_url == "https://example.com/cdn/my_file.txt"
+    )

python/tests/test_types.py

@@ -1,11 +1,12 @@
 import io
 import pickle
 import urllib.request
-from urllib.response import addinfourl
 from email.message import Message
 from unittest import mock
+from urllib.response import addinfourl
 
 import pytest
+from cog import __version__
 from cog.types import Secret, URLFile, get_filename_from_url, get_filename_from_urlopen
 
 
@@ -26,6 +27,19 @@ def test_urlfile_protocol_validation():
         URLFile("data:text/plain,hello")
 
 
+@mock.patch("urllib.request.urlopen", return_value=file_fixture("hello world"))
+def test_urlfile_headers(mock_urlopen: mock.Mock):
+    u = URLFile("https://example.com/some-path", filename="my_file.txt")
+    u.read()
+
+    assert mock_urlopen.call_count == 1
+
+    req: urllib.request.Request = mock_urlopen.call_args[0][0]
+    assert req.full_url == "https://example.com/some-path"
+    assert req.headers.get("User-agent") == f"cog/{__version__}"
+    assert req.headers.get("Accept") == "*/*"
+
+
 @mock.patch("urllib.request.urlopen", return_value=file_fixture("hello world"))
 def test_urlfile_custom_filename(mock_urlopen):
     u = URLFile("https://example.com/some-path", filename="my_file.txt")