Adopt to aiohttp 2.x changes

twtxt/helper.py

@@ -62,21 +62,25 @@ def style_source(source, porcelain=False):
 
 def style_source_with_status(source, status, porcelain=False):
     if porcelain:
-        return "{nick}\t{url}\t{status}".format(
+        return "{nick}\t{url}\t{status}\t{content_length}\t{last_modified}".format(
             nick=source.nick,
             url=source.url,
-            status=status)
+            status=status.status_code,
+            content_length=status.content_length,
+            last_modified=status.last_modified)
     else:
-        if status == 200:
-            scolor, smessage = "green", str(status)
+        if status.status_code == 200:
+            scolor, smessage = "green", str(status.status_code)
         elif status:
-            scolor, smessage = "red", str(status)
+            scolor, smessage = "red", str(status.status_code)
         else:
             scolor, smessage = "red", "ERROR"
-        return "➤ {nick} @ {url} ({status})".format(
+        return "➤ {nick} @ {url} [{content_length}, {last_modified}] ({status})".format(
             nick=click.style(source.nick, bold=True, fg=scolor),
             url=source.url,
-            status=click.style(smessage, fg=scolor))
+            status=click.style(smessage, fg=scolor),
+            content_length=status.natural_content_length,
+            last_modified=status.natural_last_modified)
 
 
 def validate_created_at(ctx, param, value):

twtxt/twhttp.py

@@ -10,24 +10,57 @@
 
 import asyncio
 import logging
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
 from ssl import CertificateError
 
 import aiohttp
 import click
+import humanize
 
-from twtxt.cache import Cache
 from twtxt.helper import generate_user_agent
 from twtxt.parser import parse_tweets
 
 logger = logging.getLogger(__name__)
 
 
+class SourceResponse:
+    """A :class:`SourceResponse` contains information about a :class:`Source`’s HTTP request.
+
+    :param int status_code: response status code
+    :param str content_length: Content-Length header field
+    :param str last_modified: Last-Modified header field
+    """
+
+    def __init__(self, status_code, content_length, last_modified):
+        self.status_code = status_code
+        self.content_length = content_length
+        self.last_modified = last_modified
+
+    @property
+    def natural_content_length(self):
+        return humanize.naturalsize(self.content_length)
+
+    @property
+    def natural_last_modified(self):
+        last_modified = parsedate_to_datetime(self.last_modified)
+        now = datetime.now(timezone.utc)
+        tense = "from now" if last_modified > now else "ago"
+        return "{0} {1}".format(humanize.naturaldelta(now - last_modified), tense)
+
+
 @asyncio.coroutine
 def retrieve_status(client, source):
     status = None
     try:
         response = yield from client.head(source.url)
-        status = response.status
+        if response.headers.get("Content-Length"):
+            content_length = response.headers.get("Content-Length")
+        else:
+            content_length = 0
+        status = SourceResponse(status_code=response.status,
+                                content_length=content_length,
+                                last_modified=response.headers.get("Last-Modified"))
         yield from response.release()
     except CertificateError as e:
         click.echo("✗ SSL Certificate Error: The feed's ({0}) SSL certificate is untrusted. Try using HTTP, "
@@ -108,9 +141,9 @@ def process_sources_for_file(client, sources, limit, cache=None):
 
 
 def get_remote_tweets(sources, limit=None, timeout=5.0, cache=None):
-    conn = aiohttp.TCPConnector(conn_timeout=timeout, use_dns_cache=True)
+    conn = aiohttp.TCPConnector(use_dns_cache=True)
     headers = generate_user_agent()
-    with aiohttp.ClientSession(connector=conn, headers=headers) as client:
+    with aiohttp.ClientSession(connector=conn, headers=headers, conn_timeout=timeout) as client:
         loop = asyncio.get_event_loop()
 
         def start_loop(client, sources, limit, cache=None):
@@ -122,9 +155,9 @@ def start_loop(client, sources, limit, cache=None):
 
 
 def get_remote_status(sources, timeout=5.0):
-    conn = aiohttp.TCPConnector(conn_timeout=timeout, use_dns_cache=True)
+    conn = aiohttp.TCPConnector(use_dns_cache=True)
     headers = generate_user_agent()
-    with aiohttp.ClientSession(connector=conn, headers=headers) as client:
+    with aiohttp.ClientSession(connector=conn, headers=headers, conn_timeout=timeout) as client:
         loop = asyncio.get_event_loop()
         result = loop.run_until_complete(process_sources_for_status(client, sources))
     return result