Sending also "count" in resource queries when there's no "follow". Fixes #374

.travis.yml

@@ -15,7 +15,7 @@ matrix:
       language: generic
       env:
         - TOX_ENV=py27
-        - PYTHON_VERSION='2.7'
+        - PYTHON_VERSION='2.7.9'
     - os: osx
       language: generic
       env:
@@ -40,7 +40,8 @@ branches:
 before_install: |
     if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       # From https://pythonhosted.org/CodeChat/.travis.yml.html
-      brew install pyenv-virtualenv
+      # Homebrew currently fails after updating. See also: https://discuss.circleci.com/t/brew-install-fails-while-updating/32992/4
+      HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv-virtualenv
       eval "$(pyenv init -)"
       eval "$(pyenv virtualenv-init -)"
       # See https://github.com/travis-ci/travis-ci/issues/4834, but

requirements.in

@@ -1,13 +1,15 @@
 click
 docker-py
-PyYAML
-requests
+#PyYAML
+#requests
 retrying
 six
 tqdm
 
 scrapinghub>=2.0.3
 
+pip<19.3
+
 # address known vulnerabilities
 requests>=2.20.0    # CVE-2018-18074
 pyyaml>=4.2b1       # CVE-2017-18342

requirements.txt

@@ -20,3 +20,6 @@ six==1.10.0
 tqdm==4.11.2
 urllib3==1.25.3           # via requests
 websocket-client==0.37.0  # via docker-py
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip==19.2.3

setup.py

@@ -31,7 +31,7 @@
     install_requires=[
         'click',
         'docker-py',
-        'pip',
+        'pip<19.3',
         'PyYAML',
         'retrying',
         'requests',

shub/utils.py

@@ -551,7 +551,7 @@ def job_resource_iter(job, resource, output_json=False, follow=True,
         follow = False
     resource_iter = resource.iter_json if output_json else resource.iter_values
     if not follow:
-        for item in resource_iter(startafter=last_item_key):
+        for item in resource_iter(startafter=last_item_key, count=tail):
             yield item
         return
     while True:
@@ -637,6 +637,8 @@ def download_from_pypi(dest, pkg=None, reqfile=None, extra_args=None):
         no_wheel = ['--no-binary=:all:']
     if pip_version >= LooseVersion('8'):
         cmd = 'download'
+    if pip_version >= LooseVersion('19.3'):
+        raise NotImplementedError('Expecting pip<19.3')
     with patch_sys_executable():
         pip_main([cmd, '-d', dest, '--no-deps'] + no_wheel + extra_args +
                  target)

tests/test_utils.py

@@ -220,6 +220,7 @@ def test_job_resource_iter(self, mock_sleep):
         job = MagicMock(spec=['key', 'metadata', 'resource'])
         job.key = 'jobkey'
         job.metadata = {'state': 'running'}
+        job.resource.stats.return_value = {'totals': {'input_values': 0}}
 
         def make_items(iterable):
             return [json.dumps({'_key': x}) for x in iterable]
@@ -252,14 +253,16 @@ def jri_result(follow, tail=None):
                 output_json=True,
             ))
 
-        job.resource.iter_json = magic_iter
+        job.resource.iter_json = Mock(wraps=magic_iter)
 
         magic_iter.stage = 0
         self.assertEqual(jri_result(False), make_items([1, 2, 3]))
+        self.assertIsNone(job.resource.iter_json.call_args[1].get('count'))
         self.assertFalse(mock_sleep.called)
 
         magic_iter.stage = 0
-        self.assertEqual(jri_result(True), make_items([1, 2, 3, 4, 5, 6]))
+        self.assertEqual(jri_result(True, tail=6), make_items([1, 2, 3, 4, 5, 6]))
+        self.assertIsNone(job.resource.iter_json.call_args[1].get('count'))
         self.assertTrue(mock_sleep.called)
 
         magic_iter.stage = 0
@@ -269,6 +272,7 @@ def jri_result(follow, tail=None):
         magic_iter.stage = 2
         job.resource.stats.return_value = {'totals': {'input_values': 1000}}
         self.assertEqual(jri_result(True, tail=3), [])
+        self.assertEqual(job.resource.iter_json.call_args[1].get('count'), 3)
 
     @patch('shub.utils.requests.get', autospec=True)
     def test_latest_github_release(self, mock_get):
@@ -371,6 +375,10 @@ def _call(*args, **kwargs):
         pipargs = _call('tmpdir', reqfile='req.txt')
         self.assertEqual(pipargs.index('-r') + 1, pipargs.index('req.txt'))
 
+        # pip>=19.3 shall be unsupported for now
+        mock_pip.__version__ = '19.3'
+        self.assertRaises(NotImplementedError, _call, ['tmpdir'], {'pkg': 'shub'})
+
         # Replace deprecated commands in newer versions
         mock_pip.__version__ = '7.1.2.dev0'
         pipargs = _call('tmpdir', pkg='shub')