diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..925b8ac --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [master] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..fd906de --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-docstring-first + - id: check-yaml + - id: debug-statements + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: requirements-txt-fixer + - id: double-quote-string-fixer + - repo: https://github.com/asottile/reorder_python_imports + rev: v2.6.0 + hooks: + - id: reorder-python-imports + - repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v1.5.7 + hooks: + - id: autopep8 + - repo: https://github.com/asottile/yesqa + rev: v1.3.0 + hooks: + - id: yesqa + - repo: https://gitlab.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: [--max-line-length=120] diff --git a/LICENSE b/LICENSE index 556b37a..2d68ede 100644 --- a/LICENSE +++ b/LICENSE @@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 7c824b3..8f79e57 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) + Scoville - A tool to attribute size information in MVT tiles. Current scoville commands: diff --git a/requirements.txt b/requirements.txt index d7d6524..63bab90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ click +msgpack +Pillow requests requests_futures squarify -msgpack -Pillow diff --git a/scoville/command.py b/scoville/command.py index 0e7e84a..eaa43aa 100644 --- a/scoville/command.py +++ b/scoville/command.py @@ -1,5 +1,7 @@ -import click import json + +import click + from scoville.mvt import Tile @@ -106,7 +108,7 @@ def _info(mvt_file, kind, d3_json): if res.status_code == 200: tile = Tile(res.content) else: - click.echo("Failed to fetch tile, status was %r" % + click.echo('Failed to fetch tile, status was %r' % (res.status_code,)) return @@ -156,16 +158,16 @@ def compare(tiles_file, url1, url2, kind): tile_urls1 = read_urls(tiles_file, url1) tile_urls2 = read_urls(tiles_file, url2) for tile, url1, url2 in zip(tiles, tile_urls1, tile_urls2): - print("===BEGIN===>%s" % tile) + print('===BEGIN===>%s' % tile) - print("url1: --->%s" % url1) + print('url1: --->%s' % url1) _info(url1, kind, None) print() - print("url2: --->%s" % url2) + print('url2: --->%s' % url2) _info(url1, kind, None) - print("<===END===%s\n\n" % tile) + print('<===END===%s\n\n' % tile) @cli.command() @@ -346,9 +348,10 @@ def outliers(tiles_file, url, cache, nprocs, num_outliers_per_layer): result = calculate_outliers(tiles, num_outliers_per_layer, cache, nprocs) for name in sorted(result.keys()): - click.secho("Layer %r" % name, fg='green', bold=True) + click.secho('Layer %r' % name, fg='green', bold=True) for size, features_size, properties_size, url in sorted(result[name]): - click.echo("t:%8d f:%8d p:%8d %s" % (size, features_size, properties_size, url)) + click.echo('t:%8d f:%8d p:%8d %s' % + (size, features_size, properties_size, url)) def scoville_main(): diff --git a/scoville/mvt.py b/scoville/mvt.py index cfca6d8..0cb4408 100644 --- a/scoville/mvt.py +++ b/scoville/mvt.py @@ -1,5 +1,8 @@ -from scoville.pbf import Message, WireType -from enum import Enum, IntEnum +from enum import Enum +from enum import IntEnum + +from scoville.pbf import Message +from scoville.pbf import WireType class GeomType(Enum): @@ -67,15 +70,15 @@ def _decode_value(data): value = field.as_int32() != 0 else: - raise ValueError("Unexpected tag %d while decoding value" + raise ValueError('Unexpected tag %d while decoding value' % (field.tag,)) # the MVT spec says that there should be one and only one field in the # Value message, so check for that. if value is None: - raise ValueError("Found no fields when decoding value") + raise ValueError('Found no fields when decoding value') if count > 1: - raise ValueError("Found multiple fields when decoding value") + raise ValueError('Found multiple fields when decoding value') return value @@ -134,7 +137,7 @@ def __unpack(self): self._geom_cmds_size += field.size else: - raise ValueError("Unknown Feature tag %d" % field.tag) + raise ValueError('Unknown Feature tag %d' % field.tag) self.unpacked = True @@ -239,10 +242,10 @@ def __init__(self, field): self.extent = field.as_uint32() else: - raise ValueError("Unknown Layer tag %d" % field.tag) + raise ValueError('Unknown Layer tag %d' % field.tag) if self.name is None: - raise ValueError("Layer missing name, but name is required") + raise ValueError('Layer missing name, but name is required') def __iter__(self): return iter(self.features) @@ -261,7 +264,7 @@ def __next__(self): if field.tag != Tile.Tags.LAYER: raise ValueError( - "Expecting layer with tag %d, got tag %d instead." + 'Expecting layer with tag %d, got tag %d instead.' % (Tile.Tags.LAYER, field.tag)) return Layer(field) @@ -289,7 +292,7 @@ class Tile(object): class Tags(IntEnum): LAYER = 3 - def __init__(self, data, name=""): + def __init__(self, data, name=''): self.data = data self.name = name diff --git a/scoville/pbf.py b/scoville/pbf.py index adf75c2..c6a6e3e 100644 --- a/scoville/pbf.py +++ b/scoville/pbf.py @@ -76,7 +76,7 @@ def as_memoryview(self): def as_packed(self, wire_type): if wire_type is WireType.length_delimited: - raise ValueError("Length delimited wire types cannot be packed.") + raise ValueError('Length delimited wire types cannot be packed.') return _LengthDelimited._iter( self.buf, self.tag, _WIRE_TYPES[wire_type]) @@ -144,8 +144,8 @@ def varint(self): def get_bytes(self, num_bytes): if self.pos + num_bytes > self.end: - raise EOFError("Unexpected end of PBF data, attempting to read " - "%d bytes from position %d goes past end at %d" + raise EOFError('Unexpected end of PBF data, attempting to read ' + '%d bytes from position %d goes past end at %d' % (num_bytes, self.pos, self.end)) m = self.buf[self.pos:self.pos+num_bytes] self.pos += num_bytes @@ -153,7 +153,7 @@ def get_bytes(self, num_bytes): def get_byte(self): if self.pos == self.end: - raise EOFError("Unexpected end of PBF data at byte %d" % self.pos) + raise EOFError('Unexpected end of PBF data at byte %d' % self.pos) v = self.buf[self.pos] self.pos += 1 diff --git a/scoville/percentiles.py b/scoville/percentiles.py index aa04ff2..c4104a3 100644 --- a/scoville/percentiles.py +++ b/scoville/percentiles.py @@ -1,5 +1,7 @@ -import requests from collections import defaultdict + +import requests + from scoville.mvt import Tile @@ -12,7 +14,7 @@ def _fetch_http(url): # TODO: retry? better error handling! if res.status_code != requests.codes.ok: - print("Got tile response %d for %s" % (res.status_code, url)) + print('Got tile response %d for %s' % (res.status_code, url)) return None return res.content @@ -137,7 +139,8 @@ def add(self, tile_url): return tile = Tile(data) for layer in tile: - self._insert(layer.name, layer.size, layer.features_size, layer.properties_size, tile_url) + self._insert(layer.name, layer.size, layer.features_size, + layer.properties_size, tile_url) def encode(self): from msgpack import packb @@ -191,7 +194,8 @@ def parallel(tile_urls, factory, nprocs): workers = [] for i in range(0, nprocs): - w = Process(target=worker, args=(input_queue, output_queue, factory.create())) + w = Process(target=worker, args=( + input_queue, output_queue, factory.create())) w.start() workers.append(w) @@ -248,7 +252,8 @@ def factory_fn(): return Aggregator(cache) if nprocs > 1: - results = parallel(tile_urls, FactoryFunctionHolder(factory_fn), nprocs) + results = parallel( + tile_urls, FactoryFunctionHolder(factory_fn), nprocs) else: results = sequential(tile_urls, factory_fn) @@ -283,7 +288,8 @@ def factory_fn(): return LargestN(num_outliers, cache) if nprocs > 1: - results = parallel(tile_urls, FactoryFunctionHolder(factory_fn), nprocs) + results = parallel( + tile_urls, FactoryFunctionHolder(factory_fn), nprocs) else: results = sequential(tile_urls, factory_fn) diff --git a/scoville/proxy.py b/scoville/proxy.py index fd05e78..7f0ae3f 100644 --- a/scoville/proxy.py +++ b/scoville/proxy.py @@ -1,13 +1,14 @@ import http.server +import re import socketserver + import pkg_resources -import re import requests import squarify -from scoville.mvt import Tile +from scoville.mvt import Tile -TILE_PATTERN = re.compile('^/tiles/([0-9]+)/([0-9]+)/([0-9]+)\.png$') +TILE_PATTERN = re.compile('^/tiles/([0-9]+)/([0-9]+)/([0-9]+)/.png$') class Treemap(object): @@ -28,7 +29,7 @@ def render(self, tiles): sizes.sort(reverse=True) width = height = 256 - im = Image.new("RGB", (width, height), "black") + im = Image.new('RGB', (width, height), 'black') values = squarify.normalize_sizes([r[0] for r in sizes], width, height) rects = squarify.squarify(values, 0, 0, width, height) @@ -60,7 +61,7 @@ def render(self, tiles): text_w, text_h = font.getsize(tile.name) center = (width / 2, height / 2) top_left_up_a_couple = (center[0] - text_w / 2, - 2 * text_h) + 2 * text_h) draw.text(top_left_up_a_couple, tile.name, fill='black', font=font) del draw @@ -96,14 +97,14 @@ def render(self, tiles): assert len(tiles) == ntiles ** 2 width = height = 256 - im = Image.new("RGB", (width, height), "black") + im = Image.new('RGB', (width, height), 'black') draw = ImageDraw.Draw(im) scale = width / ntiles assert width == scale * ntiles - parent_coord_name = "" + parent_coord_name = '' for x in range(0, ntiles): for y in range(0, ntiles): tile = tiles[(x, y)] @@ -114,7 +115,7 @@ def render(self, tiles): parent_coord_name = tile.name draw.rectangle( - [x * scale, y * scale, (x+1) * scale, (y+1) * scale], + [x * scale, y * scale, (x + 1) * scale, (y + 1) * scale], fill=colour) if parent_coord_name: @@ -132,10 +133,10 @@ def render(self, tiles): class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): - if self.path in ("/", "/index.html", "/style.css", "/map.js"): + if self.path in ('/', '/index.html', '/style.css', '/map.js'): template_name = self.path[1:] if not template_name: - template_name = "index.html" + template_name = 'index.html' self.send_template(template_name) return @@ -145,8 +146,8 @@ def do_GET(self): z, x, y = list(map(int, m.groups())) if 0 <= z < 16 and \ - 0 <= x < (1 << z) and \ - 0 <= y < (1 << z): + 0 <= x < (1 << z) and \ + 0 <= y < (1 << z): self.send_tile(z, x, y) return @@ -167,9 +168,9 @@ def send_tile(self, z, x, y): for name, coord in tile_map.items(): z, x, y = coord url = self.server.url_pattern \ - .replace("{z}", str(z)) \ - .replace("{x}", str(x)) \ - .replace("{y}", str(y)) + .replace('{z}', str(z)) \ + .replace('{x}', str(x)) \ + .replace('{y}', str(y)) futures[name] = session.get(url) @@ -180,7 +181,9 @@ def send_tile(self, z, x, y): self.send_response(res.status_code) return - tiles[name] = Tile(res.content, "%s/%s/%s" % (parent_coord[0], parent_coord[1], parent_coord[2])) + tiles[name] = Tile(res.content, '%s/%s/%s' % + (parent_coord[0], parent_coord[1], + parent_coord[2])) im = self.server.renderer.render(tiles) @@ -194,7 +197,8 @@ def send_tile(self, z, x, y): def send_template(self, template_name): from jinja2 import Template - resource = pkg_resources.resource_string(__name__, "proxy/" + template_name) + resource = pkg_resources.resource_string( + __name__, 'proxy/' + template_name) template = Template(resource.decode()) data = template.render(port=self.server.server_port) @@ -213,7 +217,7 @@ def __init__(self, server_address, handler_class, url_pattern, renderer): def serve_http(url, port, renderer): - httpd = ThreadedHTTPServer(("", port), Handler, url, renderer) - print("Listening on port %d. Point your browser towards " - "http://localhost:%d/" % (port, port)) + httpd = ThreadedHTTPServer(('', port), Handler, url, renderer) + print('Listening on port %d. Point your browser towards ' + 'http://localhost:%d/' % (port, port)) httpd.serve_forever() diff --git a/setup.py b/setup.py index 10796d5..209e3dc 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,12 @@ -from setuptools import setup, find_packages +from setuptools import find_packages +from setuptools import setup version = '0.2.0' setup( name='scoville', version=version, - description="A tool for attributing MVT tile size.", + description='A tool for attributing MVT tile size.', long_description=open('README.md').read(), classifiers=[ # strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers diff --git a/tests/test_mvt.py b/tests/test_mvt.py index ea2a660..89e1336 100644 --- a/tests/test_mvt.py +++ b/tests/test_mvt.py @@ -22,7 +22,7 @@ def test_encode_unicode_property(self): layers = list(t) self.assertEqual(len(layers), 1) water_layer = layers[0] - self.assertEqual(water_layer.name, "water") + self.assertEqual(water_layer.name, 'water') features = list(water_layer) self.assertEqual(len(features), 1) feature = features[0]