From cbf51456ede00d9d8bcf5df61c82a7d2de7178eb Mon Sep 17 00:00:00 2001 From: Nikita Barskov Date: Tue, 9 Apr 2024 00:18:23 +0200 Subject: [PATCH 1/2] Update dependencies using `poetry lock` --- poetry.lock | 226 +++++++++++++++++++++++++++---------------------- pyproject.toml | 38 ++++----- 2 files changed, 143 insertions(+), 121 deletions(-) diff --git a/poetry.lock b/poetry.lock index d641872..7e70ad9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -638,13 +638,13 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "griffe" -version = "0.42.0" +version = "0.42.1" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." optional = false python-versions = ">=3.8" files = [ - {file = "griffe-0.42.0-py3-none-any.whl", hash = "sha256:384df6b802a60f70e65fdb7e83f5b27e2da869a12eac85b25b55250012dbc263"}, - {file = "griffe-0.42.0.tar.gz", hash = "sha256:fb83ee602701ffdf99c9a6bf5f0a5a3bd877364b3bffb2c451dc8fbd9645b0cf"}, + {file = "griffe-0.42.1-py3-none-any.whl", hash = "sha256:7e805e35617601355edcac0d3511cedc1ed0cb1f7645e2d336ae4b05bbae7b3b"}, + {file = "griffe-0.42.1.tar.gz", hash = "sha256:57046131384043ed078692b85d86b76568a686266cc036b9b56b704466f803ce"}, ] [package.dependencies] @@ -811,13 +811,13 @@ files = [ [[package]] name = "httpcore" -version = "1.0.4" +version = "1.0.5" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-1.0.4-py3-none-any.whl", hash = "sha256:ac418c1db41bade2ad53ae2f3834a3a0f5ae76b56cf5aa497d2d033384fc7d73"}, - {file = "httpcore-1.0.4.tar.gz", hash = "sha256:cb2839ccfcba0d2d3c1131d3c3e26dfc327326fbe7a5dc0dbfe9f6c9151bb022"}, + {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, + {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, ] [package.dependencies] @@ -828,7 +828,7 @@ h11 = ">=0.13,<0.15" asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<0.25.0)"] +trio = ["trio (>=0.22.0,<0.26.0)"] [[package]] name = "httpx" @@ -878,13 +878,13 @@ files = [ [[package]] name = "ipykernel" -version = "6.29.3" +version = "6.29.4" description = "IPython Kernel for Jupyter" optional = false python-versions = ">=3.8" files = [ - {file = "ipykernel-6.29.3-py3-none-any.whl", hash = "sha256:5aa086a4175b0229d4eca211e181fb473ea78ffd9869af36ba7694c947302a21"}, - {file = "ipykernel-6.29.3.tar.gz", hash = "sha256:e14c250d1f9ea3989490225cc1a542781b095a18a19447fcf2b5eaf7d0ac5bd2"}, + {file = "ipykernel-6.29.4-py3-none-any.whl", hash = "sha256:1181e653d95c6808039c509ef8e67c4126b3b3af7781496c7cbfb5ed938a27da"}, + {file = "ipykernel-6.29.4.tar.gz", hash = "sha256:3d44070060f9475ac2092b760123fadf105d2e2493c24848b6691a7c4f42af5c"}, ] [package.dependencies] @@ -911,13 +911,13 @@ test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio [[package]] name = "ipython" -version = "8.22.2" +version = "8.23.0" description = "IPython: Productive Interactive Computing" optional = false python-versions = ">=3.10" files = [ - {file = "ipython-8.22.2-py3-none-any.whl", hash = "sha256:3c86f284c8f3d8f2b6c662f885c4889a91df7cd52056fd02b7d8d6195d7f56e9"}, - {file = "ipython-8.22.2.tar.gz", hash = "sha256:2dcaad9049f9056f1fef63514f176c7d41f930daa78d05b82a176202818f2c14"}, + {file = "ipython-8.23.0-py3-none-any.whl", hash = "sha256:07232af52a5ba146dc3372c7bf52a0f890a23edf38d77caef8d53f9cdc2584c1"}, + {file = "ipython-8.23.0.tar.gz", hash = "sha256:7468edaf4f6de3e1b912e57f66c241e6fd3c7099f2ec2136e239e142e800274d"}, ] [package.dependencies] @@ -930,12 +930,14 @@ prompt-toolkit = ">=3.0.41,<3.1.0" pygments = ">=2.4.0" stack-data = "*" traitlets = ">=5.13.0" +typing-extensions = {version = "*", markers = "python_version < \"3.12\""} [package.extras] -all = ["ipython[black,doc,kernel,nbconvert,nbformat,notebook,parallel,qtconsole,terminal]", "ipython[test,test-extra]"] +all = ["ipython[black,doc,kernel,matplotlib,nbconvert,nbformat,notebook,parallel,qtconsole]", "ipython[test,test-extra]"] black = ["black"] doc = ["docrepr", "exceptiongroup", "ipykernel", "ipython[test]", "matplotlib", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "sphinxcontrib-jquery", "stack-data", "typing-extensions"] kernel = ["ipykernel"] +matplotlib = ["matplotlib"] nbconvert = ["nbconvert"] nbformat = ["nbformat"] notebook = ["ipywidgets", "notebook"] @@ -1034,7 +1036,6 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, - {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, ] [[package]] @@ -1262,19 +1263,19 @@ test = ["jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-jupyter[server] (> [[package]] name = "jupyterlab" -version = "4.1.5" +version = "4.1.6" description = "JupyterLab computational environment" optional = false python-versions = ">=3.8" files = [ - {file = "jupyterlab-4.1.5-py3-none-any.whl", hash = "sha256:3bc843382a25e1ab7bc31d9e39295a9f0463626692b7995597709c0ab236ab2c"}, - {file = "jupyterlab-4.1.5.tar.gz", hash = "sha256:c9ad75290cb10bfaff3624bf3fbb852319b4cce4c456613f8ebbaa98d03524db"}, + {file = "jupyterlab-4.1.6-py3-none-any.whl", hash = "sha256:cf3e862bc10dbf4331e4eb37438634f813c238cfc62c71c640b3b3b2caa089a8"}, + {file = "jupyterlab-4.1.6.tar.gz", hash = "sha256:7935f36ba26eb615183a4f5c2bbca5791b5108ce2a00b5505f8cfd100d53648e"}, ] [package.dependencies] async-lru = ">=1.0.0" httpx = ">=0.25.0" -ipykernel = "*" +ipykernel = ">=6.5.0" jinja2 = ">=3.0.3" jupyter-core = "*" jupyter-lsp = ">=2.0.0" @@ -1290,6 +1291,7 @@ dev = ["build", "bump2version", "coverage", "hatch", "pre-commit", "pytest-cov", docs = ["jsx-lexer", "myst-parser", "pydata-sphinx-theme (>=0.13.0)", "pytest", "pytest-check-links", "pytest-jupyter", "sphinx (>=1.8,<7.3.0)", "sphinx-copybutton"] docs-screenshots = ["altair (==5.2.0)", "ipython (==8.16.1)", "ipywidgets (==8.1.1)", "jupyterlab-geojson (==3.4.0)", "jupyterlab-language-pack-zh-cn (==4.0.post6)", "matplotlib (==3.8.2)", "nbconvert (>=7.0.0)", "pandas (==2.2.0)", "scipy (==1.12.0)", "vega-datasets (==0.9.0)"] test = ["coverage", "pytest (>=7.0)", "pytest-check-links (>=0.7)", "pytest-console-scripts", "pytest-cov", "pytest-jupyter (>=0.5.3)", "pytest-timeout", "pytest-tornasync", "requests", "requests-cache", "virtualenv"] +upgrade-extension = ["copier (>=8.0,<9.0)", "jinja2-time (<0.3)", "pydantic (<2.0)", "pyyaml-include (<2.0)", "tomli-w (<2.0)"] [[package]] name = "jupyterlab-pygments" @@ -1304,13 +1306,13 @@ files = [ [[package]] name = "jupyterlab-server" -version = "2.25.4" +version = "2.26.0" description = "A set of server components for JupyterLab and JupyterLab like applications." optional = false python-versions = ">=3.8" files = [ - {file = "jupyterlab_server-2.25.4-py3-none-any.whl", hash = "sha256:eb645ecc8f9b24bac5decc7803b6d5363250e16ec5af814e516bc2c54dd88081"}, - {file = "jupyterlab_server-2.25.4.tar.gz", hash = "sha256:2098198e1e82e0db982440f9b5136175d73bea2cd42a6480aa6fd502cb23c4f9"}, + {file = "jupyterlab_server-2.26.0-py3-none-any.whl", hash = "sha256:54622cbd330526a385ee0c1fdccdff3a1e7219bf3e864a335284a1270a1973df"}, + {file = "jupyterlab_server-2.26.0.tar.gz", hash = "sha256:9b3ba91cf2837f7f124fca36d63f3ca80ace2bed4898a63dd47e6598c1ab006f"}, ] [package.dependencies] @@ -1338,15 +1340,33 @@ files = [ {file = "jupyterlab_widgets-3.0.10.tar.gz", hash = "sha256:04f2ac04976727e4f9d0fa91cdc2f1ab860f965e504c29dbd6a65c882c9d04c0"}, ] +[[package]] +name = "loguru" +version = "0.7.2" +description = "Python logging made (stupidly) simple" +optional = false +python-versions = ">=3.5" +files = [ + {file = "loguru-0.7.2-py3-none-any.whl", hash = "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb"}, + {file = "loguru-0.7.2.tar.gz", hash = "sha256:e671a53522515f34fd406340ee968cb9ecafbc4b36c679da03c18fd8d0bd51ac"}, +] + +[package.dependencies] +colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} +win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} + +[package.extras] +dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.4.1)", "mypy (==v1.5.1)", "pre-commit (==3.4.0)", "pytest (==6.1.2)", "pytest (==7.4.0)", "pytest-cov (==2.12.1)", "pytest-cov (==4.1.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.0.0)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.3.0)", "tox (==3.27.1)", "tox (==4.11.0)"] + [[package]] name = "markdown" -version = "3.5.2" +version = "3.6" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.8" files = [ - {file = "Markdown-3.5.2-py3-none-any.whl", hash = "sha256:d43323865d89fc0cb9b20c75fc8ad313af307cc087e84b657d9eec768eddeadd"}, - {file = "Markdown-3.5.2.tar.gz", hash = "sha256:e1ac7b3dc550ee80e602e71c1d168002f062e49f1b11e26a36264dafd4df2ef8"}, + {file = "Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f"}, + {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"}, ] [package.extras] @@ -1534,13 +1554,13 @@ mkdocs = ">=1.0.3" [[package]] name = "mkdocs-material" -version = "9.5.16" +version = "9.5.17" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.16-py3-none-any.whl", hash = "sha256:32fce3cd8ecbd5dca6e5887cc0cf5bc78707a36f7d0f6f1bbbe9edaf428b8055"}, - {file = "mkdocs_material-9.5.16.tar.gz", hash = "sha256:8b89b639592660f24657bb058de4aff0060cd0383148f8f51711201730f17503"}, + {file = "mkdocs_material-9.5.17-py3-none-any.whl", hash = "sha256:14a2a60119a785e70e765dd033e6211367aca9fc70230e577c1cf6a326949571"}, + {file = "mkdocs_material-9.5.17.tar.gz", hash = "sha256:06ae1275a72db1989cf6209de9e9ecdfbcfdbc24c58353877b2bb927dbe413e4"}, ] [package.dependencies] @@ -1588,13 +1608,13 @@ mkdocs = ">=1.2" [[package]] name = "mkdocstrings" -version = "0.24.1" +version = "0.24.3" description = "Automatic documentation from sources, for MkDocs." optional = false python-versions = ">=3.8" files = [ - {file = "mkdocstrings-0.24.1-py3-none-any.whl", hash = "sha256:b4206f9a2ca8a648e222d5a0ca1d36ba7dee53c88732818de183b536f9042b5d"}, - {file = "mkdocstrings-0.24.1.tar.gz", hash = "sha256:cc83f9a1c8724fc1be3c2fa071dd73d91ce902ef6a79710249ec8d0ee1064401"}, + {file = "mkdocstrings-0.24.3-py3-none-any.whl", hash = "sha256:5c9cf2a32958cd161d5428699b79c8b0988856b0d4a8c5baf8395fc1bf4087c3"}, + {file = "mkdocstrings-0.24.3.tar.gz", hash = "sha256:f327b234eb8d2551a306735436e157d0a22d45f79963c60a8b585d5f7a94c1d2"}, ] [package.dependencies] @@ -1614,19 +1634,18 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] [[package]] name = "mkdocstrings-python" -version = "1.9.0" +version = "1.9.2" description = "A Python handler for mkdocstrings." optional = false python-versions = ">=3.8" files = [ - {file = "mkdocstrings_python-1.9.0-py3-none-any.whl", hash = "sha256:fad27d7314b4ec9c0359a187b477fb94c65ef561fdae941dca1b717c59aae96f"}, - {file = "mkdocstrings_python-1.9.0.tar.gz", hash = "sha256:6e1a442367cf75d30cf69774cbb1ad02aebec58bfff26087439df4955efecfde"}, + {file = "mkdocstrings_python-1.9.2-py3-none-any.whl", hash = "sha256:96d82f6424e08db6245e4a15ca95619f4ecd0ddd254c0aa590d4181814e16ee5"}, + {file = "mkdocstrings_python-1.9.2.tar.gz", hash = "sha256:8546a103c9b22e1778c72c887696acc39a6635fedde3c912ce00f967518a8847"}, ] [package.dependencies] griffe = ">=0.37" -markdown = ">=3.3,<3.6" -mkdocstrings = ">=0.20" +mkdocstrings = ">=0.24.2" [[package]] name = "mypy" @@ -1687,18 +1706,18 @@ files = [ [[package]] name = "mypy-protobuf" -version = "3.5.0" +version = "3.6.0" description = "Generate mypy stub files from protobuf specs" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-protobuf-3.5.0.tar.gz", hash = "sha256:21f270da0a9792a9dac76b0df463c027e561664ab6973c59be4e4d064dfe67dc"}, - {file = "mypy_protobuf-3.5.0-py3-none-any.whl", hash = "sha256:0d0548c6b9a6faf14ce1a9ce2831c403a5c1f2a9363e85b1e2c51d5d57aa8393"}, + {file = "mypy-protobuf-3.6.0.tar.gz", hash = "sha256:02f242eb3409f66889f2b1a3aa58356ec4d909cdd0f93115622e9e70366eca3c"}, + {file = "mypy_protobuf-3.6.0-py3-none-any.whl", hash = "sha256:56176e4d569070e7350ea620262478b49b7efceba4103d468448f1d21492fd6c"}, ] [package.dependencies] -protobuf = ">=4.23.4" -types-protobuf = ">=4.23.0.2" +protobuf = ">=4.25.3" +types-protobuf = ">=4.24" [[package]] name = "nbclient" @@ -1724,13 +1743,13 @@ test = ["flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "nbconvert (>= [[package]] name = "nbconvert" -version = "7.16.2" +version = "7.16.3" description = "Converting Jupyter Notebooks (.ipynb files) to other formats. Output formats include asciidoc, html, latex, markdown, pdf, py, rst, script. nbconvert can be used both as a Python library (`import nbconvert`) or as a command line tool (invoked as `jupyter nbconvert ...`)." optional = false python-versions = ">=3.8" files = [ - {file = "nbconvert-7.16.2-py3-none-any.whl", hash = "sha256:0c01c23981a8de0220255706822c40b751438e32467d6a686e26be08ba784382"}, - {file = "nbconvert-7.16.2.tar.gz", hash = "sha256:8310edd41e1c43947e4ecf16614c61469ebc024898eb808cce0999860fc9fb16"}, + {file = "nbconvert-7.16.3-py3-none-any.whl", hash = "sha256:ddeff14beeeedf3dd0bc506623e41e4507e551736de59df69a91f86700292b3b"}, + {file = "nbconvert-7.16.3.tar.gz", hash = "sha256:a6733b78ce3d47c3f85e504998495b07e6ea9cf9bf6ec1c98dda63ec6ad19142"}, ] [package.dependencies] @@ -1756,24 +1775,24 @@ docs = ["ipykernel", "ipython", "myst-parser", "nbsphinx (>=0.2.12)", "pydata-sp qtpdf = ["nbconvert[qtpng]"] qtpng = ["pyqtwebengine (>=5.15)"] serve = ["tornado (>=6.1)"] -test = ["flaky", "ipykernel", "ipywidgets (>=7.5)", "pytest"] +test = ["flaky", "ipykernel", "ipywidgets (>=7.5)", "pytest (>=7)"] webpdf = ["playwright"] [[package]] name = "nbformat" -version = "5.10.3" +version = "5.10.4" description = "The Jupyter Notebook format" optional = false python-versions = ">=3.8" files = [ - {file = "nbformat-5.10.3-py3-none-any.whl", hash = "sha256:d9476ca28676799af85385f409b49d95e199951477a159a576ef2a675151e5e8"}, - {file = "nbformat-5.10.3.tar.gz", hash = "sha256:60ed5e910ef7c6264b87d644f276b1b49e24011930deef54605188ddeb211685"}, + {file = "nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b"}, + {file = "nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a"}, ] [package.dependencies] -fastjsonschema = "*" +fastjsonschema = ">=2.15" jsonschema = ">=2.6" -jupyter-core = "*" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" traitlets = ">=5.1" [package.extras] @@ -1876,18 +1895,18 @@ files = [ [[package]] name = "parso" -version = "0.8.3" +version = "0.8.4" description = "A Python Parser" optional = false python-versions = ">=3.6" files = [ - {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, - {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, + {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"}, + {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"}, ] [package.extras] -qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] -testing = ["docopt", "pytest (<6.0.0)"] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] +testing = ["docopt", "pytest"] [[package]] name = "pathspec" @@ -2047,13 +2066,13 @@ tests = ["pytest"] [[package]] name = "pycparser" -version = "2.21" +version = "2.22" description = "C parser in Python" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" files = [ - {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, - {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] [[package]] @@ -2363,7 +2382,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2371,16 +2389,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2397,7 +2407,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2405,7 +2414,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2845,39 +2853,39 @@ files = [ [[package]] name = "ruff" -version = "0.3.4" +version = "0.3.5" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.3.4-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:60c870a7d46efcbc8385d27ec07fe534ac32f3b251e4fc44b3cbfd9e09609ef4"}, - {file = "ruff-0.3.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6fc14fa742e1d8f24910e1fff0bd5e26d395b0e0e04cc1b15c7c5e5fe5b4af91"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3ee7880f653cc03749a3bfea720cf2a192e4f884925b0cf7eecce82f0ce5854"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf133dd744f2470b347f602452a88e70dadfbe0fcfb5fd46e093d55da65f82f7"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f3860057590e810c7ffea75669bdc6927bfd91e29b4baa9258fd48b540a4365"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:986f2377f7cf12efac1f515fc1a5b753c000ed1e0a6de96747cdf2da20a1b369"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fd98e85869603e65f554fdc5cddf0712e352fe6e61d29d5a6fe087ec82b76c"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64abeed785dad51801b423fa51840b1764b35d6c461ea8caef9cf9e5e5ab34d9"}, - {file = "ruff-0.3.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df52972138318bc7546d92348a1ee58449bc3f9eaf0db278906eb511889c4b50"}, - {file = "ruff-0.3.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:98e98300056445ba2cc27d0b325fd044dc17fcc38e4e4d2c7711585bd0a958ed"}, - {file = "ruff-0.3.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:519cf6a0ebed244dce1dc8aecd3dc99add7a2ee15bb68cf19588bb5bf58e0488"}, - {file = "ruff-0.3.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bb0acfb921030d00070539c038cd24bb1df73a2981e9f55942514af8b17be94e"}, - {file = "ruff-0.3.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cf187a7e7098233d0d0c71175375c5162f880126c4c716fa28a8ac418dcf3378"}, - {file = "ruff-0.3.4-py3-none-win32.whl", hash = "sha256:af27ac187c0a331e8ef91d84bf1c3c6a5dea97e912a7560ac0cef25c526a4102"}, - {file = "ruff-0.3.4-py3-none-win_amd64.whl", hash = "sha256:de0d5069b165e5a32b3c6ffbb81c350b1e3d3483347196ffdf86dc0ef9e37dd6"}, - {file = "ruff-0.3.4-py3-none-win_arm64.whl", hash = "sha256:6810563cc08ad0096b57c717bd78aeac888a1bfd38654d9113cb3dc4d3f74232"}, - {file = "ruff-0.3.4.tar.gz", hash = "sha256:f0f4484c6541a99862b693e13a151435a279b271cff20e37101116a21e2a1ad1"}, + {file = "ruff-0.3.5-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:aef5bd3b89e657007e1be6b16553c8813b221ff6d92c7526b7e0227450981eac"}, + {file = "ruff-0.3.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:89b1e92b3bd9fca249153a97d23f29bed3992cff414b222fcd361d763fc53f12"}, + {file = "ruff-0.3.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e55771559c89272c3ebab23326dc23e7f813e492052391fe7950c1a5a139d89"}, + {file = "ruff-0.3.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dabc62195bf54b8a7876add6e789caae0268f34582333cda340497c886111c39"}, + {file = "ruff-0.3.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a05f3793ba25f194f395578579c546ca5d83e0195f992edc32e5907d142bfa3"}, + {file = "ruff-0.3.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:dfd3504e881082959b4160ab02f7a205f0fadc0a9619cc481982b6837b2fd4c0"}, + {file = "ruff-0.3.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87258e0d4b04046cf1d6cc1c56fadbf7a880cc3de1f7294938e923234cf9e498"}, + {file = "ruff-0.3.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:712e71283fc7d9f95047ed5f793bc019b0b0a29849b14664a60fd66c23b96da1"}, + {file = "ruff-0.3.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a532a90b4a18d3f722c124c513ffb5e5eaff0cc4f6d3aa4bda38e691b8600c9f"}, + {file = "ruff-0.3.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:122de171a147c76ada00f76df533b54676f6e321e61bd8656ae54be326c10296"}, + {file = "ruff-0.3.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d80a6b18a6c3b6ed25b71b05eba183f37d9bc8b16ace9e3d700997f00b74660b"}, + {file = "ruff-0.3.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a7b6e63194c68bca8e71f81de30cfa6f58ff70393cf45aab4c20f158227d5936"}, + {file = "ruff-0.3.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a759d33a20c72f2dfa54dae6e85e1225b8e302e8ac655773aff22e542a300985"}, + {file = "ruff-0.3.5-py3-none-win32.whl", hash = "sha256:9d8605aa990045517c911726d21293ef4baa64f87265896e491a05461cae078d"}, + {file = "ruff-0.3.5-py3-none-win_amd64.whl", hash = "sha256:dc56bb16a63c1303bd47563c60482a1512721053d93231cf7e9e1c6954395a0e"}, + {file = "ruff-0.3.5-py3-none-win_arm64.whl", hash = "sha256:faeeae9905446b975dcf6d4499dc93439b131f1443ee264055c5716dd947af55"}, + {file = "ruff-0.3.5.tar.gz", hash = "sha256:a067daaeb1dc2baf9b82a32dae67d154d95212080c80435eb052d95da647763d"}, ] [[package]] name = "send2trash" -version = "1.8.2" +version = "1.8.3" description = "Send file to trash natively under Mac OS X, Windows and Linux" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "Send2Trash-1.8.2-py3-none-any.whl", hash = "sha256:a384719d99c07ce1eefd6905d2decb6f8b7ed054025bb0e618919f945de4f679"}, - {file = "Send2Trash-1.8.2.tar.gz", hash = "sha256:c132d59fa44b9ca2b1699af5c86f57ce9f4c5eb56629d5d55fbb7a35f84e2312"}, + {file = "Send2Trash-1.8.3-py3-none-any.whl", hash = "sha256:0c31227e0bd08961c7665474a3d1ef7193929fedda4233843689baa056be46c9"}, + {file = "Send2Trash-1.8.3.tar.gz", hash = "sha256:b18e7a3966d99871aefeb00cfbcfdced55ce4871194810fc71f4aa484b953abf"}, ] [package.extras] @@ -3029,24 +3037,24 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "types-aiofiles" -version = "23.2.0.20240331" +version = "23.2.0.20240403" description = "Typing stubs for aiofiles" optional = false python-versions = ">=3.8" files = [ - {file = "types-aiofiles-23.2.0.20240331.tar.gz", hash = "sha256:8dd2a90cdcf16f9cfc9b07d77cd8eb989977c6bab2eba5b65e5929632ac77f53"}, - {file = "types_aiofiles-23.2.0.20240331-py3-none-any.whl", hash = "sha256:40c18c68956aaf1af6a37f997ae26ff6af124e52e16ec4f2473d51c720f79cd7"}, + {file = "types-aiofiles-23.2.0.20240403.tar.gz", hash = "sha256:1ffcf8f5f72b81f71139f754ea2610ab0017f27ba4fd771e187b07840ee49c0f"}, + {file = "types_aiofiles-23.2.0.20240403-py3-none-any.whl", hash = "sha256:adeeb4b999f19fda2dfe91c07857ff54701b6ee9b227b523a5a7be92125a2c5f"}, ] [[package]] name = "types-protobuf" -version = "4.24.0.20240311" +version = "4.24.0.20240408" description = "Typing stubs for protobuf" optional = false python-versions = ">=3.8" files = [ - {file = "types-protobuf-4.24.0.20240311.tar.gz", hash = "sha256:c80426f9fb9b21aee514691e96ab32a5cd694a82e2ac07964b352c3e7e0182bc"}, - {file = "types_protobuf-4.24.0.20240311-py3-none-any.whl", hash = "sha256:8e039486df058141cb221ab99f88c5878c08cca4376db1d84f63279860aa09cd"}, + {file = "types-protobuf-4.24.0.20240408.tar.gz", hash = "sha256:c03a44357b03c233c8c5864ce3e07dd9c766a00497d271496923f7ae3cb9e1de"}, + {file = "types_protobuf-4.24.0.20240408-py3-none-any.whl", hash = "sha256:9b87cd279378693071247227f52e89738af7c8d6f06dbdd749b0cf473c4916ce"}, ] [[package]] @@ -3062,13 +3070,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.11.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, ] [[package]] @@ -3207,7 +3215,21 @@ files = [ {file = "widgetsnbextension-4.0.10.tar.gz", hash = "sha256:64196c5ff3b9a9183a8e699a4227fb0b7002f252c814098e66c4d1cd0644688f"}, ] +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +optional = false +python-versions = ">=3.5" +files = [ + {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, + {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, +] + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.12" -content-hash = "4f63273dfbc62d26129dede17ac2fd09cc6ee7f57093884c1419d6c91b76e467" +content-hash = "38284c10cc30d1c02544dbd235ff4e3ecc6aa39402564bc6d011cf3f13b1a9ad" diff --git a/pyproject.toml b/pyproject.toml index 3b428c9..831a5cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,14 +10,14 @@ packages = [ [tool.poetry.dependencies] python = ">=3.11,<3.12" -grpcio = "^1.62.0" -grpcio-tools = "^1.62.0" -protobuf = "^4.25.3" -googleapis-common-protos = "^1.62.0" -pydantic = "^2.6.3" -pydantic-settings = "^2.2.1" -aiofiles = "^23.2.1" -pytest = "^8.1.1" +grpcio = "^1.62" +grpcio-tools = "^1.62" +googleapis-common-protos = "^1.62" +protobuf = "^4.25" +pydantic = "^2.6" +pydantic-settings = "^2.2" +aiofiles = "^23.2" +loguru = "^0.7" [tool.poetry.group.dev.dependencies] mypy = ">=1" @@ -27,19 +27,16 @@ types-protobuf = ">=4" mypy-protobuf = ">=3" pytest = ">=8" pytest-cov = ">=4" -types-aiofiles = "^23.2.0.20240331" -jupyter = "^1.0.0" +pytest-asyncio = ">=0" +types-aiofiles = ">=23" +jupyter = ">=1" [tool.poetry.group.docs.dependencies] -mkdocstrings-python = "^1.8.0" -mkdocs-gen-files = "^0.5.0" -mkdocs-literate-nav = "^0.6.1" -mkdocs-section-index ="^0.3.8" -mkdocs-material = "^9.5.16" - - -[tool.poetry.group.deb.dependencies] -pytest-asyncio = "^0.23.5.post1" +mkdocstrings-python = "^1" +mkdocs-gen-files = "^0" +mkdocs-literate-nav = "^0" +mkdocs-section-index ="^0" +mkdocs-material = "^9" [tool.mypy] strict = true @@ -75,6 +72,9 @@ exclude = [ "__pypackages__", ] +[tool.ruff.format] +docstring-code-format = true + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From aafceb985f1ac4bd0520fc0381cc606390804ef8 Mon Sep 17 00:00:00 2001 From: Nikita Barskov Date: Tue, 9 Apr 2024 00:36:24 +0200 Subject: [PATCH 2/2] Minor changes to Materials CSV source - Move `MaterialsSource` to `csv/base.py` because we don't have any others implementations so far, - Update types in annotations, - Add `delimiter` field to CSV Source (we can expect a customer using tabs, commas and etc), - Add additional validation logic when reading a file (a read will fail if a file does not contain a required column), - Minor improvements and test changes. --- src/volur/sdk/client.py | 2 +- src/volur/sdk/sources/__init__.py | 13 --- src/volur/sdk/sources/csv/asynchronous.py | 119 ++++++++++++---------- src/volur/sdk/sources/csv/base.py | 28 +++++ src/volur/sdk/sources/csv/shared.py | 78 +++++++++++--- src/volur/sdk/sources/csv/synchronous.py | 8 +- tests/test_sources/csv/test_shared.py | 29 ++++-- tests/test_upload_materials.py | 8 +- 8 files changed, 184 insertions(+), 101 deletions(-) create mode 100644 src/volur/sdk/sources/csv/base.py diff --git a/src/volur/sdk/client.py b/src/volur/sdk/client.py index c24af08..24f48f9 100644 --- a/src/volur/sdk/client.py +++ b/src/volur/sdk/client.py @@ -5,7 +5,7 @@ from pydantic import ValidationError from pydantic_settings import BaseSettings, SettingsConfigDict from volur.pork.materials.v1alpha3 import material_pb2, material_pb2_grpc -from volur.sdk.sources import MaterialSource +from volur.sdk.sources.csv.base import MaterialSource class VolurClientSettings(BaseSettings): diff --git a/src/volur/sdk/sources/__init__.py b/src/volur/sdk/sources/__init__.py index 00f28a9..e69de29 100644 --- a/src/volur/sdk/sources/__init__.py +++ b/src/volur/sdk/sources/__init__.py @@ -1,13 +0,0 @@ -import abc -from typing import Iterator - -from pydantic import BaseModel -from volur.pork.materials.v1alpha3 import material_pb2 - - -class MaterialSource(BaseModel): - @abc.abstractmethod - def __iter__(self: "MaterialSource") -> Iterator[material_pb2.Material]: ... # type: ignore[override] - - @abc.abstractmethod - def __next__(self: "MaterialSource") -> material_pb2.Material: ... diff --git a/src/volur/sdk/sources/csv/asynchronous.py b/src/volur/sdk/sources/csv/asynchronous.py index f8b497c..e2d8259 100644 --- a/src/volur/sdk/sources/csv/asynchronous.py +++ b/src/volur/sdk/sources/csv/asynchronous.py @@ -1,16 +1,13 @@ -import csv -from pathlib import Path -from typing import Any, AsyncIterator, Iterator +from typing import AsyncIterator, Iterator -import aiofiles +from loguru import logger from pydantic import Field from volur.pork.materials.v1alpha3 import material_pb2 -from volur.pork.materials.v1alpha3.material_pb2 import Material from volur.pork.shared.v1alpha1.characteristic_pb2 import ( Characteristic, ) from volur.pork.shared.v1alpha1.quantity_pb2 import Quantity -from volur.sdk.sources import MaterialSource +from volur.sdk.sources.csv.base import MaterialSource from volur.sdk.sources.csv.shared import ( CharacteristicColumn, Column, @@ -18,6 +15,7 @@ fetch_value, load_characteristic_value, load_quantity, + read, ) @@ -28,8 +26,11 @@ def __iter__(self: "MaterialSource") -> Iterator[material_pb2.Material]: # type def __next__(self: "MaterialSource") -> material_pb2.Material: raise NotImplementedError - _data: AsyncIterator[Material] | None = None + _data: AsyncIterator[material_pb2.Material] | None = None path: str + delimiter: str = Field( + ",", description="The separator used in the CSV file. Default is comma." + ) material_id_column: Column = Field( ..., description=""" @@ -62,8 +63,8 @@ def __next__(self: "MaterialSource") -> material_pb2.Material: QuantityColumn(column_name="QUANTITY_LBS", unit="pound"), ], ) - characteristics_columns: list[CharacteristicColumn] | None = Field( - default=None, + characteristics_columns: list[CharacteristicColumn] = Field( + default_factory=list, description=""" The column names used for getting the material characteristics in the Material entity @@ -87,65 +88,73 @@ def __next__(self: "MaterialSource") -> material_pb2.Material: ], ) - def __aiter__(self: "MaterialsCSVFileAsyncSource") -> "MaterialsCSVFileAsyncSource": + def __aiter__( + self: "MaterialsCSVFileAsyncSource", + ) -> AsyncIterator[material_pb2.Material]: self._data = self._load() return self - async def __anext__(self: "MaterialsCSVFileAsyncSource") -> Material: + async def __anext__( + self: "MaterialsCSVFileAsyncSource", + ) -> material_pb2.Material: if self._data is None: self._data = self._load() - data = await self._data.__anext__() + data = await anext(self._data) if data is None: raise StopAsyncIteration() return data async def _load( self: "MaterialsCSVFileAsyncSource", - ) -> AsyncIterator[Material]: - _ = Path(self.path) - if not _.exists(): - raise ValueError("file does not exist") - if not _.is_file(): - raise ValueError("path is not a file") - async with aiofiles.open(_, mode="r") as source: - content = await source.read() - reader: Iterator[dict[str, Any]] = csv.DictReader(content.splitlines()) - for row in reader: - material = Material() + ) -> AsyncIterator[material_pb2.Material]: + logger.info("reading data from a CSV file") + reader = await read( + self.path, + [ + self.material_id_column, + self.plant_id_column, + self.quantity_column, + *self.characteristics_columns, + ], + self.delimiter, + ) + for _, row in enumerate(reader): + material = material_pb2.Material() + if ( + value := fetch_value( + row, + self.material_id_column, + ).value_string + ) is not None: + material.material_id = value + if self.plant_id_column: if ( value := fetch_value( row, - self.material_id_column, + self.plant_id_column, ).value_string ) is not None: - material.material_id = value - if self.plant_id_column: - if ( - value := fetch_value( - row, - self.plant_id_column, - ).value_string - ) is not None: - material.plant = value - if self.quantity_column: - quantity_value = load_quantity( - row.get(self.quantity_column.column_name), - self.quantity_column, - ) - quantity = Quantity() - quantity.value.CopyFrom(quantity_value) - material.quantity.CopyFrom(quantity) - if self.characteristics_columns: - material.characteristics.extend( - [ - Characteristic( - name=column.characteristic_name, - value=load_characteristic_value( - row.get(column.column_name), - column, - ), - ) - for column in self.characteristics_columns - ] - ) - yield material + material.plant = value + if self.quantity_column: + quantity_value = load_quantity( + row.get(self.quantity_column.column_name), + self.quantity_column, + ) + quantity = Quantity() + quantity.value.CopyFrom(quantity_value) + material.quantity.CopyFrom(quantity) + if self.characteristics_columns: + material.characteristics.extend( + [ + Characteristic( + name=column.characteristic_name, + value=load_characteristic_value( + row.get(column.column_name), + column, + ), + ) + for column in self.characteristics_columns + ] + ) + yield material + logger.info("finished reading data from a CSV file") diff --git a/src/volur/sdk/sources/csv/base.py b/src/volur/sdk/sources/csv/base.py new file mode 100644 index 0000000..4367854 --- /dev/null +++ b/src/volur/sdk/sources/csv/base.py @@ -0,0 +1,28 @@ +import abc +from typing import AsyncIterator, Iterator + +from pydantic import BaseModel +from volur.pork.materials.v1alpha3 import material_pb2 + + +class MaterialSource(BaseModel): + """ + Base class for material sources. + + This class in an abstract class that defines the interface for material CSV + source. + """ + + @abc.abstractmethod + def __iter__(self: "MaterialSource") -> Iterator[material_pb2.Material]: ... # type: ignore[override] + + @abc.abstractmethod + def __next__(self: "MaterialSource") -> material_pb2.Material: ... + + @abc.abstractmethod + def __aiter__(self: "MaterialSource") -> AsyncIterator[material_pb2.Material]: ... + + @abc.abstractmethod + async def __anext__( + self: "MaterialSource", + ) -> material_pb2.Material: ... diff --git a/src/volur/sdk/sources/csv/shared.py b/src/volur/sdk/sources/csv/shared.py index 8d7057e..b20234d 100644 --- a/src/volur/sdk/sources/csv/shared.py +++ b/src/volur/sdk/sources/csv/shared.py @@ -1,15 +1,14 @@ -from typing import Literal +import csv +from pathlib import Path +from typing import Iterator, Literal +import aiofiles from pydantic import BaseModel, Field from volur.pork.shared.v1alpha1 import characteristic_pb2, quantity_pb2 class Column(BaseModel): column_name: str - - -class CharacteristicColumn(Column): - characteristic_name: str data_type: Literal[ "string", "bool", @@ -17,7 +16,11 @@ class CharacteristicColumn(Column): "float", "datetime", "date", - ] + ] = Field("string", description="The data type of the column") + + +class CharacteristicColumn(Column): + characteristic_name: str def load_characteristic_value( @@ -25,7 +28,7 @@ def load_characteristic_value( column: CharacteristicColumn, ) -> characteristic_pb2.CharacteristicValue: if value is None: - raise ValueError("value is missing") + return characteristic_pb2.CharacteristicValue() if column.data_type == "string": return characteristic_pb2.CharacteristicValue(value_string=value) elif column.data_type == "integer": @@ -54,7 +57,7 @@ def load_quantity( column: QuantityColumn, ) -> quantity_pb2.QuantityValue: if value is None: - raise ValueError("value is missing") + return quantity_pb2.QuantityValue() try: if column.unit == "kilogram": return quantity_pb2.QuantityValue(kilogram=float(value)) @@ -93,12 +96,55 @@ class Value(BaseModel): def fetch_value(row: dict[str, str], column: Column) -> Value: value = row.get(column.column_name) if value is None: - raise ValueError(f"column {column.column_name} does not exist") - if value.isdigit(): - return Value(value_integer=int(value)) - elif value.replace(".", "", 1).isdigit(): - return Value(value_float=float(value)) - elif value.lower() in ["true", "false"]: - return Value(value_bool=bool(value)) - else: + return Value() + if column.data_type == "string": return Value(value_string=value) + elif column.data_type == "integer": + try: + _ = int(value) + return Value(value_integer=_) + except ValueError as error: + raise ValueError( + f"column {column.column_name} is not an integer" + ) from error + elif column.data_type == "float": + try: + _ = float(value) + return Value(value_float=_) + except ValueError as error: + raise ValueError(f"column {column.column_name} is not an float") from error + elif column.data_type == "bool": + if value.lower() in ["true", "false"]: + return Value(value_bool=bool(value)) + else: + raise ValueError(f"column {column.column_name} is not an bool") + else: + raise ValueError(f"unknown data type {column.data_type}") + + +async def read( + path: str, + columns: list[Column | None], + delimeter: str, +) -> Iterator[dict[str, str]]: + _ = Path(path) + if not _.exists(): + raise ValueError("file does not exist") + if not _.is_file(): + raise ValueError("path is not a file") + async with aiofiles.open(_, mode="r") as source: + content = await source.read() + reader = csv.DictReader( + content.splitlines(), + delimiter=delimeter, + ) + required_columns = set([_.column_name for _ in columns if _ is not None]) + columns_present_in_file = set(reader.fieldnames) # type: ignore[arg-type] + missing_columns = required_columns.difference( + columns_present_in_file, + ) + if missing_columns: + raise ValueError( + f"missing columns in " f"the csv file: {','.join(missing_columns)}" + ) + return reader diff --git a/src/volur/sdk/sources/csv/synchronous.py b/src/volur/sdk/sources/csv/synchronous.py index 95ab7a1..afe4ae3 100644 --- a/src/volur/sdk/sources/csv/synchronous.py +++ b/src/volur/sdk/sources/csv/synchronous.py @@ -1,11 +1,11 @@ import csv from pathlib import Path -from typing import Any, AsyncIterator, Iterator +from typing import Any, AsyncIterator, Coroutine, Iterator from pydantic import ConfigDict, Field from volur.pork.materials.v1alpha3 import material_pb2 from volur.pork.shared.v1alpha1 import characteristic_pb2, quantity_pb2 -from volur.sdk.sources import MaterialSource +from volur.sdk.sources.csv.base import MaterialSource from volur.sdk.sources.csv.shared import ( CharacteristicColumn, Column, @@ -20,7 +20,9 @@ class MaterialsCSVFileSource(MaterialSource): def __aiter__(self: "MaterialSource") -> AsyncIterator[material_pb2.Material]: raise NotImplementedError - def __anext__(self: "MaterialSource") -> material_pb2.Material: + def __anext__( + self: "MaterialSource", + ) -> Coroutine[None, None, material_pb2.Material]: raise NotImplementedError model_config = ConfigDict( diff --git a/tests/test_sources/csv/test_shared.py b/tests/test_sources/csv/test_shared.py index 06bcc96..a808af0 100644 --- a/tests/test_sources/csv/test_shared.py +++ b/tests/test_sources/csv/test_shared.py @@ -236,11 +236,9 @@ def test_quantity_column( column_name="value", unit="piece", ), - quantity_pb2.QuantityValue( - piece=2, - ), - True, - "value is missing", + quantity_pb2.QuantityValue(), + False, + "", ), ( "random-value-that-can-not-be-converted", @@ -299,7 +297,10 @@ def test_load_quantity( { "column_name": "1", }, - Column(column_name="column_name"), + Column( + column_name="column_name", + data_type="integer", + ), Value( value_integer=1, ), @@ -310,7 +311,10 @@ def test_load_quantity( { "column_name": "1.2345", }, - Column(column_name="column_name"), + Column( + column_name="column_name", + data_type="float", + ), Value( value_float=1.2345, ), @@ -321,7 +325,10 @@ def test_load_quantity( { "column_name": "True", }, - Column(column_name="column_name"), + Column( + column_name="column_name", + data_type="bool", + ), Value( value_bool=True, ), @@ -333,9 +340,9 @@ def test_load_quantity( "column_name": "string-column-value", }, Column(column_name="column-name-that-does-not-exist"), - None, - True, - "column column-name-that-does-not-exist does not exist", + Value(), + False, + "", ), ], ) diff --git a/tests/test_upload_materials.py b/tests/test_upload_materials.py index ad7ee80..4cbf5c9 100644 --- a/tests/test_upload_materials.py +++ b/tests/test_upload_materials.py @@ -1,5 +1,5 @@ # type: ignore -from typing import Any, Dict, Iterator +from typing import Any, AsyncIterator, Dict, Iterator import google.rpc.status_pb2 import grpc @@ -7,7 +7,7 @@ from pydantic import ConfigDict, Field from volur.pork.materials.v1alpha3 import material_pb2 from volur.sdk.client import VolurClient -from volur.sdk.sources import MaterialSource +from volur.sdk.sources.csv.base import MaterialSource class UploadMaterialInformationStub: @@ -34,6 +34,10 @@ def __call__( class FakeMaterialsCSVFileSource(MaterialSource): + def __aiter__(self: "MaterialSource") -> AsyncIterator[material_pb2.Material]: ... + + async def __anext__(self: "MaterialSource") -> material_pb2.Material: ... + model_config = ConfigDict( arbitrary_types_allowed=True, )