feat: removed pylint and added black and ruff and formatted codebase …

…using black and linting using ruff

set max line length as 100

.github/workflows/ci.yml

@@ -18,12 +18,12 @@ jobs:
       - name: Install dependencies
         run: |
           poetry install --with dev
-      - name: Run linter :pylint
+      - name: Run black
         run: |
-          poetry run pylint src
-      - name: Run mypy :type_checking
+          poetry run black --check .
+      - name: Run ruff
         run: |
-          poetry run mypy --install-types --non-interactive src
+          poetry run ruff check .
 
   test:
     needs: lint

.pre-commit-config.yaml

@@ -1,28 +1,35 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v4.6.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace
+
+  - repo: https://github.com/psf/black
+    rev: 24.8.0 # Use the latest version
+    hooks:
+      - id: black
+        args: ["--line-length=100"]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.5.6" # Use the latest version
+    hooks:
+      - id: ruff
+        args: ["--line-length=100"]
+
   - repo: local
     hooks:
-      - id: pylint
-        name: Run pylint
-        entry: poetry run pylint
-        language: system
-        types: [python]
-        args: ["src"]
-        stages: [commit]
       - id: pytest
         name: Run pytest
         entry: poetry run pytest
         language: system
         pass_filenames: false
         always_run: true
         stages: [push]
+
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.0"
+    rev: "v1.11.1"
     hooks:
       - id: mypy
         args: [--install-types, --non-interactive]

pyproject.toml

@@ -22,20 +22,26 @@ beautifulsoup4 = "^4.12.3"
 lxml = "^5.2.2"
 colorama = "^0.4.6"
 requests = "^2.32.3"
-pylint = "3.0.2"
 aiohttp = "^3.10.0"
 
 [tool.poetry.group.dev.dependencies]
 responses = "^0.13.4"
-pylint = "^3.0.2"
 mypy = "^1.10.0"
 pytest-cov = "^5.0.0"
 requests-mock = "^1.12.1"
 pre-commit = ">=2.15,<3.0"
 pytest-asyncio = "^0.23.8"
 pytest = "^8.3.2"
 aioresponses = "^0.7.6"
+black = "^24.8.0"
+ruff = "^0.5.6"
 
+[tool.ruff]
+line-length = 100
+select = ["E", "F", "I", "N"]
+
+[tool.black]
+line-length = 100
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

script.py

@@ -1,4 +1,5 @@
 import subprocess
 
+
 def post_install() -> None:
     subprocess.run(["poetry", "run", "pre-commit", "install"], check=True)

src/tiny_web_crawler/__init__.py

@@ -1,2 +1,4 @@
 from tiny_web_crawler.core.spider import Spider
 from tiny_web_crawler.core.spider_settings import SpiderSettings
+
+__all__ = ["Spider", "SpiderSettings"]