ENH semi-automatic parser (#141)

* feat&fix(jsonParser): refactoring

* feat(jsonParser): PDF to CSV

* feat(data): CdL

jsonParser/semi-automatic/converter.py

@@ -0,0 +1,34 @@
+import aspose.pdf as pdf
+import os
+import click
+from glob import glob
+
+@click.command()
+@click.option('--input', '-i', help='Input path with PDF files', required=True)
+def main(input:str):
+    if not os.path.isdir(input):
+        raise FileNotFoundError(f"'{input}' is not a folder or does not exist")
+    folder = input
+    folders = glob(f"{folder}/*.pdf")
+    for file_pdf in folders:
+        output_file = os.path.join(file_pdf.replace(".pdf", ".csv"))
+        print(f"Converting {file_pdf} to {output_file}...")
+        convert_PDF_to_CSV(file_pdf, output_file)
+
+def convert_PDF_to_CSV(infile:str, outfile:str):
+    # Load input PDF document
+    document = pdf.Document(infile)
+
+    # Initialize the ExcelSaveOptions
+    excelSaveOptions = pdf.ExcelSaveOptions()
+
+    # Set CSV format
+    excelSaveOptions.format= pdf.ExcelSaveOptions.ExcelFormat.CSV
+
+    # Convert the PDF to Comma-Separated Values
+    document.save(outfile, excelSaveOptions)
+
+    print("Rendering process completed")
+
+if __name__ == "__main__":
+    main()

jsonParser/semi-automatic/parse_election_results.py

@@ -1,17 +1,39 @@
-import csv
 import json
 import click
 import os
+import csv
+from typing import List
+from glob import glob
 
 @click.command()
-@click.option('--input', '-i', help='Input CSV file', required=True)
-@click.option('--output', '-o', help='Output JSON file', required=True)
+@click.option('--input', '-i', help='Input CSV file or path with CSV files', required=True)
+@click.option('--output', '-o', help='Output JSON file or folder', required=True)
 def main(input:str, output:str):
     check_file_exists(input)
 
+    input_files = []
+    output_files = []
+    if os.path.isdir(input):
+        input_files = glob(input + "/*.csv")
+        if len(input_files) == 0:
+            input_files = glob(input + "/*/*.csv")
+            if len(input_files) == 0:
+                raise FileNotFoundError(f"No CSV files found in '{input}'")
+        for file in input_files:
+            output_files.append(os.path.join(output, file.split("/")[-1].replace(".csv", ".json")))
+    else:
+        input_files.append(input)
+        output_files.append(output)
+
+    for input, output in zip(input_files, output_files):
+        print(f"Creating JSON file '{output}' from CSV file '{input}'...")
+        create_json_file(input, output)
+        print("#"*50)
+
+def create_json_file(input:str, output:str) -> None:
     # Read the CSV file
     with open(input, 'r', encoding='utf-8-sig') as csv_file:
-        csv_reader = csv.reader(csv_file, delimiter=';')
+        csv_reader = csv.reader(csv_file, delimiter=',')
         rows_list = create_list(csv_reader)
 
     # Create the dictionary that will contain the data
@@ -44,13 +66,12 @@ def main(input:str, output:str):
     # Save the JSON data to a file
     with open(output, 'w', encoding='utf-8') as json_file:
         json_file.write(json_output)
-        
+
 def check_file_exists(file_path:str) -> None:
-    if not os.path.isfile(file_path):
+    if not os.path.isfile(file_path) and not os.path.isdir(file_path):
         raise FileNotFoundError(f"File '{file_path}' not found")
 
 def create_list(csv_reader:List[str]) -> List[str]:
-
     """Create a list of rows from the CSV file
 
     Args:
@@ -60,10 +81,24 @@ def create_list(csv_reader:List[str]) -> List[str]:
         list: The list of rows of the CSV file
     """
     rows_list = []
-    for row in csv_reader:
-        # Replace \xa0 with a space in the entire row
-        row = [cell.replace('\xa0', ' ') for cell in row]
-        rows_list.append(row)
+    for line in csv_reader:
+        for i in range(len(line)):
+            line[i] = line[i].replace("\"", "").replace("\n", "").replace("\xa0", " ").replace("Evaluation Only. Created with Aspose.PDF. Copyright 2002-2023 Aspose Pty Ltd.", "")
+        if len(line) == 1 and (line[0] == "" or line[0] == "VOTI DI LISTA" or line[0] == "BIENNIO 2023/2025" or "ELEZIONI RAPPRESENTANTI" in line[0]):
+            continue
+        if line[0] == "VOTI DI LISTA" or line[0] == "BIENNIO 2023/2025" or "ELEZIONI RAPPRESENTANTI" in line[0]:
+            continue
+        line = [x.strip() for x in line if x.strip() != "" and "aequo" not in x.strip()]
+        if len(line) == 0:
+            continue
+        if "DIPARTIMENTO" in line[0]:
+            l = " ".join(line)
+            line = [l.split("-")[0].strip()]
+        if len(line) > 1:
+            if line[1] in line[0]:
+                line[0] = line[0].replace(line[1], "")
+                line.append(line[1])
+        rows_list.append(line)
     return rows_list
 
 def get_name_and_seats(rows_list:list, data:dict) -> list:
@@ -77,8 +112,10 @@ def get_name_and_seats(rows_list:list, data:dict) -> list:
         list: The list of rows of the CSV file
     """
     row = rows_list[0]
+    print(row)
     data["dipartimento"] = str(row[0])
     row = rows_list[1]
+    print(row)
     data["seggi_da_assegnare"] = row[1]
     rows_list = rows_list[4:]
     return rows_list
@@ -100,6 +137,7 @@ def get_list_information(rows_list: list, data: dict) -> list:
         if row[0].strip() == "TOTALE":
             data["liste"].append({"totale": int(row[1].strip())})
             break
+        print(row)
         lista = {
             "nome": str(row[0].strip()),
             "seggi": {
@@ -142,7 +180,7 @@ def get_votation_information(rows_list:list, data:dict) -> list:
         elif row[0].strip() == "VOTANTI":
             data["votanti"] = {
                 "totali": int(row[1].strip()),
-                "percentuale": float(row[4].strip().replace(",", ".")),
+                "percentuale": float(row[3].strip().replace(",", ".")),
                 "seggio_n_telematico": int(row[-1].strip())
             }
         elif row[0].strip() == "TOTALE ELETTORI AVENTI DIRITTO":
@@ -176,10 +214,10 @@ def get_candidates_information(rows_list:list, data:dict) -> None:
                 "lista": list_name,
                 "voti": {
                     "totali": int(row[1].strip()),
-                    "seggio_telematico": int(row[-4].strip())
+                    "seggio_telematico": int(row[-1].strip())
                 }
             }
-            if "ELETTO" in row[3].strip():
+            if "ELETTO" in row:
                 data["eletti"].append(candidate)
             else:
                 data["non_eletti"].append(candidate)

src/data/2023-2025/Corso di Laurea/Beni_culturali.json

@@ -0,0 +1,119 @@
+{
+    "schede": {
+        "bianche": {
+            "totali": 5,
+            "seggio_n_telematico": 5
+        },
+        "nulle": {
+            "totali": 0
+        },
+        "contestate": {
+            "totali": 0
+        }
+    },
+    "liste": [
+        {
+            "nome": "Beni Culturali Disum",
+            "seggi": {
+                "seggi_pieni": "9",
+                "resti": "0,0",
+                "seggi_ai_resti": "0",
+                "seggi_totali": "9"
+            },
+            "voti": {
+                "totali": "127",
+                "seggio_telematico": "127"
+            }
+        },
+        {
+            "totale": 127
+        }
+    ],
+    "eletti": [
+        {
+            "nominativo": "SANFILIPPO Lorenzo",
+            "lista": "Beni Culturali Disum",
+            "voti": {
+                "totali": 34,
+                "seggio_telematico": 34
+            }
+        },
+        {
+            "nominativo": "PETRALIA Salvatore Giuseppe",
+            "lista": "Beni Culturali Disum",
+            "voti": {
+                "totali": 34,
+                "seggio_telematico": 34
+            }
+        },
+        {
+            "nominativo": "CUSUMANO Francesca Maria",
+            "lista": "Beni Culturali Disum",
+            "voti": {
+                "totali": 13,
+                "seggio_telematico": 13
+            }
+        },
+        {
+            "nominativo": "CINO Anastasia",
+            "lista": "Beni Culturali Disum",
+            "voti": {
+                "totali": 2,
+                "seggio_telematico": 2
+            }
+        },
+        {
+            "nominativo": "RUGGIERI Edith Maria Gae",
+            "lista": "Beni Culturali Disum",
+            "voti": {
+                "totali": 5,
+                "seggio_telematico": 5
+            }
+        },
+        {
+            "nominativo": "DI STEFANO Agnese Pia",
+            "lista": "Beni Culturali Disum",
+            "voti": {
+                "totali": 2,
+                "seggio_telematico": 2
+            }
+        },
+        {
+            "nominativo": "ASSENNATO Concetta Ambra",
+            "lista": "Beni Culturali Disum",
+            "voti": {
+                "totali": 8,
+                "seggio_telematico": 8
+            }
+        },
+        {
+            "nominativo": "LO PIERO William",
+            "lista": "Beni Culturali Disum",
+            "voti": {
+                "totali": 16,
+                "seggio_telematico": 16
+            }
+        },
+        {
+            "nominativo": "BENTIVEGNA Riccardo",
+            "lista": "Beni Culturali Disum",
+            "voti": {
+                "totali": 9,
+                "seggio_telematico": 9
+            }
+        }
+    ],
+    "non_eletti": [],
+    "dipartimento": "BENI CULTURALI-L-1-Laurea Triennale (D.M.270/2004)",
+    "seggi_da_assegnare": "9",
+    "quoziente": 14.111,
+    "votanti": {
+        "totali": 132,
+        "percentuale": 21.09,
+        "seggio_n_telematico": 132
+    },
+    "elettori": {
+        "totali": 626,
+        "seggio_n_telematico": 626
+    }
+}