Now saving and displaying past outages reported by the app

.gitignore

@@ -271,3 +271,5 @@ pyrightconfig.json
 *.xhtml
 
 # End of https://www.toptal.com/developers/gitignore/api/python,flask,web
+
+tmp.py

app.py

@@ -21,16 +21,26 @@ def urlOfService(services, service):
         return services[service]["url"]
     return "NaN"
 
-def updateStatusService(services, service):
+def updateStatusService(services, service, session=None):
     url = urlOfService(services, service)
     if url == "NaN":
         print("[LOG]: You passed a service that is not tracked")
         return False
 
     print(f"[LOG]: HTTP request for {services[service]["url"]}")
-    response = requests.get(url).status_code < 400 # Starting 400 codes are error for HTTP GET
 
+    headers = {"User-Agent": "Mozilla/5.0 (X11; CrOS x86_64 12871.102.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.141 Safari/537.36"}
+    if session is not None:
+        # This line is sooooo slow
+        response = session.head(url, headers=headers).status_code < 400  # Using a session to speed up refresh in a batch
+    else:
+        # This line is sooooo slow
+        response = requests.head(url, headers=headers).status_code < 400 # Starting 400 codes are error for HTTP GET
+
+
+    print("[LOG]: got status ")
     jsonUtility.updateStatus(services, service, response)
+    dataReport.reportStatus(services, service)
 
     return response
 
@@ -44,8 +54,10 @@ def statusService(services, service):
 
 def refreshServices(services):
     print("[LOG]: Refreshing the services")
+    session = requests.Session()
+
     for service in services.keys():
-        updateStatusService(services, service)
+        updateStatusService(services, service, session)
 
 # Setup Scheduler to periodically check the status of the website
 scheduler = BackgroundScheduler()
@@ -133,9 +145,17 @@ def page_not_found(error):
 @app.route("/extract")
 def extractLog():
     get_what_to_extract = request.args.get("get")
+    print(get_what_to_extract.split("_"))
 
-    if get_what_to_extract in services.keys() or get_what_to_extract == "request":
-        with open("data/" + get_what_to_extract + "/log.csv", "r") as file:
+    if get_what_to_extract.split("_")[0] in services.keys() or get_what_to_extract == "request":
+        print("in keys")
+        if len(get_what_to_extract.split("_")) > 1:
+            # when we want to extract the past outages not the user outages
+            path = "data/" + get_what_to_extract.split("_")[0] + "/outageReport.csv"
+        else:
+            path = "data/" + get_what_to_extract + "/log.csv"
+
+        with open(path, "r") as file:
             csv_data = list(csv.reader(file, delimiter=","))
 
         response = make_response()
@@ -147,7 +167,7 @@ def extractLog():
 
         return response
     else:
-        return 404
+        return render_template("404.html")
 
 @app.route('/robots.txt')
 @app.route('/sitemap.xml')

data/404-Test/log.csv

@@ -7,3 +7,4 @@ date,UP
 2024-01-22T22:28:36,False
 2024-01-23T09:55:10,False
 2024-01-23T12:15:12,False
+2024-01-23T20:44:12,False

data/404-Test/outageReport.csv

@@ -0,0 +1,13 @@
+date,UP
+2024-01-24T07:38:21,False
+2024-01-24T07:39:17,False
+2024-01-24T07:41:35,False
+2024-01-24T07:42:45,False
+2024-01-24T07:43:26,False
+2024-01-24T07:45:07,False
+2024-01-24T07:51:12,False
+2024-01-24T07:56:47,False
+2024-01-24T07:59:04,False
+2024-01-24T08:00:38,False
+2024-01-24T08:06:13,False
+2024-01-24T08:07:51,False

data/ADE-Scheduler/outageReport.csv

@@ -0,0 +1,10 @@
+date,UP
+2024-01-24T07:39:40,True
+2024-01-24T07:41:58,True
+2024-01-24T07:43:48,True
+2024-01-24T07:45:30,True
+2024-01-24T07:51:34,True
+2024-01-24T07:57:10,True
+2024-01-24T07:59:27,True
+2024-01-24T08:01:00,True
+2024-01-24T08:06:36,True

data/ADE/outageReport.csv

@@ -0,0 +1,10 @@
+date,UP
+2024-01-24T07:39:39,True
+2024-01-24T07:41:57,True
+2024-01-24T07:43:48,True
+2024-01-24T07:45:29,True
+2024-01-24T07:51:34,True
+2024-01-24T07:57:10,True
+2024-01-24T07:59:26,True
+2024-01-24T08:01:00,True
+2024-01-24T08:06:35,True

data/Comproved/outageReport.csv

@@ -0,0 +1,8 @@
+date,UP
+2024-01-24T07:40:23,True
+2024-01-24T07:44:31,True
+2024-01-24T07:46:13,True
+2024-01-24T07:52:17,True
+2024-01-24T07:57:53,True
+2024-01-24T08:01:43,True
+2024-01-24T08:07:19,True

data/Gradescope/outageReport.csv

@@ -0,0 +1,8 @@
+date,UP
+2024-01-24T07:40:25,True
+2024-01-24T07:44:33,True
+2024-01-24T07:46:14,True
+2024-01-24T07:52:19,True
+2024-01-24T07:57:55,True
+2024-01-24T08:01:45,True
+2024-01-24T08:07:20,True

data/Intranet/outageReport.csv

@@ -0,0 +1,7 @@
+date,UP
+2024-01-24T07:40:47,True
+2024-01-24T07:46:15,True
+2024-01-24T07:52:19,True
+2024-01-24T07:57:56,True
+2024-01-24T08:01:45,True
+2024-01-24T08:07:21,True

data/LEPL1104/outageReport.csv

@@ -0,0 +1,7 @@
+date,UP
+2024-01-24T07:40:48,True
+2024-01-24T07:46:15,True
+2024-01-24T07:52:20,True
+2024-01-24T07:57:56,True
+2024-01-24T08:01:46,True
+2024-01-24T08:07:22,True

data/LEPL1201/outageReport.csv

@@ -0,0 +1,7 @@
+date,UP
+2024-01-24T07:40:49,True
+2024-01-24T07:46:16,True
+2024-01-24T07:52:20,True
+2024-01-24T07:57:57,True
+2024-01-24T08:01:46,True
+2024-01-24T08:07:22,True

data/Moodle/outageReport.csv

@@ -0,0 +1,5 @@
+date,UP
+2024-01-24T07:46:25,True
+2024-01-24T07:52:43,True
+2024-01-24T07:58:19,True
+2024-01-24T08:02:09,True

data/UCLouvain/outageReport.csv

@@ -0,0 +1,5 @@
+date,UP
+2024-01-24T07:46:26,True
+2024-01-24T07:52:43,True
+2024-01-24T07:58:20,True
+2024-01-24T08:02:09,True

dataReport.py

@@ -31,53 +31,66 @@ def addBlankCSV():
                 log.write(cols)
 
 
-def plot(service):
-    path = filepath + service + "/log.csv"
-    size = os.path.getsize(path)
+def plot(service, onlyOutageReport=False):
+    if not onlyOutageReport:
+        print("[LOG]: also plotting for user Report")
+        toPlot = dict(userReport = filepath + service + "/outageReport.csv")
+    else:
+        toPlot = dict(outageReport = filepath + service + "/outageReport.csv")
 
-    fig,ax = plt.subplots()
 
-    if len(cols) + 5 >= size:
-        # It means we have just created the skeleton of the logs and there is no current logs
-        text_kwargs = dict(ha='center', va='center', fontsize=28)
+    for report in toPlot:
+        path = toPlot[report]
 
-        fig.text(0.5, 0.5, 'No Data at The moment', **text_kwargs)
-    else:
-        # Great there is some data !
-        # Define a custom converter for the date column
-        date_converter = lambda x: np.datetime64(x.decode("utf-8"))
+        size = os.path.getsize(path)
+
+        fig,ax = plt.subplots()
 
-        # Specify the data types and converters for each column
-        dtypes = np.dtype([("date", "datetime64[s]"), ("UP", "bool")])
-        converters = {"date": date_converter}
+        if len(cols) + 5 >= size:
+            # It means we have just created the skeleton of the logs and there is no current logs
+            text_kwargs = dict(ha='center', va='center', fontsize=28)
+
+            fig.text(0.5, 0.5, 'No Data at The moment', **text_kwargs)
+        else:
+            # Great there is some data !
+            # Define a custom converter for the date column
+            date_converter = lambda x: np.datetime64(x.decode("utf-8"))
+
+            # Specify the data types and converters for each column
+            dtypes = np.dtype([("date", "datetime64[s]"), ("UP", "bool")])
+            converters = {"date": date_converter}
 
-        # Load the CSV file into a NumPy array
-        data = np.genfromtxt(path, delimiter=",", names=True, dtype=dtypes, converters=converters)
+            # Load the CSV file into a NumPy array
+            data = np.genfromtxt(path, delimiter=",", names=True, dtype=dtypes, converters=converters)
 
-        # Print the loaded data
-        timeArray = data["date"]
-        UPArray = data["UP"]
-
-        timeNow = np.datetime64("now")
-
-        ax.plot(timeArray, UPArray, marker = "o")
-        ax.axvline(x=timeNow, linestyle="--", color="gray", alpha=0.5)
-
-        now_kwargs = dict(color="red",ha='left', va='bottom')
-
-        # To protect against unwanted behavior
-        if timeArray.size > 2: 
-            ax.set_xlim(timeArray[0] - np.timedelta64(3, "m"), timeNow + np.timedelta64(3, "m"))
-            ax.text(timeNow - np.timedelta64((timeNow - timeArray[0]))*0.34 , 0.5, f"Last Report\n{timeNow}", **now_kwargs)
+            # Print the loaded data
+            timeArray = data["date"]
+            UPArray = data["UP"]
+
+            timeNow = np.datetime64("now")
+
+            ax.plot(timeArray, UPArray, marker = "o")
+            ax.axvline(x=timeNow, linestyle="--", color="gray", alpha=0.5)
+
+            now_kwargs = dict(color="red",ha='left', va='bottom')
+
+            # To protect against unwanted behavior
+            if timeArray.size > 2: 
+                ax.set_xlim(timeArray[0] - np.timedelta64(3, "m"), timeNow + np.timedelta64(3, "m"))
+                ax.text(timeNow - np.timedelta64((timeNow - timeArray[0]))*0.34 , 0.5, f"Last Report\n{timeNow}", **now_kwargs)
+            else:
+                ax.text(timeNow, 0.5, f"Last Report\n{timeNow}", **now_kwargs)
+
+            if report == "userReport":
+                ax.set_title(f"Status reported by users for {service}")
+            else:
+                ax.set_title(f"Past status for {service}")
+            ax.set_ylabel("Up or Down")
+            ax.set_xlabel("Date and Time")
+        if not onlyOutageReport:
+            toPlot = dict(userReport = filepath + service + "/outageReport.csv")
         else:
-            ax.text(timeNow, 0.5, f"Last Report\n{timeNow}", **now_kwargs)
-
-
-        ax.set_title(f"Status reported by users for {service}")
-        ax.set_ylabel("Up or Down")
-        ax.set_xlabel("Date and Time")
-
-    fig.savefig("static/img/log/" + service + ".png")  
+            fig.savefig("static/img/log/" + report + service + ".png")  
 
 def addReport(service, user_choice):
     """Function to add a new line to the logs of each service
@@ -130,6 +143,46 @@ def newRequest(serviceName, url, info):
     with open(log, "a") as file:
         file.write(date + "," + serviceName + "," + url + "," + info + "\n")
 
+def blankStatus():
+    for service in serviceList:
+        try:
+            os.mkdir(filepath + service)        
+
+        except FileExistsError:
+            pass 
+        except:
+            raise ValueError(f"[LOG]: Something went wrong with creating the folder {service}")
+
+    for service in serviceList:
+        with open(filepath + service + "/outageReport.csv", "w") as log:
+                log.write(cols)
+
+def reportStatus(services, service):
+    """Add the log of the current status of the site so we can track it throughout time
+
+    Args:
+        services (dict): the loaded services.json
+        service (string): the specific service we want to report
+    """
+
+    print("[LOG]: starting Report")
+    date = services[service]["Last access time"]
+    UP = services[service]["Last status"]
+
+    path = filepath + service + "/outageReport.csv"
+
+    with open(path, "a") as out:
+        out.write(date + "," + str(UP) + "\n")
+
+    print("[LOG]: Finished Report")
+    print("[LOG]: starting plot")
+
+
+    plot(service, True)
+    print("[LOG]: Finished plot")
+
+
+
 
 if __name__ == "__main__":
     dataExtraction()

services.json

@@ -1,51 +1,51 @@
 {
     "404-Test": {
-        "Last access time": "2024-01-23T20:41:43",
+        "Last access time": "2024-01-24T08:07:51",
         "Last status": false,
         "url": "https://www.google.com/404"
     },
     "ADE": {
-        "Last access time": "2024-01-23T20:41:43",
+        "Last access time": "2024-01-24T08:06:35",
         "Last status": true,
         "url": "https://horaire.uclouvain.be/direct/"
     },
     "ADE-Scheduler": {
-        "Last access time": "2024-01-23T20:41:44",
+        "Last access time": "2024-01-24T08:06:36",
         "Last status": true,
         "url": "https://ade-scheduler.info.ucl.ac.be/calendar/"
     },
     "Comproved": {
-        "Last access time": "2024-01-23T20:41:45",
+        "Last access time": "2024-01-24T08:07:19",
         "Last status": true,
         "url": "https://app.comproved.com/universite-de-louvain"
     },
     "Gradescope": {
-        "Last access time": "2024-01-23T20:41:46",
+        "Last access time": "2024-01-24T08:07:20",
         "Last status": true,
         "url": "https://www.gradescope.com/"
     },
     "Intranet": {
-        "Last access time": "2024-01-23T20:41:47",
+        "Last access time": "2024-01-24T08:07:21",
         "Last status": true,
         "url": "https://intranet.uclouvain.be"
     },
     "LEPL1104": {
-        "Last access time": "2024-01-23T20:41:48",
+        "Last access time": "2024-01-24T08:07:22",
         "Last status": true,
         "url": "https://perso.uclouvain.be/vincent.legat/zouLab/epl1104.php"
     },
     "LEPL1201": {
-        "Last access time": "2024-01-23T20:41:24",
+        "Last access time": "2024-01-24T08:07:22",
         "Last status": true,
         "url": "https://perso.uclouvain.be/vincent.legat/zouLab/epl1201.php"
     },
     "Moodle": {
-        "Last access time": "2024-01-23T20:41:25",
+        "Last access time": "2024-01-24T08:02:09",
         "Last status": true,
         "url": "https://moodle.uclouvain.be/"
     },
     "UCLouvain": {
-        "Last access time": "2024-01-23T20:41:27",
+        "Last access time": "2024-01-24T08:02:09",
         "Last status": true,
         "url": "https://www.uclouvain.be/"
     }