From e69bb6a8ec90a7107a6ffc87214d6ab4cf3135e2 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 25 Jul 2024 13:22:26 -0400 Subject: [PATCH 001/314] Add django ORM --- backend/Dockerfile.python | 7 +- backend/requirements.txt | 2 + backend/serverless.yml | 12 - backend/src/api/functions.yml | 2 +- backend/src/api/python-app.py | 40 -- backend/src/xfd_django/manage.py | 22 + backend/src/xfd_django/xfd_api/__init__.py | 0 backend/src/xfd_django/xfd_api/admin.py | 3 + backend/src/xfd_django/xfd_api/apps.py | 6 + .../xfd_django/xfd_api/migrations/__init__.py | 0 backend/src/xfd_django/xfd_api/models.py | 478 ++++++++++++++++++ backend/src/xfd_django/xfd_api/tests.py | 3 + backend/src/xfd_django/xfd_api/views.py | 58 +++ backend/src/xfd_django/xfd_django/__init__.py | 0 backend/src/xfd_django/xfd_django/asgi.py | 49 ++ backend/src/xfd_django/xfd_django/settings.py | 151 ++++++ backend/src/xfd_django/xfd_django/urls.py | 22 + backend/src/xfd_django/xfd_django/wsgi.py | 16 + 18 files changed, 816 insertions(+), 55 deletions(-) delete mode 100644 backend/src/api/python-app.py create mode 100755 backend/src/xfd_django/manage.py create mode 100644 backend/src/xfd_django/xfd_api/__init__.py create mode 100644 backend/src/xfd_django/xfd_api/admin.py create mode 100644 backend/src/xfd_django/xfd_api/apps.py create mode 100644 backend/src/xfd_django/xfd_api/migrations/__init__.py create mode 100644 backend/src/xfd_django/xfd_api/models.py create mode 100644 backend/src/xfd_django/xfd_api/tests.py create mode 100644 backend/src/xfd_django/xfd_api/views.py create mode 100644 backend/src/xfd_django/xfd_django/__init__.py create mode 100644 backend/src/xfd_django/xfd_django/asgi.py create mode 100644 backend/src/xfd_django/xfd_django/settings.py create mode 100644 backend/src/xfd_django/xfd_django/urls.py create mode 100644 backend/src/xfd_django/xfd_django/wsgi.py diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 7e11f533..e7eb41c3 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -8,7 +8,10 @@ COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt # Copy the FastAPI application -COPY src/api/python-app.py . +COPY src/xfd_django . + +# Set environment variable +ENV DJANGO_SETTINGS_MODULE=xfd_django.settings # Command to run the FastAPI application -CMD ["uvicorn", "python-app:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "--workers", "4", "xfd_django.asgi:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] diff --git a/backend/requirements.txt b/backend/requirements.txt index b1ec7ec2..dfaa01ee 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,3 +1,5 @@ fastapi==0.111.0 mangum==0.17.0 uvicorn==0.30.1 +django +psycopg2-binary \ No newline at end of file diff --git a/backend/serverless.yml b/backend/serverless.yml index b346a5b9..9c6fa74d 100644 --- a/backend/serverless.yml +++ b/backend/serverless.yml @@ -33,10 +33,6 @@ provider: Principal: '*' Action: execute-api:Invoke Resource: execute-api:/${self:provider.stage}/*/* - Condition: - IpAddress: - aws:SourceIp: - - ${file(env.yml):${self:provider.stage}.DMZ_CIDR, ''} logs: restApi: true deploymentBucket: @@ -147,13 +143,6 @@ resources: VisibilityTimeout: 18000 # 5 hours MaximumMessageSize: 262144 # 256 KB MessageRetentionPeriod: 604800 # 7 days - XpanseQueue: - Type: AWS::SQS::Queue - Properties: - QueueName: ${self:provider.stage}-xpanse-queue - VisibilityTimeout: 18000 # 5 hours - MaximumMessageSize: 262144 # 256 KB - MessageRetentionPeriod: 604800 # 7 days functions: - ${file(./src/tasks/functions.yml)} @@ -163,4 +152,3 @@ plugins: - serverless-domain-manager - serverless-webpack - serverless-dotenv-plugin - - serverless-python-requirements diff --git a/backend/src/api/functions.yml b/backend/src/api/functions.yml index 481c4d1c..80270d47 100644 --- a/backend/src/api/functions.yml +++ b/backend/src/api/functions.yml @@ -21,7 +21,7 @@ api: # provisionedConcurrency: 1 fastApi: - handler: python-app.handler + handler: src/xfd_django/xfd_django/asgi.handler runtime: python3.11 events: - http: diff --git a/backend/src/api/python-app.py b/backend/src/api/python-app.py deleted file mode 100644 index 2adfba4c..00000000 --- a/backend/src/api/python-app.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -A simple FastAPI application with a healthcheck endpoint, designed to run on AWS Lambda using the Mangum adapter. - -Third-Party Libraries: -- fastapi: A modern, fast (high-performance), web framework for building APIs with Python 3.6+ based on standard Python type hints. -- mangum: An adapter for using ASGI applications with AWS Lambda & API Gateway. - -This application includes: -- A single healthcheck endpoint (`/healthcheck`) that returns a JSON response with the status of the application. - -Example usage: -1. Deploy this application to AWS Lambda using a serverless framework. -2. Access the healthcheck endpoint via the deployed API Gateway URL to verify the service is running. - -Dependencies: -- fastapi==0.111.0 -- mangum==0.17.0 -- uvicorn==0.30.1 (for local development) -""" - -# Third-Party Libraries -from fastapi import FastAPI -from mangum import Mangum - -app = FastAPI() - - -# Healthcheck endpoint -@app.get("/healthcheck") -async def healthcheck(): - """ - Healthcheck endpoint. - - Returns: - dict: A dictionary containing the health status of the application. - """ - return {"status": "ok"} - - -handler = Mangum(app) diff --git a/backend/src/xfd_django/manage.py b/backend/src/xfd_django/manage.py new file mode 100755 index 00000000..1d98fa5e --- /dev/null +++ b/backend/src/xfd_django/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'xfd_django.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/backend/src/xfd_django/xfd_api/__init__.py b/backend/src/xfd_django/xfd_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/admin.py b/backend/src/xfd_django/xfd_api/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/backend/src/xfd_django/xfd_api/apps.py b/backend/src/xfd_django/xfd_api/apps.py new file mode 100644 index 00000000..01d4d0dc --- /dev/null +++ b/backend/src/xfd_django/xfd_api/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class XfdApiConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'xfd_api' diff --git a/backend/src/xfd_django/xfd_api/migrations/__init__.py b/backend/src/xfd_django/xfd_api/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py new file mode 100644 index 00000000..cc352704 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/models.py @@ -0,0 +1,478 @@ +# This is an auto-generated Django model module. +# You'll have to do the following manually to clean this up: +# * Rearrange models' order +# * Make sure each model has one field with primary_key=True +# * Make sure each ForeignKey and OneToOneField has `on_delete` set to the desired behavior +# * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table +# Feel free to rename the models, but don't rename db_table values or field names. +from django.db import models + + +class ApiKey(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + lastused = models.DateTimeField(db_column='lastUsed', blank=True, null=True) # Field name made lowercase. + hashedkey = models.TextField(db_column='hashedKey') # Field name made lowercase. + lastfour = models.TextField(db_column='lastFour') # Field name made lowercase. + userid = models.ForeignKey('User', models.DO_NOTHING, db_column='userId', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'api_key' + + +class Assessment(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + rscid = models.CharField(db_column='rscId', unique=True) # Field name made lowercase. + type = models.CharField() + userid = models.ForeignKey('User', models.DO_NOTHING, db_column='userId', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'assessment' + + +class Category(models.Model): + id = models.UUIDField(primary_key=True) + name = models.CharField() + number = models.CharField(unique=True) + shortname = models.CharField(db_column='shortName', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'category' + + +class Cpe(models.Model): + id = models.UUIDField(primary_key=True) + name = models.CharField() + version = models.CharField() + vendor = models.CharField() + lastseenat = models.DateTimeField(db_column='lastSeenAt') # Field name made lowercase. + + class Meta: + managed = False + db_table = 'cpe' + unique_together = (('name', 'version', 'vendor'),) + + +class Cve(models.Model): + id = models.UUIDField(primary_key=True) + name = models.CharField(unique=True, blank=True, null=True) + publishedat = models.DateTimeField(db_column='publishedAt', blank=True, null=True) # Field name made lowercase. + modifiedat = models.DateTimeField(db_column='modifiedAt', blank=True, null=True) # Field name made lowercase. + status = models.CharField(blank=True, null=True) + description = models.CharField(blank=True, null=True) + cvssv2source = models.CharField(db_column='cvssV2Source', blank=True, null=True) # Field name made lowercase. + cvssv2type = models.CharField(db_column='cvssV2Type', blank=True, null=True) # Field name made lowercase. + cvssv2version = models.CharField(db_column='cvssV2Version', blank=True, null=True) # Field name made lowercase. + cvssv2vectorstring = models.CharField(db_column='cvssV2VectorString', blank=True, null=True) # Field name made lowercase. + cvssv2basescore = models.CharField(db_column='cvssV2BaseScore', blank=True, null=True) # Field name made lowercase. + cvssv2baseseverity = models.CharField(db_column='cvssV2BaseSeverity', blank=True, null=True) # Field name made lowercase. + cvssv2exploitabilityscore = models.CharField(db_column='cvssV2ExploitabilityScore', blank=True, null=True) # Field name made lowercase. + cvssv2impactscore = models.CharField(db_column='cvssV2ImpactScore', blank=True, null=True) # Field name made lowercase. + cvssv3source = models.CharField(db_column='cvssV3Source', blank=True, null=True) # Field name made lowercase. + cvssv3type = models.CharField(db_column='cvssV3Type', blank=True, null=True) # Field name made lowercase. + cvssv3version = models.CharField(db_column='cvssV3Version', blank=True, null=True) # Field name made lowercase. + cvssv3vectorstring = models.CharField(db_column='cvssV3VectorString', blank=True, null=True) # Field name made lowercase. + cvssv3basescore = models.CharField(db_column='cvssV3BaseScore', blank=True, null=True) # Field name made lowercase. + cvssv3baseseverity = models.CharField(db_column='cvssV3BaseSeverity', blank=True, null=True) # Field name made lowercase. + cvssv3exploitabilityscore = models.CharField(db_column='cvssV3ExploitabilityScore', blank=True, null=True) # Field name made lowercase. + cvssv3impactscore = models.CharField(db_column='cvssV3ImpactScore', blank=True, null=True) # Field name made lowercase. + cvssv4source = models.CharField(db_column='cvssV4Source', blank=True, null=True) # Field name made lowercase. + cvssv4type = models.CharField(db_column='cvssV4Type', blank=True, null=True) # Field name made lowercase. + cvssv4version = models.CharField(db_column='cvssV4Version', blank=True, null=True) # Field name made lowercase. + cvssv4vectorstring = models.CharField(db_column='cvssV4VectorString', blank=True, null=True) # Field name made lowercase. + cvssv4basescore = models.CharField(db_column='cvssV4BaseScore', blank=True, null=True) # Field name made lowercase. + cvssv4baseseverity = models.CharField(db_column='cvssV4BaseSeverity', blank=True, null=True) # Field name made lowercase. + cvssv4exploitabilityscore = models.CharField(db_column='cvssV4ExploitabilityScore', blank=True, null=True) # Field name made lowercase. + cvssv4impactscore = models.CharField(db_column='cvssV4ImpactScore', blank=True, null=True) # Field name made lowercase. + weaknesses = models.TextField(blank=True, null=True) + references = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'cve' + + +class CveCpesCpe(models.Model): + cveid = models.OneToOneField(Cve, models.DO_NOTHING, db_column='cveId', primary_key=True) # Field name made lowercase. The composite primary key (cveId, cpeId) found, that is not supported. The first column is selected. + cpeid = models.ForeignKey(Cpe, models.DO_NOTHING, db_column='cpeId') # Field name made lowercase. + + class Meta: + managed = False + db_table = 'cve_cpes_cpe' + unique_together = (('cveid', 'cpeid'),) + + +class Domain(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + syncedat = models.DateTimeField(db_column='syncedAt', blank=True, null=True) # Field name made lowercase. + ip = models.CharField(blank=True, null=True) + fromrootdomain = models.CharField(db_column='fromRootDomain', blank=True, null=True) # Field name made lowercase. + subdomainsource = models.CharField(db_column='subdomainSource', blank=True, null=True) # Field name made lowercase. + iponly = models.BooleanField(db_column='ipOnly', blank=True, null=True) # Field name made lowercase. + reversename = models.CharField(db_column='reverseName', max_length=512) # Field name made lowercase. + name = models.CharField(max_length=512) + screenshot = models.CharField(max_length=512, blank=True, null=True) + country = models.CharField(blank=True, null=True) + asn = models.CharField(blank=True, null=True) + cloudhosted = models.BooleanField(db_column='cloudHosted') # Field name made lowercase. + ssl = models.JSONField(blank=True, null=True) + censyscertificatesresults = models.JSONField(db_column='censysCertificatesResults') # Field name made lowercase. + trustymailresults = models.JSONField(db_column='trustymailResults') # Field name made lowercase. + discoveredbyid = models.ForeignKey('Scan', models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) # Field name made lowercase. + organizationid = models.ForeignKey('Organization', models.DO_NOTHING, db_column='organizationId') # Field name made lowercase. + + class Meta: + managed = False + db_table = 'domain' + unique_together = (('name', 'organizationid'),) + + +class Notification(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + startdatetime = models.DateTimeField(db_column='startDatetime', blank=True, null=True) # Field name made lowercase. + enddatetime = models.DateTimeField(db_column='endDatetime', blank=True, null=True) # Field name made lowercase. + maintenancetype = models.CharField(db_column='maintenanceType', blank=True, null=True) # Field name made lowercase. + status = models.CharField(blank=True, null=True) + updatedby = models.CharField(db_column='updatedBy', blank=True, null=True) # Field name made lowercase. + message = models.CharField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'notification' + + +class Organization(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + acronym = models.CharField(unique=True, blank=True, null=True) + name = models.CharField() + rootdomains = models.TextField(db_column='rootDomains') # Field name made lowercase. This field type is a guess. + ipblocks = models.TextField(db_column='ipBlocks') # Field name made lowercase. This field type is a guess. + ispassive = models.BooleanField(db_column='isPassive') # Field name made lowercase. + pendingdomains = models.TextField(db_column='pendingDomains') # Field name made lowercase. This field type is a guess. + country = models.CharField(blank=True, null=True) + state = models.CharField(blank=True, null=True) + regionid = models.CharField(db_column='regionId', blank=True, null=True) # Field name made lowercase. + statefips = models.IntegerField(db_column='stateFips', blank=True, null=True) # Field name made lowercase. + statename = models.CharField(db_column='stateName', blank=True, null=True) # Field name made lowercase. + county = models.CharField(blank=True, null=True) + countyfips = models.IntegerField(db_column='countyFips', blank=True, null=True) # Field name made lowercase. + type = models.CharField(blank=True, null=True) + parentid = models.ForeignKey('self', models.DO_NOTHING, db_column='parentId', blank=True, null=True) # Field name made lowercase. + createdbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'organization' + + +class OrganizationTag(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + name = models.CharField(unique=True) + + class Meta: + managed = False + db_table = 'organization_tag' + + +class OrganizationTagOrganizationsOrganization(models.Model): + organizationtagid = models.OneToOneField(OrganizationTag, models.DO_NOTHING, db_column='organizationTagId', primary_key=True) # Field name made lowercase. The composite primary key (organizationTagId, organizationId) found, that is not supported. The first column is selected. + organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') # Field name made lowercase. + + class Meta: + managed = False + db_table = 'organization_tag_organizations_organization' + unique_together = (('organizationtagid', 'organizationid'),) + + +class QueryResultCache(models.Model): + identifier = models.CharField(blank=True, null=True) + time = models.BigIntegerField() + duration = models.IntegerField() + query = models.TextField() + result = models.TextField() + + class Meta: + managed = False + db_table = 'query-result-cache' + + +class Question(models.Model): + id = models.UUIDField(primary_key=True) + name = models.CharField() + description = models.CharField(blank=True, null=True) + longform = models.CharField(db_column='longForm') # Field name made lowercase. + number = models.CharField() + categoryid = models.ForeignKey(Category, models.DO_NOTHING, db_column='categoryId', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'question' + unique_together = (('categoryid', 'number'),) + + +class QuestionResourcesResource(models.Model): + questionid = models.OneToOneField(Question, models.DO_NOTHING, db_column='questionId', primary_key=True) # Field name made lowercase. The composite primary key (questionId, resourceId) found, that is not supported. The first column is selected. + resourceid = models.ForeignKey('Resource', models.DO_NOTHING, db_column='resourceId') # Field name made lowercase. + + class Meta: + managed = False + db_table = 'question_resources_resource' + unique_together = (('questionid', 'resourceid'),) + + +class Resource(models.Model): + id = models.UUIDField(primary_key=True) + description = models.CharField() + name = models.CharField() + type = models.CharField() + url = models.CharField(unique=True) + + class Meta: + managed = False + db_table = 'resource' + + +class Response(models.Model): + id = models.UUIDField(primary_key=True) + selection = models.CharField() + assessmentid = models.ForeignKey(Assessment, models.DO_NOTHING, db_column='assessmentId', blank=True, null=True) # Field name made lowercase. + questionid = models.ForeignKey(Question, models.DO_NOTHING, db_column='questionId', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'response' + unique_together = (('assessmentid', 'questionid'),) + + +class Role(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + role = models.CharField() + approved = models.BooleanField() + createdbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) # Field name made lowercase. + approvedbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='approvedById', related_name='role_approvedbyid_set', blank=True, null=True) # Field name made lowercase. + userid = models.ForeignKey('User', models.DO_NOTHING, db_column='userId', related_name='role_userid_set', blank=True, null=True) # Field name made lowercase. + organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'role' + unique_together = (('userid', 'organizationid'),) + + +class SavedSearch(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + name = models.CharField() + searchterm = models.CharField(db_column='searchTerm') # Field name made lowercase. + sortdirection = models.CharField(db_column='sortDirection') # Field name made lowercase. + sortfield = models.CharField(db_column='sortField') # Field name made lowercase. + count = models.IntegerField() + filters = models.JSONField() + searchpath = models.CharField(db_column='searchPath') # Field name made lowercase. + createvulnerabilities = models.BooleanField(db_column='createVulnerabilities') # Field name made lowercase. + vulnerabilitytemplate = models.JSONField(db_column='vulnerabilityTemplate') # Field name made lowercase. + createdbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'saved_search' + + +class Scan(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + name = models.CharField() + arguments = models.TextField() # This field type is a guess. + frequency = models.IntegerField() + lastrun = models.DateTimeField(db_column='lastRun', blank=True, null=True) # Field name made lowercase. + isgranular = models.BooleanField(db_column='isGranular') # Field name made lowercase. + isusermodifiable = models.BooleanField(db_column='isUserModifiable', blank=True, null=True) # Field name made lowercase. + issinglescan = models.BooleanField(db_column='isSingleScan') # Field name made lowercase. + manualrunpending = models.BooleanField(db_column='manualRunPending') # Field name made lowercase. + createdbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'scan' + + +class ScanOrganizationsOrganization(models.Model): + scanid = models.OneToOneField(Scan, models.DO_NOTHING, db_column='scanId', primary_key=True) # Field name made lowercase. The composite primary key (scanId, organizationId) found, that is not supported. The first column is selected. + organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') # Field name made lowercase. + + class Meta: + managed = False + db_table = 'scan_organizations_organization' + unique_together = (('scanid', 'organizationid'),) + + +class ScanTagsOrganizationTag(models.Model): + scanid = models.OneToOneField(Scan, models.DO_NOTHING, db_column='scanId', primary_key=True) # Field name made lowercase. The composite primary key (scanId, organizationTagId) found, that is not supported. The first column is selected. + organizationtagid = models.ForeignKey(OrganizationTag, models.DO_NOTHING, db_column='organizationTagId') # Field name made lowercase. + + class Meta: + managed = False + db_table = 'scan_tags_organization_tag' + unique_together = (('scanid', 'organizationtagid'),) + + +class ScanTask(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + status = models.TextField() + type = models.TextField() + fargatetaskarn = models.TextField(db_column='fargateTaskArn', blank=True, null=True) # Field name made lowercase. + input = models.TextField(blank=True, null=True) + output = models.TextField(blank=True, null=True) + requestedat = models.DateTimeField(db_column='requestedAt', blank=True, null=True) # Field name made lowercase. + startedat = models.DateTimeField(db_column='startedAt', blank=True, null=True) # Field name made lowercase. + finishedat = models.DateTimeField(db_column='finishedAt', blank=True, null=True) # Field name made lowercase. + queuedat = models.DateTimeField(db_column='queuedAt', blank=True, null=True) # Field name made lowercase. + organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId', blank=True, null=True) # Field name made lowercase. + scanid = models.ForeignKey(Scan, models.DO_NOTHING, db_column='scanId', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'scan_task' + + +class ScanTaskOrganizationsOrganization(models.Model): + scantaskid = models.OneToOneField(ScanTask, models.DO_NOTHING, db_column='scanTaskId', primary_key=True) # Field name made lowercase. The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. + organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') # Field name made lowercase. + + class Meta: + managed = False + db_table = 'scan_task_organizations_organization' + unique_together = (('scantaskid', 'organizationid'),) + + +class Service(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + servicesource = models.TextField(db_column='serviceSource', blank=True, null=True) # Field name made lowercase. + port = models.IntegerField() + service = models.CharField(blank=True, null=True) + lastseen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) # Field name made lowercase. + banner = models.TextField(blank=True, null=True) + products = models.JSONField() + censysmetadata = models.JSONField(db_column='censysMetadata') # Field name made lowercase. + censysipv4results = models.JSONField(db_column='censysIpv4Results') # Field name made lowercase. + intrigueidentresults = models.JSONField(db_column='intrigueIdentResults') # Field name made lowercase. + shodanresults = models.JSONField(db_column='shodanResults') # Field name made lowercase. + wappalyzerresults = models.JSONField(db_column='wappalyzerResults') # Field name made lowercase. + domainid = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) # Field name made lowercase. + discoveredbyid = models.ForeignKey(Scan, models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'service' + unique_together = (('port', 'domainid'),) + + +class TypeormMetadata(models.Model): + type = models.CharField() + database = models.CharField(blank=True, null=True) + schema = models.CharField(blank=True, null=True) + table = models.CharField(blank=True, null=True) + name = models.CharField(blank=True, null=True) + value = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'typeorm_metadata' + + +class User(models.Model): + id = models.UUIDField(primary_key=True) + cognitoid = models.CharField(db_column='cognitoId', unique=True, blank=True, null=True) # Field name made lowercase. + logingovid = models.CharField(db_column='loginGovId', unique=True, blank=True, null=True) # Field name made lowercase. + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + firstname = models.CharField(db_column='firstName') # Field name made lowercase. + lastname = models.CharField(db_column='lastName') # Field name made lowercase. + fullname = models.CharField(db_column='fullName') # Field name made lowercase. + email = models.CharField(unique=True) + invitepending = models.BooleanField(db_column='invitePending') # Field name made lowercase. + loginblockedbymaintenance = models.BooleanField(db_column='loginBlockedByMaintenance') # Field name made lowercase. + dateacceptedterms = models.DateTimeField(db_column='dateAcceptedTerms', blank=True, null=True) # Field name made lowercase. + acceptedtermsversion = models.TextField(db_column='acceptedTermsVersion', blank=True, null=True) # Field name made lowercase. + lastloggedin = models.DateTimeField(db_column='lastLoggedIn', blank=True, null=True) # Field name made lowercase. + usertype = models.TextField(db_column='userType') # Field name made lowercase. + regionid = models.CharField(db_column='regionId', blank=True, null=True) # Field name made lowercase. + state = models.CharField(blank=True, null=True) + oktaid = models.CharField(db_column='oktaId', unique=True, blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'user' + + +class Vulnerability(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + lastseen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) # Field name made lowercase. + title = models.CharField() + cve = models.TextField(blank=True, null=True) + cwe = models.TextField(blank=True, null=True) + cpe = models.TextField(blank=True, null=True) + description = models.CharField() + references = models.JSONField() + cvss = models.DecimalField(max_digits=65535, decimal_places=65535, blank=True, null=True) + severity = models.TextField(blank=True, null=True) + needspopulation = models.BooleanField(db_column='needsPopulation') # Field name made lowercase. + state = models.CharField() + substate = models.CharField() + source = models.CharField() + notes = models.CharField() + actions = models.JSONField() + structureddata = models.JSONField(db_column='structuredData') # Field name made lowercase. + iskev = models.BooleanField(db_column='isKev', blank=True, null=True) # Field name made lowercase. + kevresults = models.JSONField(db_column='kevResults', blank=True, null=True) # Field name made lowercase. + domainid = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) # Field name made lowercase. + serviceid = models.ForeignKey(Service, models.DO_NOTHING, db_column='serviceId', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'vulnerability' + unique_together = (('domainid', 'title'),) + + +class Webpage(models.Model): + id = models.UUIDField(primary_key=True) + createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. + updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + syncedat = models.DateTimeField(db_column='syncedAt', blank=True, null=True) # Field name made lowercase. + lastseen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) # Field name made lowercase. + s3key = models.CharField(db_column='s3Key', blank=True, null=True) # Field name made lowercase. + url = models.CharField() + status = models.DecimalField(max_digits=65535, decimal_places=65535) + responsesize = models.DecimalField(db_column='responseSize', max_digits=65535, decimal_places=65535, blank=True, null=True) # Field name made lowercase. + headers = models.JSONField() + domainid = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) # Field name made lowercase. + discoveredbyid = models.ForeignKey(Scan, models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) # Field name made lowercase. + + class Meta: + managed = False + db_table = 'webpage' + unique_together = (('url', 'domainid'),) \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/tests.py b/backend/src/xfd_django/xfd_api/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py new file mode 100644 index 00000000..6fa0ca58 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/views.py @@ -0,0 +1,58 @@ + + +from django.shortcuts import render + +# Third party imports +from fastapi import ( + APIRouter, + Depends, + File, + HTTPException, + Request, + Security, + UploadFile, + status, +) +from .models import ApiKey + +api_router = APIRouter() + + +# Healthcheck endpoint +@api_router.get("/healthcheck") +async def healthcheck(): + """ + Healthcheck endpoint. + + Returns: + dict: A dictionary containing the health status of the application. + """ + return {"status": "ok2"} + + +@api_router.get("/apikeys") +async def get_api_keys(): + """ + Get all API keys. + + Returns: + list: A list of all API keys. + """ + try: + print("API KEYS!!!!!!") + api_keys = ApiKey.objects.all() + # return api_keys + return [ + { + "id": api_key.id, + "created_at": api_key.createdat, + "updated_at": api_key.updatedat, + "last_used": api_key.lastused, + "hashed_key": api_key.hashedkey, + "last_four": api_key.lastfour, + "user_id": api_key.userid.id if api_key.userid else None, + } + for api_key in api_keys + ] + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_django/__init__.py b/backend/src/xfd_django/xfd_django/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_django/asgi.py b/backend/src/xfd_django/xfd_django/asgi.py new file mode 100644 index 00000000..644aa174 --- /dev/null +++ b/backend/src/xfd_django/xfd_django/asgi.py @@ -0,0 +1,49 @@ +""" +ASGI config for xfd_django project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/ +""" +import os +import django +from django.apps import apps +from django.conf import settings +from django.core.asgi import get_asgi_application +from fastapi import FastAPI +from fastapi.middleware.wsgi import WSGIMiddleware +from mangum import Mangum +from starlette.middleware.cors import CORSMiddleware + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xfd_django.settings") +django.setup() + +# Import views after Django setup +from xfd_api.views import api_router + +application = get_asgi_application() + +# Below this comment is custom code +apps.populate(settings.INSTALLED_APPS) + + +def get_application() -> FastAPI: + """get_application function.""" + app = FastAPI(title=settings.PROJECT_NAME, debug=settings.DEBUG) + app.add_middleware( + CORSMiddleware, + allow_origins=settings.ALLOWED_HOSTS or ["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + app.include_router(api_router) + app.mount("/", WSGIMiddleware(get_asgi_application())) + + return app + +app = get_application() + +handler = Mangum(app) + \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_django/settings.py b/backend/src/xfd_django/xfd_django/settings.py new file mode 100644 index 00000000..925de8fa --- /dev/null +++ b/backend/src/xfd_django/xfd_django/settings.py @@ -0,0 +1,151 @@ +""" +Django settings for xfd_django project. + +Generated by 'django-admin startproject' using Django 4.2. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.2/ref/settings/ +""" + +# Standard Python Libraries +import mimetypes +import os + +# Python built-in +from pathlib import Path + +# Third-Party Libraries +from django.contrib.messages import constants as messages + +mimetypes.add_type("text/css", ".css", True) +mimetypes.add_type("text/html", ".html", True) + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +#TODO: GET THAT LATER +SECRET_KEY = 'django-insecure-255j80npx26z%x0@-7p@(qs9(yvtuuln#xuhxt_x$bbevvxnm!' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [".execute-api.us-east-1.amazonaws.com", 'https://api.staging-cd.crossfeed.cyber.dhs.gov'] + +MESSAGE_TAGS = { + messages.DEBUG: "alert-secondary", + messages.INFO: "alert-info", + messages.SUCCESS: "alert-success", + messages.WARNING: "alert-warning", + messages.ERROR: "alert-danger", +} + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "xfd_api.apps.XfdApiConfig", +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'xfd_django.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'xfd_django.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": "crossfeed", + "USER": "crossfeed", + "PASSWORD": "password", + "HOST": "db", + "PORT": "5432", + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.2/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +PROJECT_NAME = "XFD Python API" + +DJANGO_SETTINGS_MODULE = "xfd_django.settings" diff --git a/backend/src/xfd_django/xfd_django/urls.py b/backend/src/xfd_django/xfd_django/urls.py new file mode 100644 index 00000000..026b20cb --- /dev/null +++ b/backend/src/xfd_django/xfd_django/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for xfd_django project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path('admin/', admin.site.urls), +] diff --git a/backend/src/xfd_django/xfd_django/wsgi.py b/backend/src/xfd_django/xfd_django/wsgi.py new file mode 100644 index 00000000..14c13f58 --- /dev/null +++ b/backend/src/xfd_django/xfd_django/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for xfd_django project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'xfd_django.settings') + +application = get_wsgi_application() From 81fed975b3c31ef8daf89c9c138d70ecfca6f782 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 25 Jul 2024 15:48:59 -0400 Subject: [PATCH 002/314] Fix async error. NOTE: May want to only enable on local --- backend/src/xfd_django/xfd_django/asgi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/xfd_django/xfd_django/asgi.py b/backend/src/xfd_django/xfd_django/asgi.py index 644aa174..4dc03a83 100644 --- a/backend/src/xfd_django/xfd_django/asgi.py +++ b/backend/src/xfd_django/xfd_django/asgi.py @@ -17,6 +17,7 @@ from starlette.middleware.cors import CORSMiddleware os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xfd_django.settings") +os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true" django.setup() # Import views after Django setup From 4c530a6b77b5d79214c4284c3ba185c0a5c2e4d2 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Fri, 26 Jul 2024 11:40:10 -0400 Subject: [PATCH 003/314] Add authentication for organizations endpoint --- backend/requirements.txt | 3 +- backend/src/xfd_django/xfd_api/auth.py | 40 ++++++++++++++++++++ backend/src/xfd_django/xfd_api/jwt_utils.py | 26 +++++++++++++ backend/src/xfd_django/xfd_api/views.py | 42 +++++++++++++++++++-- 4 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/auth.py create mode 100644 backend/src/xfd_django/xfd_api/jwt_utils.py diff --git a/backend/requirements.txt b/backend/requirements.txt index dfaa01ee..1a7f0395 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,4 +2,5 @@ fastapi==0.111.0 mangum==0.17.0 uvicorn==0.30.1 django -psycopg2-binary \ No newline at end of file +psycopg2-binary +PyJWT \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py new file mode 100644 index 00000000..de6c2e74 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -0,0 +1,40 @@ +from fastapi import Depends, HTTPException, Security, status +from fastapi.security import APIKeyHeader, OAuth2PasswordBearer +from xfd_api.jwt_utils import decode_jwt_token +from xfd_api.models import ApiKey, User +from hashlib import sha256 + +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") +api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) + +def get_current_user(token: str = Depends(oauth2_scheme)): + user = decode_jwt_token(token) + if user is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired token", + ) + return user + +def get_user_by_api_key(api_key: str): + hashed_key = sha256(api_key.encode()).hexdigest() + try: + api_key_instance = ApiKey.objects.get(hashedkey=hashed_key) + return api_key_instance.userid + except ApiKey.DoesNotExist: + return None + +def get_current_active_user( + api_key: str = Security(api_key_header), + token: str = Depends(oauth2_scheme) +): + if api_key: + user = get_user_by_api_key(api_key) + else: + user = decode_jwt_token(token) + if user is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid authentication credentials", + ) + return user diff --git a/backend/src/xfd_django/xfd_api/jwt_utils.py b/backend/src/xfd_django/xfd_api/jwt_utils.py new file mode 100644 index 00000000..b4fe35db --- /dev/null +++ b/backend/src/xfd_django/xfd_api/jwt_utils.py @@ -0,0 +1,26 @@ +# xfd_api/jwt_utils.py + +import jwt +from datetime import datetime, timedelta +from django.conf import settings +from django.contrib.auth import get_user_model +from jwt import ExpiredSignatureError, InvalidTokenError + +User = get_user_model() +SECRET_KEY = settings.SECRET_KEY + +def create_jwt_token(user): + payload = { + 'id': str(user.id), + 'email': user.email, + 'exp': datetime.utcnow() + timedelta(hours=1) + } + return jwt.encode(payload, SECRET_KEY, algorithm='HS256') + +def decode_jwt_token(token): + try: + payload = jwt.decode(token, SECRET_KEY, algorithms=['HS256']) + user = User.objects.get(id=payload['id']) + return user + except (ExpiredSignatureError, InvalidTokenError, User.DoesNotExist): + return None diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 6fa0ca58..d89fa103 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -1,8 +1,6 @@ from django.shortcuts import render - -# Third party imports from fastapi import ( APIRouter, Depends, @@ -13,7 +11,8 @@ UploadFile, status, ) -from .models import ApiKey +from .auth import get_current_active_user +from .models import ApiKey, Organization, User api_router = APIRouter() @@ -39,7 +38,6 @@ async def get_api_keys(): list: A list of all API keys. """ try: - print("API KEYS!!!!!!") api_keys = ApiKey.objects.all() # return api_keys return [ @@ -54,5 +52,41 @@ async def get_api_keys(): } for api_key in api_keys ] + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.get("/organizations") +async def get_organizations(current_user: User = Depends(get_current_active_user)): + """ + Get all organizations. + Returns: + list: A list of all organizations. + """ + try: + organizations = Organization.objects.all() + return [ + { + "id": organization.id, + "name": organization.name, + "acronym": organization.acronym, + "root_domains": organization.rootdomains, + "ip_blocks": organization.ipblocks, + "is_passive": organization.ispassive, + "country": organization.country, + "state": organization.state, + "region_id": organization.regionid, + "state_fips": organization.statefips, + "state_name": organization.statename, + "county": organization.county, + "county_fips": organization.countyfips, + "type": organization.type, + "parent_id": organization.parentid.id if organization.parentid else None, + "created_by_id": organization.createdbyid.id if organization.createdbyid else None, + "created_at": organization.createdat, + "updated_at": organization.updatedat, + } + for organization in organizations + ] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) \ No newline at end of file From 1bee135002b309bd78b8200255516b8e9315f7a4 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Thu, 1 Aug 2024 08:36:29 -0500 Subject: [PATCH 004/314] Make aws-sdk a dev dependency. --- backend/package-lock.json | 139 ++++++++++++++++++++++++++++++++++++-- backend/package.json | 6 +- 2 files changed, 136 insertions(+), 9 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 14e9c4f8..61492653 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -4,13 +4,10 @@ "packages": { "": { "dependencies": { - "@aws-sdk/client-cloudwatch-logs": "^3.417.0", - "@aws-sdk/client-ssm": "^3.414.0", "@elastic/elasticsearch": "~7.10.0", "@thefaultvault/tfv-cpe-parser": "^1.3.0", "@types/dockerode": "^3.3.19", "amqplib": "^0.10.3", - "aws-sdk": "^2.1551.0", "axios": "^1.6", "bufferutil": "^4.0.7", "class-transformer": "^0.3.1", @@ -47,6 +44,8 @@ "ws": "^8.18.0" }, "devDependencies": { + "@aws-sdk/client-cloudwatch-logs": "^3.417.0", + "@aws-sdk/client-ssm": "^3.414.0", "@jest/globals": "^29", "@types/aws-lambda": "^8.10.62", "@types/cors": "^2.8.17", @@ -60,6 +59,7 @@ "@types/uuid": "^9.0.7", "@typescript-eslint/eslint-plugin": "^5.59", "@typescript-eslint/parser": "^5.59", + "aws-sdk": "^2.1551.0", "debug": "^4.3.4", "dockerode": "^3.3.1", "dotenv": "^16.0", @@ -150,11 +150,13 @@ "@aws-sdk/types": "^3.222.0", "tslib": "^1.11.1" }, + "dev": true, "integrity": "sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==", "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/crc32/node_modules/tslib": { + "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -180,11 +182,13 @@ "dependencies": { "tslib": "^1.11.1" }, + "dev": true, "integrity": "sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==", "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/ie11-detection/node_modules/tslib": { + "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -221,11 +225,13 @@ "@aws-sdk/util-utf8-browser": "^3.0.0", "tslib": "^1.11.1" }, + "dev": true, "integrity": "sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/sha256-browser/node_modules/tslib": { + "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -236,11 +242,13 @@ "@aws-sdk/types": "^3.222.0", "tslib": "^1.11.1" }, + "dev": true, "integrity": "sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/sha256-js/node_modules/tslib": { + "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -249,11 +257,13 @@ "dependencies": { "tslib": "^1.11.1" }, + "dev": true, "integrity": "sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==", "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/supports-web-crypto/node_modules/tslib": { + "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -264,11 +274,13 @@ "@aws-sdk/util-utf8-browser": "^3.0.0", "tslib": "^1.11.1" }, + "dev": true, "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==", "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/util/node_modules/tslib": { + "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -958,6 +970,7 @@ "tslib": "^2.5.0", "uuid": "^8.3.2" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1002,6 +1015,7 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1050,6 +1064,7 @@ "fast-xml-parser": "4.2.5", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1064,6 +1079,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1084,6 +1100,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1105,6 +1122,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1120,6 +1138,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1137,6 +1156,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1151,6 +1171,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1165,6 +1186,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1178,6 +1200,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1192,6 +1215,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1206,6 +1230,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1223,6 +1248,7 @@ "@smithy/util-middleware": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1238,6 +1264,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1283,6 +1310,7 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1295,6 +1323,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1307,6 +1336,7 @@ "@aws-sdk/types": "3.413.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1321,6 +1351,7 @@ "bowser": "^2.11.0", "tslib": "^2.5.0" }, + "dev": true, "integrity": "sha512-7j/qWcRO2OBZBre2fC6V6M0PAS9n7k6i+VtofPkkhxC2DZszLJElqnooF9hGmVGYK3zR47Np4WjURXKIEZclWg==", "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.413.0.tgz", "version": "3.413.0" @@ -1332,6 +1363,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1352,6 +1384,7 @@ "@smithy/types": "^2.3.3", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1363,6 +1396,7 @@ "bin": { "uuid": "dist/bin/uuid" }, + "dev": true, "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "version": "8.3.2" @@ -1572,6 +1606,7 @@ "tslib": "^2.5.0", "uuid": "^8.3.2" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1616,6 +1651,7 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1664,6 +1700,7 @@ "fast-xml-parser": "4.2.5", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1678,6 +1715,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1698,6 +1736,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1719,6 +1758,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1734,6 +1774,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1751,6 +1792,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1765,6 +1807,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1779,6 +1822,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1792,6 +1836,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1806,6 +1851,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1820,6 +1866,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1837,6 +1884,7 @@ "@smithy/util-middleware": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1852,6 +1900,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1897,6 +1946,7 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1909,6 +1959,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1921,6 +1972,7 @@ "@aws-sdk/types": "3.413.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1935,6 +1987,7 @@ "bowser": "^2.11.0", "tslib": "^2.5.0" }, + "dev": true, "integrity": "sha512-7j/qWcRO2OBZBre2fC6V6M0PAS9n7k6i+VtofPkkhxC2DZszLJElqnooF9hGmVGYK3zR47Np4WjURXKIEZclWg==", "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.413.0.tgz", "version": "3.413.0" @@ -1946,6 +1999,7 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1966,6 +2020,7 @@ "@smithy/types": "^2.3.3", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1977,6 +2032,7 @@ "bin": { "uuid": "dist/bin/uuid" }, + "dev": true, "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "version": "8.3.2" @@ -2592,6 +2648,7 @@ "@smithy/util-middleware": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -2890,6 +2947,7 @@ "@smithy/types": "^2.2.2", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -2926,6 +2984,7 @@ "dependencies": { "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -3008,6 +3067,7 @@ "dependencies": { "tslib": "^2.3.1" }, + "dev": true, "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", "version": "3.259.0" @@ -5587,6 +5647,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5621,6 +5682,7 @@ "@smithy/util-middleware": "^2.0.6", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5636,6 +5698,7 @@ "@smithy/url-parser": "^2.0.13", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5650,6 +5713,7 @@ "@smithy/util-hex-encoding": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "integrity": "sha512-iqR6OuOV3zbQK8uVs9o+9AxhVk8kW9NAxA71nugwUB+kTY9C35pUd0A5/m4PRT0Y0oIW7W4kgnSR3fdYXQjECw==", "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-2.0.5.tgz", "version": "2.0.5" @@ -5717,6 +5781,7 @@ "@smithy/util-base64": "^2.0.1", "tslib": "^2.5.0" }, + "dev": true, "integrity": "sha512-PStY3XO1Ksjwn3wMKye5U6m6zxXpXrXZYqLy/IeCbh3nM9QB3Jgw/B0PUSLUWKdXg4U8qgEu300e3ZoBvZLsDg==", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-2.2.6.tgz", "version": "2.2.6" @@ -5726,6 +5791,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5752,6 +5818,7 @@ "@smithy/util-utf8": "^2.0.2", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5778,6 +5845,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "integrity": "sha512-XsGYhVhvEikX1Yz0kyIoLssJf2Rs6E0U2w2YuKdT4jSra5A/g8V2oLROC1s56NldbgnpesTYB2z55KCHHbKyjw==", "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-2.0.13.tgz", "version": "2.0.13" @@ -5786,6 +5854,7 @@ "dependencies": { "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5810,6 +5879,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5822,6 +5892,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5839,6 +5910,7 @@ "@smithy/util-middleware": "^2.0.6", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5857,6 +5929,7 @@ "tslib": "^2.5.0", "uuid": "^8.3.2" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5869,6 +5942,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5880,6 +5954,7 @@ "bin": { "uuid": "dist/bin/uuid" }, + "dev": true, "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "version": "8.3.2" @@ -5889,6 +5964,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5901,6 +5977,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5915,6 +5992,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5930,6 +6008,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5942,6 +6021,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5954,6 +6034,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5980,6 +6061,7 @@ "@smithy/util-uri-escape": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5992,6 +6074,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6003,6 +6086,7 @@ "dependencies": { "@smithy/types": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6015,6 +6099,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6033,6 +6118,7 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6047,6 +6133,7 @@ "@smithy/util-stream": "^2.0.20", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6058,6 +6145,7 @@ "dependencies": { "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6071,6 +6159,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "integrity": "sha512-okWx2P/d9jcTsZWTVNnRMpFOE7fMkzloSFyM53fA7nLKJQObxM2T4JlZ5KitKKuXq7pxon9J6SF2kCwtdflIrA==", "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.13.tgz", "version": "2.0.13" @@ -6080,6 +6169,7 @@ "@smithy/util-buffer-from": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6091,6 +6181,7 @@ "dependencies": { "tslib": "^2.5.0" }, + "dev": true, "integrity": "sha512-JdDuS4ircJt+FDnaQj88TzZY3+njZ6O+D3uakS32f2VNnDo3vyEuNdBOh/oFd8Df1zSZOuH1HEChk2AOYDezZg==", "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-2.0.0.tgz", "version": "2.0.0" @@ -6099,6 +6190,7 @@ "dependencies": { "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6111,6 +6203,7 @@ "@smithy/is-array-buffer": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6122,6 +6215,7 @@ "dependencies": { "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6137,6 +6231,7 @@ "bowser": "^2.11.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">= 10.0.0" }, @@ -6154,6 +6249,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">= 10.0.0" }, @@ -6179,6 +6275,7 @@ "dependencies": { "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6191,6 +6288,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6204,6 +6302,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">= 14.0.0" }, @@ -6222,6 +6321,7 @@ "@smithy/util-utf8": "^2.0.2", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6233,6 +6333,7 @@ "dependencies": { "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6245,6 +6346,7 @@ "@smithy/util-buffer-from": "^2.0.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6258,6 +6360,7 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, + "dev": true, "engines": { "node": ">=14.0.0" }, @@ -7650,6 +7753,7 @@ "version": "1.0.0" }, "node_modules/available-typed-arrays": { + "dev": true, "engines": { "node": ">= 0.4" }, @@ -7673,17 +7777,20 @@ "uuid": "8.0.0", "xml2js": "0.6.2" }, + "dev": true, "engines": { "node": ">= 10.0.0" }, - "integrity": "sha512-nUaAzS7cheaKF8lV0AVJBqteuoYIgQ5UgpZaoRR44D7HA1f6iCYFISF6WH6d0hQvpxPDIXr5NlVt0cHyp/Sx1g==", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1551.0.tgz", - "version": "2.1551.0" + "hasInstallScript": true, + "integrity": "sha512-IhEcdGmiplF3l/pCROxEYIdi0s+LZ2VkbMAq3RgoXTHxY5cgqVRNaqsEsgIHev2Clxa9V08HttnIERTIUqb1+Q==", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1665.0.tgz", + "version": "2.1665.0" }, "node_modules/aws-sdk/node_modules/uuid": { "bin": { "uuid": "dist/bin/uuid" }, + "dev": true, "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", "version": "8.0.0" @@ -7962,6 +8069,7 @@ "version": "3.2.0" }, "node_modules/bowser": { + "dev": true, "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", "version": "2.11.0" @@ -8063,6 +8171,7 @@ "ieee754": "^1.1.4", "isarray": "^1.0.0" }, + "dev": true, "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", "version": "4.9.2" @@ -10006,6 +10115,7 @@ "version": "4.0.7" }, "node_modules/events": { + "dev": true, "engines": { "node": ">=0.4.x" }, @@ -10339,6 +10449,7 @@ "dependencies": { "strnum": "^1.0.5" }, + "dev": true, "funding": [ { "type": "paypal", @@ -10608,6 +10719,7 @@ "dependencies": { "is-callable": "^1.1.3" }, + "dev": true, "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "version": "0.3.3" @@ -11087,6 +11199,7 @@ "dependencies": { "get-intrinsic": "^1.1.3" }, + "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" }, @@ -11215,6 +11328,7 @@ "dependencies": { "has-symbols": "^1.0.2" }, + "dev": true, "engines": { "node": ">= 0.4" }, @@ -11363,6 +11477,7 @@ "version": "0.4.24" }, "node_modules/ieee754": { + "dev": true, "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "version": "1.1.13" @@ -11500,6 +11615,7 @@ "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" }, + "dev": true, "engines": { "node": ">= 0.4" }, @@ -11544,6 +11660,7 @@ "version": "3.2.1" }, "node_modules/is-callable": { + "dev": true, "engines": { "node": ">= 0.4" }, @@ -11610,6 +11727,7 @@ "dependencies": { "has-tostringtag": "^1.0.0" }, + "dev": true, "engines": { "node": ">= 0.4" }, @@ -11764,6 +11882,7 @@ "dependencies": { "which-typed-array": "^1.1.11" }, + "dev": true, "engines": { "node": ">= 0.4" }, @@ -11805,6 +11924,7 @@ "version": "2.2.0" }, "node_modules/isarray": { + "dev": true, "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "version": "1.0.0" @@ -14406,6 +14526,7 @@ "version": "8.1.1" }, "node_modules/jmespath": { + "dev": true, "engines": { "node": ">= 0.6.0" }, @@ -16625,6 +16746,7 @@ }, "node_modules/querystring": { "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "dev": true, "engines": { "node": ">=0.4.x" }, @@ -18101,6 +18223,7 @@ "version": "1.0.5" }, "node_modules/strnum": { + "dev": true, "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", "version": "1.0.5" @@ -19298,6 +19421,7 @@ "punycode": "1.3.2", "querystring": "0.2.0" }, + "dev": true, "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", "version": "0.10.3" @@ -19312,6 +19436,7 @@ "version": "1.5.10" }, "node_modules/url/node_modules/punycode": { + "dev": true, "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", "version": "1.3.2" @@ -19336,6 +19461,7 @@ "is-typed-array": "^1.1.3", "which-typed-array": "^1.1.2" }, + "dev": true, "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", "version": "0.12.5" @@ -19627,6 +19753,7 @@ "gopd": "^1.0.1", "has-tostringtag": "^1.0.0" }, + "dev": true, "engines": { "node": ">= 0.4" }, diff --git a/backend/package.json b/backend/package.json index 9fdc49d8..3a75e794 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,13 +1,10 @@ { "author": "", "dependencies": { - "@aws-sdk/client-cloudwatch-logs": "^3.417.0", - "@aws-sdk/client-ssm": "^3.414.0", "@elastic/elasticsearch": "~7.10.0", "@thefaultvault/tfv-cpe-parser": "^1.3.0", "@types/dockerode": "^3.3.19", "amqplib": "^0.10.3", - "aws-sdk": "^2.1551.0", "axios": "^1.6", "bufferutil": "^4.0.7", "class-transformer": "^0.3.1", @@ -45,6 +42,8 @@ }, "description": "", "devDependencies": { + "@aws-sdk/client-cloudwatch-logs": "^3.417.0", + "@aws-sdk/client-ssm": "^3.414.0", "@jest/globals": "^29", "@types/aws-lambda": "^8.10.62", "@types/cors": "^2.8.17", @@ -58,6 +57,7 @@ "@types/uuid": "^9.0.7", "@typescript-eslint/eslint-plugin": "^5.59", "@typescript-eslint/parser": "^5.59", + "aws-sdk": "^2.1551.0", "debug": "^4.3.4", "dockerode": "^3.3.1", "dotenv": "^16.0", From 1bb596fbb3b9a2d696f4642d19bde8a093e54a12 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 5 Aug 2024 15:26:10 -0400 Subject: [PATCH 005/314] Fix authentication with an api key --- backend/src/xfd_django/xfd_api/auth.py | 18 +++++++++++----- backend/src/xfd_django/xfd_api/views.py | 26 +++++++++++------------ backend/src/xfd_django/xfd_django/asgi.py | 18 +++++----------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index de6c2e74..8bfa0516 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -1,8 +1,9 @@ from fastapi import Depends, HTTPException, Security, status from fastapi.security import APIKeyHeader, OAuth2PasswordBearer -from xfd_api.jwt_utils import decode_jwt_token -from xfd_api.models import ApiKey, User from hashlib import sha256 +from .jwt_utils import decode_jwt_token +from .models import ApiKey, User +from django.utils import timezone oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) @@ -20,21 +21,28 @@ def get_user_by_api_key(api_key: str): hashed_key = sha256(api_key.encode()).hexdigest() try: api_key_instance = ApiKey.objects.get(hashedkey=hashed_key) + api_key_instance.lastused = timezone.now() + api_key_instance.save(update_fields=['lastused']) return api_key_instance.userid except ApiKey.DoesNotExist: + print("API Key not found") return None +# TODO: Uncomment the token and if not user token once the JWT from OKTA is working def get_current_active_user( api_key: str = Security(api_key_header), - token: str = Depends(oauth2_scheme) + # token: str = Depends(oauth2_scheme), ): + user = None if api_key: user = get_user_by_api_key(api_key) - else: - user = decode_jwt_token(token) + # if not user and token: + # user = decode_jwt_token(token) if user is None: + print("User not authenticated") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid authentication credentials", ) + print(f"Authenticated user: {user.id}") return user diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index d89fa103..1270c00f 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -4,15 +4,14 @@ from fastapi import ( APIRouter, Depends, - File, HTTPException, - Request, Security, - UploadFile, - status, + ) +from fastapi.security import APIKeyHeader, OAuth2PasswordBearer from .auth import get_current_active_user from .models import ApiKey, Organization, User +from typing import Any, List, Optional, Union api_router = APIRouter() @@ -29,7 +28,7 @@ async def healthcheck(): return {"status": "ok2"} -@api_router.get("/apikeys") +@api_router.get("/test-apikeys") async def get_api_keys(): """ Get all API keys. @@ -54,15 +53,14 @@ async def get_api_keys(): ] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) - -@api_router.get("/organizations") -async def get_organizations(current_user: User = Depends(get_current_active_user)): - """ - Get all organizations. - Returns: - list: A list of all organizations. - """ +@api_router.post( + "/test-orgs", + dependencies=[Depends(get_current_active_user)], + tags=["List of all Organizations"], +) +def read_orgs(current_user: User = Depends(get_current_active_user)): + """Call API endpoint to get all organizations.""" try: organizations = Organization.objects.all() return [ @@ -89,4 +87,4 @@ async def get_organizations(current_user: User = Depends(get_current_active_user for organization in organizations ] except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) \ No newline at end of file + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_django/asgi.py b/backend/src/xfd_django/xfd_django/asgi.py index 4dc03a83..09719eb3 100644 --- a/backend/src/xfd_django/xfd_django/asgi.py +++ b/backend/src/xfd_django/xfd_django/asgi.py @@ -6,28 +6,24 @@ For more information on this file, see https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/ """ + import os import django from django.apps import apps from django.conf import settings -from django.core.asgi import get_asgi_application from fastapi import FastAPI -from fastapi.middleware.wsgi import WSGIMiddleware +from fastapi.middleware.cors import CORSMiddleware from mangum import Mangum -from starlette.middleware.cors import CORSMiddleware os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xfd_django.settings") os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true" django.setup() -# Import views after Django setup -from xfd_api.views import api_router - -application = get_asgi_application() - -# Below this comment is custom code +# Ensure apps are populated apps.populate(settings.INSTALLED_APPS) +# Import views after Django setup +from xfd_api.views import api_router def get_application() -> FastAPI: """get_application function.""" @@ -40,11 +36,7 @@ def get_application() -> FastAPI: allow_headers=["*"], ) app.include_router(api_router) - app.mount("/", WSGIMiddleware(get_asgi_application())) - return app app = get_application() - handler = Mangum(app) - \ No newline at end of file From 79c423275c7702769492db70573f5cc0d99e3a50 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Tue, 6 Aug 2024 08:56:03 -0500 Subject: [PATCH 006/314] Revert "Make aws-sdk a dev dependency." This reverts commit 1bee135002b309bd78b8200255516b8e9315f7a4. --- backend/package-lock.json | 139 ++------------------------------------ backend/package.json | 6 +- 2 files changed, 9 insertions(+), 136 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 61492653..14e9c4f8 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -4,10 +4,13 @@ "packages": { "": { "dependencies": { + "@aws-sdk/client-cloudwatch-logs": "^3.417.0", + "@aws-sdk/client-ssm": "^3.414.0", "@elastic/elasticsearch": "~7.10.0", "@thefaultvault/tfv-cpe-parser": "^1.3.0", "@types/dockerode": "^3.3.19", "amqplib": "^0.10.3", + "aws-sdk": "^2.1551.0", "axios": "^1.6", "bufferutil": "^4.0.7", "class-transformer": "^0.3.1", @@ -44,8 +47,6 @@ "ws": "^8.18.0" }, "devDependencies": { - "@aws-sdk/client-cloudwatch-logs": "^3.417.0", - "@aws-sdk/client-ssm": "^3.414.0", "@jest/globals": "^29", "@types/aws-lambda": "^8.10.62", "@types/cors": "^2.8.17", @@ -59,7 +60,6 @@ "@types/uuid": "^9.0.7", "@typescript-eslint/eslint-plugin": "^5.59", "@typescript-eslint/parser": "^5.59", - "aws-sdk": "^2.1551.0", "debug": "^4.3.4", "dockerode": "^3.3.1", "dotenv": "^16.0", @@ -150,13 +150,11 @@ "@aws-sdk/types": "^3.222.0", "tslib": "^1.11.1" }, - "dev": true, "integrity": "sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==", "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/crc32/node_modules/tslib": { - "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -182,13 +180,11 @@ "dependencies": { "tslib": "^1.11.1" }, - "dev": true, "integrity": "sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==", "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/ie11-detection/node_modules/tslib": { - "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -225,13 +221,11 @@ "@aws-sdk/util-utf8-browser": "^3.0.0", "tslib": "^1.11.1" }, - "dev": true, "integrity": "sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/sha256-browser/node_modules/tslib": { - "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -242,13 +236,11 @@ "@aws-sdk/types": "^3.222.0", "tslib": "^1.11.1" }, - "dev": true, "integrity": "sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/sha256-js/node_modules/tslib": { - "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -257,13 +249,11 @@ "dependencies": { "tslib": "^1.11.1" }, - "dev": true, "integrity": "sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==", "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/supports-web-crypto/node_modules/tslib": { - "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -274,13 +264,11 @@ "@aws-sdk/util-utf8-browser": "^3.0.0", "tslib": "^1.11.1" }, - "dev": true, "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==", "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz", "version": "3.0.0" }, "node_modules/@aws-crypto/util/node_modules/tslib": { - "dev": true, "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "version": "1.14.1" @@ -970,7 +958,6 @@ "tslib": "^2.5.0", "uuid": "^8.3.2" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1015,7 +1002,6 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1064,7 +1050,6 @@ "fast-xml-parser": "4.2.5", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1079,7 +1064,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1100,7 +1084,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1122,7 +1105,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1138,7 +1120,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1156,7 +1137,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1171,7 +1151,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1186,7 +1165,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1200,7 +1178,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1215,7 +1192,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1230,7 +1206,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1248,7 +1223,6 @@ "@smithy/util-middleware": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1264,7 +1238,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1310,7 +1283,6 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1323,7 +1295,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1336,7 +1307,6 @@ "@aws-sdk/types": "3.413.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1351,7 +1321,6 @@ "bowser": "^2.11.0", "tslib": "^2.5.0" }, - "dev": true, "integrity": "sha512-7j/qWcRO2OBZBre2fC6V6M0PAS9n7k6i+VtofPkkhxC2DZszLJElqnooF9hGmVGYK3zR47Np4WjURXKIEZclWg==", "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.413.0.tgz", "version": "3.413.0" @@ -1363,7 +1332,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1384,7 +1352,6 @@ "@smithy/types": "^2.3.3", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1396,7 +1363,6 @@ "bin": { "uuid": "dist/bin/uuid" }, - "dev": true, "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "version": "8.3.2" @@ -1606,7 +1572,6 @@ "tslib": "^2.5.0", "uuid": "^8.3.2" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1651,7 +1616,6 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1700,7 +1664,6 @@ "fast-xml-parser": "4.2.5", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1715,7 +1678,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1736,7 +1698,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1758,7 +1719,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1774,7 +1734,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1792,7 +1751,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1807,7 +1765,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1822,7 +1779,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1836,7 +1792,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1851,7 +1806,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1866,7 +1820,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1884,7 +1837,6 @@ "@smithy/util-middleware": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1900,7 +1852,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1946,7 +1897,6 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1959,7 +1909,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1972,7 +1921,6 @@ "@aws-sdk/types": "3.413.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -1987,7 +1935,6 @@ "bowser": "^2.11.0", "tslib": "^2.5.0" }, - "dev": true, "integrity": "sha512-7j/qWcRO2OBZBre2fC6V6M0PAS9n7k6i+VtofPkkhxC2DZszLJElqnooF9hGmVGYK3zR47Np4WjURXKIEZclWg==", "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.413.0.tgz", "version": "3.413.0" @@ -1999,7 +1946,6 @@ "@smithy/types": "^2.3.1", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -2020,7 +1966,6 @@ "@smithy/types": "^2.3.3", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -2032,7 +1977,6 @@ "bin": { "uuid": "dist/bin/uuid" }, - "dev": true, "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "version": "8.3.2" @@ -2648,7 +2592,6 @@ "@smithy/util-middleware": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -2947,7 +2890,6 @@ "@smithy/types": "^2.2.2", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -2984,7 +2926,6 @@ "dependencies": { "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -3067,7 +3008,6 @@ "dependencies": { "tslib": "^2.3.1" }, - "dev": true, "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", "version": "3.259.0" @@ -5647,7 +5587,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5682,7 +5621,6 @@ "@smithy/util-middleware": "^2.0.6", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5698,7 +5636,6 @@ "@smithy/url-parser": "^2.0.13", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5713,7 +5650,6 @@ "@smithy/util-hex-encoding": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "integrity": "sha512-iqR6OuOV3zbQK8uVs9o+9AxhVk8kW9NAxA71nugwUB+kTY9C35pUd0A5/m4PRT0Y0oIW7W4kgnSR3fdYXQjECw==", "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-2.0.5.tgz", "version": "2.0.5" @@ -5781,7 +5717,6 @@ "@smithy/util-base64": "^2.0.1", "tslib": "^2.5.0" }, - "dev": true, "integrity": "sha512-PStY3XO1Ksjwn3wMKye5U6m6zxXpXrXZYqLy/IeCbh3nM9QB3Jgw/B0PUSLUWKdXg4U8qgEu300e3ZoBvZLsDg==", "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-2.2.6.tgz", "version": "2.2.6" @@ -5791,7 +5726,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5818,7 +5752,6 @@ "@smithy/util-utf8": "^2.0.2", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5845,7 +5778,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "integrity": "sha512-XsGYhVhvEikX1Yz0kyIoLssJf2Rs6E0U2w2YuKdT4jSra5A/g8V2oLROC1s56NldbgnpesTYB2z55KCHHbKyjw==", "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-2.0.13.tgz", "version": "2.0.13" @@ -5854,7 +5786,6 @@ "dependencies": { "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5879,7 +5810,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5892,7 +5822,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5910,7 +5839,6 @@ "@smithy/util-middleware": "^2.0.6", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5929,7 +5857,6 @@ "tslib": "^2.5.0", "uuid": "^8.3.2" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5942,7 +5869,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5954,7 +5880,6 @@ "bin": { "uuid": "dist/bin/uuid" }, - "dev": true, "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "version": "8.3.2" @@ -5964,7 +5889,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5977,7 +5901,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -5992,7 +5915,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6008,7 +5930,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6021,7 +5942,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6034,7 +5954,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6061,7 +5980,6 @@ "@smithy/util-uri-escape": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6074,7 +5992,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6086,7 +6003,6 @@ "dependencies": { "@smithy/types": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6099,7 +6015,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6118,7 +6033,6 @@ "@smithy/util-utf8": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6133,7 +6047,6 @@ "@smithy/util-stream": "^2.0.20", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6145,7 +6058,6 @@ "dependencies": { "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6159,7 +6071,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "integrity": "sha512-okWx2P/d9jcTsZWTVNnRMpFOE7fMkzloSFyM53fA7nLKJQObxM2T4JlZ5KitKKuXq7pxon9J6SF2kCwtdflIrA==", "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.13.tgz", "version": "2.0.13" @@ -6169,7 +6080,6 @@ "@smithy/util-buffer-from": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6181,7 +6091,6 @@ "dependencies": { "tslib": "^2.5.0" }, - "dev": true, "integrity": "sha512-JdDuS4ircJt+FDnaQj88TzZY3+njZ6O+D3uakS32f2VNnDo3vyEuNdBOh/oFd8Df1zSZOuH1HEChk2AOYDezZg==", "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-2.0.0.tgz", "version": "2.0.0" @@ -6190,7 +6099,6 @@ "dependencies": { "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6203,7 +6111,6 @@ "@smithy/is-array-buffer": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6215,7 +6122,6 @@ "dependencies": { "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6231,7 +6137,6 @@ "bowser": "^2.11.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">= 10.0.0" }, @@ -6249,7 +6154,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">= 10.0.0" }, @@ -6275,7 +6179,6 @@ "dependencies": { "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6288,7 +6191,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6302,7 +6204,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">= 14.0.0" }, @@ -6321,7 +6222,6 @@ "@smithy/util-utf8": "^2.0.2", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6333,7 +6233,6 @@ "dependencies": { "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6346,7 +6245,6 @@ "@smithy/util-buffer-from": "^2.0.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -6360,7 +6258,6 @@ "@smithy/types": "^2.5.0", "tslib": "^2.5.0" }, - "dev": true, "engines": { "node": ">=14.0.0" }, @@ -7753,7 +7650,6 @@ "version": "1.0.0" }, "node_modules/available-typed-arrays": { - "dev": true, "engines": { "node": ">= 0.4" }, @@ -7777,20 +7673,17 @@ "uuid": "8.0.0", "xml2js": "0.6.2" }, - "dev": true, "engines": { "node": ">= 10.0.0" }, - "hasInstallScript": true, - "integrity": "sha512-IhEcdGmiplF3l/pCROxEYIdi0s+LZ2VkbMAq3RgoXTHxY5cgqVRNaqsEsgIHev2Clxa9V08HttnIERTIUqb1+Q==", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1665.0.tgz", - "version": "2.1665.0" + "integrity": "sha512-nUaAzS7cheaKF8lV0AVJBqteuoYIgQ5UgpZaoRR44D7HA1f6iCYFISF6WH6d0hQvpxPDIXr5NlVt0cHyp/Sx1g==", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1551.0.tgz", + "version": "2.1551.0" }, "node_modules/aws-sdk/node_modules/uuid": { "bin": { "uuid": "dist/bin/uuid" }, - "dev": true, "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", "version": "8.0.0" @@ -8069,7 +7962,6 @@ "version": "3.2.0" }, "node_modules/bowser": { - "dev": true, "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", "version": "2.11.0" @@ -8171,7 +8063,6 @@ "ieee754": "^1.1.4", "isarray": "^1.0.0" }, - "dev": true, "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", "version": "4.9.2" @@ -10115,7 +10006,6 @@ "version": "4.0.7" }, "node_modules/events": { - "dev": true, "engines": { "node": ">=0.4.x" }, @@ -10449,7 +10339,6 @@ "dependencies": { "strnum": "^1.0.5" }, - "dev": true, "funding": [ { "type": "paypal", @@ -10719,7 +10608,6 @@ "dependencies": { "is-callable": "^1.1.3" }, - "dev": true, "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "version": "0.3.3" @@ -11199,7 +11087,6 @@ "dependencies": { "get-intrinsic": "^1.1.3" }, - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" }, @@ -11328,7 +11215,6 @@ "dependencies": { "has-symbols": "^1.0.2" }, - "dev": true, "engines": { "node": ">= 0.4" }, @@ -11477,7 +11363,6 @@ "version": "0.4.24" }, "node_modules/ieee754": { - "dev": true, "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "version": "1.1.13" @@ -11615,7 +11500,6 @@ "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" }, - "dev": true, "engines": { "node": ">= 0.4" }, @@ -11660,7 +11544,6 @@ "version": "3.2.1" }, "node_modules/is-callable": { - "dev": true, "engines": { "node": ">= 0.4" }, @@ -11727,7 +11610,6 @@ "dependencies": { "has-tostringtag": "^1.0.0" }, - "dev": true, "engines": { "node": ">= 0.4" }, @@ -11882,7 +11764,6 @@ "dependencies": { "which-typed-array": "^1.1.11" }, - "dev": true, "engines": { "node": ">= 0.4" }, @@ -11924,7 +11805,6 @@ "version": "2.2.0" }, "node_modules/isarray": { - "dev": true, "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "version": "1.0.0" @@ -14526,7 +14406,6 @@ "version": "8.1.1" }, "node_modules/jmespath": { - "dev": true, "engines": { "node": ">= 0.6.0" }, @@ -16746,7 +16625,6 @@ }, "node_modules/querystring": { "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "dev": true, "engines": { "node": ">=0.4.x" }, @@ -18223,7 +18101,6 @@ "version": "1.0.5" }, "node_modules/strnum": { - "dev": true, "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", "version": "1.0.5" @@ -19421,7 +19298,6 @@ "punycode": "1.3.2", "querystring": "0.2.0" }, - "dev": true, "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", "version": "0.10.3" @@ -19436,7 +19312,6 @@ "version": "1.5.10" }, "node_modules/url/node_modules/punycode": { - "dev": true, "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", "version": "1.3.2" @@ -19461,7 +19336,6 @@ "is-typed-array": "^1.1.3", "which-typed-array": "^1.1.2" }, - "dev": true, "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", "version": "0.12.5" @@ -19753,7 +19627,6 @@ "gopd": "^1.0.1", "has-tostringtag": "^1.0.0" }, - "dev": true, "engines": { "node": ">= 0.4" }, diff --git a/backend/package.json b/backend/package.json index 3a75e794..9fdc49d8 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,10 +1,13 @@ { "author": "", "dependencies": { + "@aws-sdk/client-cloudwatch-logs": "^3.417.0", + "@aws-sdk/client-ssm": "^3.414.0", "@elastic/elasticsearch": "~7.10.0", "@thefaultvault/tfv-cpe-parser": "^1.3.0", "@types/dockerode": "^3.3.19", "amqplib": "^0.10.3", + "aws-sdk": "^2.1551.0", "axios": "^1.6", "bufferutil": "^4.0.7", "class-transformer": "^0.3.1", @@ -42,8 +45,6 @@ }, "description": "", "devDependencies": { - "@aws-sdk/client-cloudwatch-logs": "^3.417.0", - "@aws-sdk/client-ssm": "^3.414.0", "@jest/globals": "^29", "@types/aws-lambda": "^8.10.62", "@types/cors": "^2.8.17", @@ -57,7 +58,6 @@ "@types/uuid": "^9.0.7", "@typescript-eslint/eslint-plugin": "^5.59", "@typescript-eslint/parser": "^5.59", - "aws-sdk": "^2.1551.0", "debug": "^4.3.4", "dockerode": "^3.3.1", "dotenv": "^16.0", From f28dfd0f40450bb169de7394d543ae80a34c87f5 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 7 Aug 2024 16:21:42 -0400 Subject: [PATCH 007/314] Update models for correct naming --- backend/src/xfd_django/xfd_api/models.py | 392 ++++++++++++----------- 1 file changed, 199 insertions(+), 193 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index cc352704..0e037e02 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -10,12 +10,12 @@ class ApiKey(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. - lastused = models.DateTimeField(db_column='lastUsed', blank=True, null=True) # Field name made lowercase. - hashedkey = models.TextField(db_column='hashedKey') # Field name made lowercase. - lastfour = models.TextField(db_column='lastFour') # Field name made lowercase. - userid = models.ForeignKey('User', models.DO_NOTHING, db_column='userId', blank=True, null=True) # Field name made lowercase. + createdAt = models.DateTimeField(auto_now_add=True, db_column='createdAt') + updatedAt = models.DateTimeField(auto_now=True, db_column='updatedAt') + lastUsed = models.DateTimeField(db_column='lastUsed', blank=True, null=True) + hashedKey = models.TextField(db_column='hashedKey') + lastFour = models.TextField(db_column='lastFour') + userId = models.ForeignKey('User', models.CASCADE, db_column='userId', blank=True, null=True) class Meta: managed = False @@ -24,11 +24,11 @@ class Meta: class Assessment(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. - rscid = models.CharField(db_column='rscId', unique=True) # Field name made lowercase. - type = models.CharField() - userid = models.ForeignKey('User', models.DO_NOTHING, db_column='userId', blank=True, null=True) # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') + rscId = models.CharField(db_column='rscId', unique=True) + type = models.CharField(max_length=255) + userId = models.ForeignKey('User', db_column='userId', blank=True, null=True, on_delete=models.CASCADE) class Meta: managed = False @@ -37,9 +37,9 @@ class Meta: class Category(models.Model): id = models.UUIDField(primary_key=True) - name = models.CharField() - number = models.CharField(unique=True) - shortname = models.CharField(db_column='shortName', blank=True, null=True) # Field name made lowercase. + name = models.CharField(max_length=255) + number = models.CharField(max_length=255, unique=True) + shortName = models.CharField(db_column='shortName', max_length=255, blank=True, null=True) class Meta: managed = False @@ -48,48 +48,49 @@ class Meta: class Cpe(models.Model): id = models.UUIDField(primary_key=True) - name = models.CharField() - version = models.CharField() - vendor = models.CharField() - lastseenat = models.DateTimeField(db_column='lastSeenAt') # Field name made lowercase. + name = models.CharField(max_length=255) + version = models.CharField(max_length=255) + vendor = models.CharField(max_length=255) + lastSeenAt = models.DateTimeField(db_column='lastSeenAt') class Meta: - managed = False db_table = 'cpe' - unique_together = (('name', 'version', 'vendor'),) + managed = False # This ensures Django does not manage the table + unique_together = (('name', 'version', 'vendor'),) # Unique constraint + class Cve(models.Model): id = models.UUIDField(primary_key=True) name = models.CharField(unique=True, blank=True, null=True) - publishedat = models.DateTimeField(db_column='publishedAt', blank=True, null=True) # Field name made lowercase. - modifiedat = models.DateTimeField(db_column='modifiedAt', blank=True, null=True) # Field name made lowercase. + publishedAt = models.DateTimeField(db_column='publishedAt', blank=True, null=True) + modifiedAt = models.DateTimeField(db_column='modifiedAt', blank=True, null=True) status = models.CharField(blank=True, null=True) description = models.CharField(blank=True, null=True) - cvssv2source = models.CharField(db_column='cvssV2Source', blank=True, null=True) # Field name made lowercase. - cvssv2type = models.CharField(db_column='cvssV2Type', blank=True, null=True) # Field name made lowercase. - cvssv2version = models.CharField(db_column='cvssV2Version', blank=True, null=True) # Field name made lowercase. - cvssv2vectorstring = models.CharField(db_column='cvssV2VectorString', blank=True, null=True) # Field name made lowercase. - cvssv2basescore = models.CharField(db_column='cvssV2BaseScore', blank=True, null=True) # Field name made lowercase. - cvssv2baseseverity = models.CharField(db_column='cvssV2BaseSeverity', blank=True, null=True) # Field name made lowercase. - cvssv2exploitabilityscore = models.CharField(db_column='cvssV2ExploitabilityScore', blank=True, null=True) # Field name made lowercase. - cvssv2impactscore = models.CharField(db_column='cvssV2ImpactScore', blank=True, null=True) # Field name made lowercase. - cvssv3source = models.CharField(db_column='cvssV3Source', blank=True, null=True) # Field name made lowercase. - cvssv3type = models.CharField(db_column='cvssV3Type', blank=True, null=True) # Field name made lowercase. - cvssv3version = models.CharField(db_column='cvssV3Version', blank=True, null=True) # Field name made lowercase. - cvssv3vectorstring = models.CharField(db_column='cvssV3VectorString', blank=True, null=True) # Field name made lowercase. - cvssv3basescore = models.CharField(db_column='cvssV3BaseScore', blank=True, null=True) # Field name made lowercase. - cvssv3baseseverity = models.CharField(db_column='cvssV3BaseSeverity', blank=True, null=True) # Field name made lowercase. - cvssv3exploitabilityscore = models.CharField(db_column='cvssV3ExploitabilityScore', blank=True, null=True) # Field name made lowercase. - cvssv3impactscore = models.CharField(db_column='cvssV3ImpactScore', blank=True, null=True) # Field name made lowercase. - cvssv4source = models.CharField(db_column='cvssV4Source', blank=True, null=True) # Field name made lowercase. - cvssv4type = models.CharField(db_column='cvssV4Type', blank=True, null=True) # Field name made lowercase. - cvssv4version = models.CharField(db_column='cvssV4Version', blank=True, null=True) # Field name made lowercase. - cvssv4vectorstring = models.CharField(db_column='cvssV4VectorString', blank=True, null=True) # Field name made lowercase. - cvssv4basescore = models.CharField(db_column='cvssV4BaseScore', blank=True, null=True) # Field name made lowercase. - cvssv4baseseverity = models.CharField(db_column='cvssV4BaseSeverity', blank=True, null=True) # Field name made lowercase. - cvssv4exploitabilityscore = models.CharField(db_column='cvssV4ExploitabilityScore', blank=True, null=True) # Field name made lowercase. - cvssv4impactscore = models.CharField(db_column='cvssV4ImpactScore', blank=True, null=True) # Field name made lowercase. + cvssV2Source = models.CharField(db_column='cvssV2Source', blank=True, null=True) + cvssV2Type = models.CharField(db_column='cvssV2Type', blank=True, null=True) + cvssV2Version = models.CharField(db_column='cvssV2Version', blank=True, null=True) + cvssV2VectorString = models.CharField(db_column='cvssV2VectorString', blank=True, null=True) + cvssV2BaseScore = models.CharField(db_column='cvssV2BaseScore', blank=True, null=True) + cvssV2BaseSeverity = models.CharField(db_column='cvssV2BaseSeverity', blank=True, null=True) + cvssV2ExploitabilityScore = models.CharField(db_column='cvssV2ExploitabilityScore', blank=True, null=True) + cvssV2ImpactScore = models.CharField(db_column='cvssV2ImpactScore', blank=True, null=True) + cvssV3Source = models.CharField(db_column='cvssV3Source', blank=True, null=True) + cvssV3Type = models.CharField(db_column='cvssV3Type', blank=True, null=True) + cvssV3Version = models.CharField(db_column='cvssV3Version', blank=True, null=True) + cvssV3VectorString = models.CharField(db_column='cvssV3VectorString', blank=True, null=True) + cvssV3BaseScore = models.CharField(db_column='cvssV3BaseScore', blank=True, null=True) + cvssV3BaseSeverity = models.CharField(db_column='cvssV3BaseSeverity', blank=True, null=True) + cvssV3ExploitabilityScore = models.CharField(db_column='cvssV3ExploitabilityScore', blank=True, null=True) + cvssV3ImpactScore = models.CharField(db_column='cvssV3ImpactScore', blank=True, null=True) + cvssV4Source = models.CharField(db_column='cvssV4Source', blank=True, null=True) + cvssV4Type = models.CharField(db_column='cvssV4Type', blank=True, null=True) + cvssV4Version = models.CharField(db_column='cvssV4Version', blank=True, null=True) + cvssV4VectorString = models.CharField(db_column='cvssV4VectorString', blank=True, null=True) + cvssV4BaseScore = models.CharField(db_column='cvssV4BaseScore', blank=True, null=True) + cvssV4BaseSeverity = models.CharField(db_column='cvssV4BaseSeverity', blank=True, null=True) + cvssV4ExploitabilityScore = models.CharField(db_column='cvssV4ExploitabilityScore', blank=True, null=True) + cvssV4ImpactScore = models.CharField(db_column='cvssV4ImpactScore', blank=True, null=True) weaknesses = models.TextField(blank=True, null=True) references = models.TextField(blank=True, null=True) @@ -99,51 +100,56 @@ class Meta: class CveCpesCpe(models.Model): - cveid = models.OneToOneField(Cve, models.DO_NOTHING, db_column='cveId', primary_key=True) # Field name made lowercase. The composite primary key (cveId, cpeId) found, that is not supported. The first column is selected. - cpeid = models.ForeignKey(Cpe, models.DO_NOTHING, db_column='cpeId') # Field name made lowercase. + cveId = models.ForeignKey(Cve, on_delete=models.CASCADE, db_column='cveId') + cpeId = models.ForeignKey(Cpe, on_delete=models.CASCADE, db_column='cpeId') class Meta: - managed = False db_table = 'cve_cpes_cpe' - unique_together = (('cveid', 'cpeid'),) + managed = False # This ensures Django does not manage the table + unique_together = (('cve', 'cpe'),) # Unique constraint class Domain(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. - syncedat = models.DateTimeField(db_column='syncedAt', blank=True, null=True) # Field name made lowercase. - ip = models.CharField(blank=True, null=True) - fromrootdomain = models.CharField(db_column='fromRootDomain', blank=True, null=True) # Field name made lowercase. - subdomainsource = models.CharField(db_column='subdomainSource', blank=True, null=True) # Field name made lowercase. - iponly = models.BooleanField(db_column='ipOnly', blank=True, null=True) # Field name made lowercase. - reversename = models.CharField(db_column='reverseName', max_length=512) # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') + syncedAt = models.DateTimeField(db_column='syncedAt', blank=True, null=True) + ip = models.CharField(max_length=255, blank=True, null=True) + fromRootDomain = models.CharField(db_column='fromRootDomain', blank=True, null=True) + subdomainSource = models.CharField(db_column='subdomainSource', max_length=255, blank=True, null=True) + ipOnly = models.BooleanField(db_column='ipOnly', default=False) + reverseName = models.CharField(db_column='reverseName', max_length=512) name = models.CharField(max_length=512) screenshot = models.CharField(max_length=512, blank=True, null=True) - country = models.CharField(blank=True, null=True) - asn = models.CharField(blank=True, null=True) - cloudhosted = models.BooleanField(db_column='cloudHosted') # Field name made lowercase. + country = models.CharField(max_length=255, blank=True, null=True) + asn = models.CharField(max_length=255, blank=True, null=True) + cloudHosted = models.BooleanField(db_column='cloudHosted', default=False) ssl = models.JSONField(blank=True, null=True) - censyscertificatesresults = models.JSONField(db_column='censysCertificatesResults') # Field name made lowercase. - trustymailresults = models.JSONField(db_column='trustymailResults') # Field name made lowercase. - discoveredbyid = models.ForeignKey('Scan', models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) # Field name made lowercase. - organizationid = models.ForeignKey('Organization', models.DO_NOTHING, db_column='organizationId') # Field name made lowercase. + censysCertificatesResults = models.JSONField(db_column='censysCertificatesResults', default=dict) + trustymailResults = models.JSONField(db_column='trustymailResults', default=dict) + discoveredById = models.ForeignKey('Scan', on_delete=models.SET_NULL, db_column='discoveredById', blank=True, null=True) + organizationId = models.ForeignKey('Organization', on_delete=models.CASCADE, db_column='organizationId') class Meta: - managed = False db_table = 'domain' - unique_together = (('name', 'organizationid'),) + managed = False # This ensures Django does not manage the table + unique_together = (('name', 'organization'),) # Unique constraint + + def save(self, *args, **kwargs): + self.name = self.name.lower() + self.reverseName = '.'.join(reversed(self.name.split('.'))) + super(Domain, self).save(*args, **kwargs) class Notification(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. - startdatetime = models.DateTimeField(db_column='startDatetime', blank=True, null=True) # Field name made lowercase. - enddatetime = models.DateTimeField(db_column='endDatetime', blank=True, null=True) # Field name made lowercase. - maintenancetype = models.CharField(db_column='maintenanceType', blank=True, null=True) # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') + startDatetime = models.DateTimeField(db_column='startDatetime', blank=True, null=True) + endDatetime = models.DateTimeField(db_column='endDatetime', blank=True, null=True) + maintenanceType = models.CharField(db_column='maintenanceType', blank=True, null=True) status = models.CharField(blank=True, null=True) - updatedby = models.CharField(db_column='updatedBy', blank=True, null=True) # Field name made lowercase. + updatedBy = models.CharField(db_column='updatedBy', blank=True, null=True) message = models.CharField(blank=True, null=True) class Meta: @@ -153,24 +159,24 @@ class Meta: class Organization(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') acronym = models.CharField(unique=True, blank=True, null=True) name = models.CharField() - rootdomains = models.TextField(db_column='rootDomains') # Field name made lowercase. This field type is a guess. - ipblocks = models.TextField(db_column='ipBlocks') # Field name made lowercase. This field type is a guess. - ispassive = models.BooleanField(db_column='isPassive') # Field name made lowercase. - pendingdomains = models.TextField(db_column='pendingDomains') # Field name made lowercase. This field type is a guess. + rootDomains = models.TextField(db_column='rootDomains') # This field type is a guess. + ipBlocks = models.TextField(db_column='ipBlocks') # This field type is a guess. + isPassive = models.BooleanField(db_column='isPassive') + pendingDomains = models.TextField(db_column='pendingDomains') # This field type is a guess. country = models.CharField(blank=True, null=True) state = models.CharField(blank=True, null=True) - regionid = models.CharField(db_column='regionId', blank=True, null=True) # Field name made lowercase. - statefips = models.IntegerField(db_column='stateFips', blank=True, null=True) # Field name made lowercase. - statename = models.CharField(db_column='stateName', blank=True, null=True) # Field name made lowercase. + regionId = models.CharField(db_column='regionId', blank=True, null=True) + stateFips = models.IntegerField(db_column='stateFips', blank=True, null=True) + stateName = models.CharField(db_column='stateName', blank=True, null=True) county = models.CharField(blank=True, null=True) - countyfips = models.IntegerField(db_column='countyFips', blank=True, null=True) # Field name made lowercase. + countyFips = models.IntegerField(db_column='countyFips', blank=True, null=True) type = models.CharField(blank=True, null=True) - parentid = models.ForeignKey('self', models.DO_NOTHING, db_column='parentId', blank=True, null=True) # Field name made lowercase. - createdbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) # Field name made lowercase. + parentId = models.ForeignKey('self', models.DO_NOTHING, db_column='parentId', blank=True, null=True) + createdById = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) class Meta: managed = False @@ -179,8 +185,8 @@ class Meta: class OrganizationTag(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') name = models.CharField(unique=True) class Meta: @@ -189,13 +195,13 @@ class Meta: class OrganizationTagOrganizationsOrganization(models.Model): - organizationtagid = models.OneToOneField(OrganizationTag, models.DO_NOTHING, db_column='organizationTagId', primary_key=True) # Field name made lowercase. The composite primary key (organizationTagId, organizationId) found, that is not supported. The first column is selected. - organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') # Field name made lowercase. + organizationTagId = models.OneToOneField(OrganizationTag, models.DO_NOTHING, db_column='organizationTagId', primary_key=True) # The composite primary key (organizationTagId, organizationId) found, that is not supported. The first column is selected. + organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') class Meta: managed = False db_table = 'organization_tag_organizations_organization' - unique_together = (('organizationtagid', 'organizationid'),) + unique_together = (('organizationTagId', 'organizationId'),) class QueryResultCache(models.Model): @@ -212,26 +218,27 @@ class Meta: class Question(models.Model): id = models.UUIDField(primary_key=True) - name = models.CharField() + name = models.CharField(max_length=255) description = models.CharField(blank=True, null=True) - longform = models.CharField(db_column='longForm') # Field name made lowercase. - number = models.CharField() - categoryid = models.ForeignKey(Category, models.DO_NOTHING, db_column='categoryId', blank=True, null=True) # Field name made lowercase. + longForm = models.CharField(db_column='longForm') + number = models.CharField(max_length=255) + categoryId = models.ForeignKey(Category, models.DO_NOTHING, db_column='categoryId', blank=True, null=True) class Meta: - managed = False db_table = 'question' - unique_together = (('categoryid', 'number'),) + managed = False + unique_together = (('category', 'number'),) +# Question and Resource many-to-many class QuestionResourcesResource(models.Model): - questionid = models.OneToOneField(Question, models.DO_NOTHING, db_column='questionId', primary_key=True) # Field name made lowercase. The composite primary key (questionId, resourceId) found, that is not supported. The first column is selected. - resourceid = models.ForeignKey('Resource', models.DO_NOTHING, db_column='resourceId') # Field name made lowercase. + questionId = models.ForeignKey('Question', on_delete=models.CASCADE, db_column='questionId') + resourceId = models.ForeignKey('Resource', on_delete=models.CASCADE, db_column='resourceId') class Meta: - managed = False db_table = 'question_resources_resource' - unique_together = (('questionid', 'resourceid'),) + managed = False + unique_together = (('question', 'resource'),) class Resource(models.Model): @@ -245,50 +252,49 @@ class Meta: managed = False db_table = 'resource' - class Response(models.Model): id = models.UUIDField(primary_key=True) selection = models.CharField() - assessmentid = models.ForeignKey(Assessment, models.DO_NOTHING, db_column='assessmentId', blank=True, null=True) # Field name made lowercase. - questionid = models.ForeignKey(Question, models.DO_NOTHING, db_column='questionId', blank=True, null=True) # Field name made lowercase. + assessmentId = models.ForeignKey(Assessment, models.DO_NOTHING, db_column='assessmentId', blank=True, null=True) + questionId = models.ForeignKey(Question, models.DO_NOTHING, db_column='questionId', blank=True, null=True) class Meta: managed = False db_table = 'response' - unique_together = (('assessmentid', 'questionid'),) + unique_together = (('assessmentId', 'questionId'),) class Role(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') role = models.CharField() approved = models.BooleanField() - createdbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) # Field name made lowercase. - approvedbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='approvedById', related_name='role_approvedbyid_set', blank=True, null=True) # Field name made lowercase. - userid = models.ForeignKey('User', models.DO_NOTHING, db_column='userId', related_name='role_userid_set', blank=True, null=True) # Field name made lowercase. - organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId', blank=True, null=True) # Field name made lowercase. + createdById = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) + approvedById = models.ForeignKey('User', models.DO_NOTHING, db_column='approvedById', related_name='role_approvedbyid_set', blank=True, null=True) + userId = models.ForeignKey('User', models.DO_NOTHING, db_column='userId', related_name='role_userid_set', blank=True, null=True) + organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId', blank=True, null=True) class Meta: managed = False db_table = 'role' - unique_together = (('userid', 'organizationid'),) + unique_together = (('userId', 'organizationId'),) class SavedSearch(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') name = models.CharField() - searchterm = models.CharField(db_column='searchTerm') # Field name made lowercase. - sortdirection = models.CharField(db_column='sortDirection') # Field name made lowercase. - sortfield = models.CharField(db_column='sortField') # Field name made lowercase. + searchTerm = models.CharField(db_column='searchTerm') + sortDirection = models.CharField(db_column='sortDirection') + sortField = models.CharField(db_column='sortField') count = models.IntegerField() filters = models.JSONField() - searchpath = models.CharField(db_column='searchPath') # Field name made lowercase. - createvulnerabilities = models.BooleanField(db_column='createVulnerabilities') # Field name made lowercase. - vulnerabilitytemplate = models.JSONField(db_column='vulnerabilityTemplate') # Field name made lowercase. - createdbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) # Field name made lowercase. + searchPath = models.CharField(db_column='searchPath') + createVulnerabilities = models.BooleanField(db_column='createVulnerabilities') + vulnerabilityTemplate = models.JSONField(db_column='vulnerabilityTemplate') + createdById = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) class Meta: managed = False @@ -297,17 +303,17 @@ class Meta: class Scan(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') name = models.CharField() - arguments = models.TextField() # This field type is a guess. + arguments = models.JSONField() frequency = models.IntegerField() - lastrun = models.DateTimeField(db_column='lastRun', blank=True, null=True) # Field name made lowercase. - isgranular = models.BooleanField(db_column='isGranular') # Field name made lowercase. - isusermodifiable = models.BooleanField(db_column='isUserModifiable', blank=True, null=True) # Field name made lowercase. - issinglescan = models.BooleanField(db_column='isSingleScan') # Field name made lowercase. - manualrunpending = models.BooleanField(db_column='manualRunPending') # Field name made lowercase. - createdbyid = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) # Field name made lowercase. + lastRun = models.DateTimeField(db_column='lastRun', blank=True, null=True) + isGranular = models.BooleanField(db_column='isGranular') + isUserModifiable = models.BooleanField(db_column='isUserModifiable', blank=True, null=True) + isSingleScan = models.BooleanField(db_column='isSingleScan') + manualRunPending = models.BooleanField(db_column='manualRunPending') + createdBy = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) class Meta: managed = False @@ -315,40 +321,40 @@ class Meta: class ScanOrganizationsOrganization(models.Model): - scanid = models.OneToOneField(Scan, models.DO_NOTHING, db_column='scanId', primary_key=True) # Field name made lowercase. The composite primary key (scanId, organizationId) found, that is not supported. The first column is selected. - organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') # Field name made lowercase. + scanId = models.OneToOneField(Scan, models.DO_NOTHING, db_column='scanId', primary_key=True) # The composite primary key (scanId, organizationId) found, that is not supported. The first column is selected. + organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') class Meta: managed = False db_table = 'scan_organizations_organization' - unique_together = (('scanid', 'organizationid'),) + unique_together = (('scanId', 'organizationId'),) class ScanTagsOrganizationTag(models.Model): - scanid = models.OneToOneField(Scan, models.DO_NOTHING, db_column='scanId', primary_key=True) # Field name made lowercase. The composite primary key (scanId, organizationTagId) found, that is not supported. The first column is selected. - organizationtagid = models.ForeignKey(OrganizationTag, models.DO_NOTHING, db_column='organizationTagId') # Field name made lowercase. + scanId = models.OneToOneField(Scan, models.DO_NOTHING, db_column='scanId', primary_key=True) # The composite primary key (scanId, organizationTagId) found, that is not supported. The first column is selected. + organizationTagId = models.ForeignKey(OrganizationTag, models.DO_NOTHING, db_column='organizationTagId') class Meta: managed = False db_table = 'scan_tags_organization_tag' - unique_together = (('scanid', 'organizationtagid'),) + unique_together = (('scanId', 'organizationTagId'),) class ScanTask(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') status = models.TextField() type = models.TextField() - fargatetaskarn = models.TextField(db_column='fargateTaskArn', blank=True, null=True) # Field name made lowercase. + fargateTaskArn = models.TextField(db_column='fargateTaskArn', blank=True, null=True) input = models.TextField(blank=True, null=True) output = models.TextField(blank=True, null=True) - requestedat = models.DateTimeField(db_column='requestedAt', blank=True, null=True) # Field name made lowercase. - startedat = models.DateTimeField(db_column='startedAt', blank=True, null=True) # Field name made lowercase. - finishedat = models.DateTimeField(db_column='finishedAt', blank=True, null=True) # Field name made lowercase. - queuedat = models.DateTimeField(db_column='queuedAt', blank=True, null=True) # Field name made lowercase. - organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId', blank=True, null=True) # Field name made lowercase. - scanid = models.ForeignKey(Scan, models.DO_NOTHING, db_column='scanId', blank=True, null=True) # Field name made lowercase. + requestedAt = models.DateTimeField(db_column='requestedAt', blank=True, null=True) + startedAt = models.DateTimeField(db_column='startedAt', blank=True, null=True) + finishedAt = models.DateTimeField(db_column='finishedAt', blank=True, null=True) + queuedAt = models.DateTimeField(db_column='queuedAt', blank=True, null=True) + organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId', blank=True, null=True) + scanId = models.ForeignKey(Scan, models.DO_NOTHING, db_column='scanId', blank=True, null=True) class Meta: managed = False @@ -356,37 +362,37 @@ class Meta: class ScanTaskOrganizationsOrganization(models.Model): - scantaskid = models.OneToOneField(ScanTask, models.DO_NOTHING, db_column='scanTaskId', primary_key=True) # Field name made lowercase. The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. - organizationid = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') # Field name made lowercase. + scanTaskId = models.OneToOneField(ScanTask, models.DO_NOTHING, db_column='scanTaskId', primary_key=True) # The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. + organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') class Meta: managed = False db_table = 'scan_task_organizations_organization' - unique_together = (('scantaskid', 'organizationid'),) + unique_together = (('scanTaskId', 'organizationId'),) class Service(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. - servicesource = models.TextField(db_column='serviceSource', blank=True, null=True) # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') + serviceSource = models.TextField(db_column='serviceSource', blank=True, null=True) port = models.IntegerField() service = models.CharField(blank=True, null=True) - lastseen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) # Field name made lowercase. + lastSeen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) banner = models.TextField(blank=True, null=True) products = models.JSONField() - censysmetadata = models.JSONField(db_column='censysMetadata') # Field name made lowercase. - censysipv4results = models.JSONField(db_column='censysIpv4Results') # Field name made lowercase. - intrigueidentresults = models.JSONField(db_column='intrigueIdentResults') # Field name made lowercase. - shodanresults = models.JSONField(db_column='shodanResults') # Field name made lowercase. - wappalyzerresults = models.JSONField(db_column='wappalyzerResults') # Field name made lowercase. - domainid = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) # Field name made lowercase. - discoveredbyid = models.ForeignKey(Scan, models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) # Field name made lowercase. + censysMetadata = models.JSONField(db_column='censysMetadata') + censysIpv4Results = models.JSONField(db_column='censysIpv4Results') + intrigueIdentResults = models.JSONField(db_column='intrigueIdentResults') + shodanResults = models.JSONField(db_column='shodanResults') + wappalyzerResults = models.JSONField(db_column='wappalyzerResults') + domainId = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) + discoveredById = models.ForeignKey(Scan, models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) class Meta: managed = False db_table = 'service' - unique_together = (('port', 'domainid'),) + unique_together = (('port', 'domainId'),) class TypeormMetadata(models.Model): @@ -404,23 +410,23 @@ class Meta: class User(models.Model): id = models.UUIDField(primary_key=True) - cognitoid = models.CharField(db_column='cognitoId', unique=True, blank=True, null=True) # Field name made lowercase. - logingovid = models.CharField(db_column='loginGovId', unique=True, blank=True, null=True) # Field name made lowercase. - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. - firstname = models.CharField(db_column='firstName') # Field name made lowercase. - lastname = models.CharField(db_column='lastName') # Field name made lowercase. - fullname = models.CharField(db_column='fullName') # Field name made lowercase. + cognitoId = models.CharField(db_column='cognitoId', unique=True, blank=True, null=True) + loginGovId = models.CharField(db_column='loginGovId', unique=True, blank=True, null=True) + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') + firstName = models.CharField(db_column='firstName') + lastName = models.CharField(db_column='lastName') + fullName = models.CharField(db_column='fullName') email = models.CharField(unique=True) - invitepending = models.BooleanField(db_column='invitePending') # Field name made lowercase. - loginblockedbymaintenance = models.BooleanField(db_column='loginBlockedByMaintenance') # Field name made lowercase. - dateacceptedterms = models.DateTimeField(db_column='dateAcceptedTerms', blank=True, null=True) # Field name made lowercase. - acceptedtermsversion = models.TextField(db_column='acceptedTermsVersion', blank=True, null=True) # Field name made lowercase. - lastloggedin = models.DateTimeField(db_column='lastLoggedIn', blank=True, null=True) # Field name made lowercase. - usertype = models.TextField(db_column='userType') # Field name made lowercase. - regionid = models.CharField(db_column='regionId', blank=True, null=True) # Field name made lowercase. + invitePending = models.BooleanField(db_column='invitePending') + loginBlockedByMaintenance = models.BooleanField(db_column='loginBlockedByMaintenance') + dateAcceptedTerms = models.DateTimeField(db_column='dateAcceptedTerms', blank=True, null=True) + acceptedTermsVersion = models.TextField(db_column='acceptedTermsVersion', blank=True, null=True) + lastLoggedIn = models.DateTimeField(db_column='lastLoggedIn', blank=True, null=True) + userType = models.TextField(db_column='userType') + regionId = models.CharField(db_column='regionId', blank=True, null=True) state = models.CharField(blank=True, null=True) - oktaid = models.CharField(db_column='oktaId', unique=True, blank=True, null=True) # Field name made lowercase. + oktaId = models.CharField(db_column='oktaId', unique=True, blank=True, null=True) class Meta: managed = False @@ -429,9 +435,9 @@ class Meta: class Vulnerability(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. - lastseen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') + lastSeen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) title = models.CharField() cve = models.TextField(blank=True, null=True) cwe = models.TextField(blank=True, null=True) @@ -440,39 +446,39 @@ class Vulnerability(models.Model): references = models.JSONField() cvss = models.DecimalField(max_digits=65535, decimal_places=65535, blank=True, null=True) severity = models.TextField(blank=True, null=True) - needspopulation = models.BooleanField(db_column='needsPopulation') # Field name made lowercase. + needsPopulation = models.BooleanField(db_column='needsPopulation') state = models.CharField() substate = models.CharField() source = models.CharField() notes = models.CharField() actions = models.JSONField() - structureddata = models.JSONField(db_column='structuredData') # Field name made lowercase. - iskev = models.BooleanField(db_column='isKev', blank=True, null=True) # Field name made lowercase. - kevresults = models.JSONField(db_column='kevResults', blank=True, null=True) # Field name made lowercase. - domainid = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) # Field name made lowercase. - serviceid = models.ForeignKey(Service, models.DO_NOTHING, db_column='serviceId', blank=True, null=True) # Field name made lowercase. + structuredData = models.JSONField(db_column='structuredData') + isKev = models.BooleanField(db_column='isKev', blank=True, null=True) + kevResults = models.JSONField(db_column='kevResults', blank=True, null=True) + domainId = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) + serviceId = models.ForeignKey(Service, models.DO_NOTHING, db_column='serviceId', blank=True, null=True) class Meta: managed = False db_table = 'vulnerability' - unique_together = (('domainid', 'title'),) + unique_together = (('domainId', 'title'),) class Webpage(models.Model): id = models.UUIDField(primary_key=True) - createdat = models.DateTimeField(db_column='createdAt') # Field name made lowercase. - updatedat = models.DateTimeField(db_column='updatedAt') # Field name made lowercase. - syncedat = models.DateTimeField(db_column='syncedAt', blank=True, null=True) # Field name made lowercase. - lastseen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) # Field name made lowercase. - s3key = models.CharField(db_column='s3Key', blank=True, null=True) # Field name made lowercase. + createdAt = models.DateTimeField(db_column='createdAt') + updatedAt = models.DateTimeField(db_column='updatedAt') + syncedAt = models.DateTimeField(db_column='syncedAt', blank=True, null=True) + lastSeen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) + s3key = models.CharField(db_column='s3Key', blank=True, null=True) url = models.CharField() status = models.DecimalField(max_digits=65535, decimal_places=65535) - responsesize = models.DecimalField(db_column='responseSize', max_digits=65535, decimal_places=65535, blank=True, null=True) # Field name made lowercase. + responseSize = models.DecimalField(db_column='responseSize', max_digits=65535, decimal_places=65535, blank=True, null=True) headers = models.JSONField() - domainid = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) # Field name made lowercase. - discoveredbyid = models.ForeignKey(Scan, models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) # Field name made lowercase. + domainId = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) + discoveredById = models.ForeignKey(Scan, models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) class Meta: managed = False db_table = 'webpage' - unique_together = (('url', 'domainid'),) \ No newline at end of file + unique_together = (('url', 'domainId'),) \ No newline at end of file From 1a9379ca2d2ea51b1962204b86eed97c10f09bf2 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Fri, 23 Aug 2024 11:48:10 -0500 Subject: [PATCH 008/314] In auth.py and views.py: add docstrings; convert field names to camel case; remove unused imports. --- backend/src/xfd_django/xfd_api/auth.py | 70 +++++++++++++++++++++++-- backend/src/xfd_django/xfd_api/views.py | 66 ++++++++++++++--------- 2 files changed, 105 insertions(+), 31 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 8bfa0516..f1c5f001 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -1,14 +1,51 @@ +""" +This module provides authentication utilities for the FastAPI application. + +It includes functions to: +- Decode JWT tokens and retrieve the current user. +- Retrieve a user by their API key. +- Ensure the current user is authenticated and active. + +Functions: + - get_current_user: Decodes a JWT token to retrieve the current user. + - get_user_by_api_key: Retrieves a user by their API key. + - get_current_active_user: Ensures the current user is authenticated and active. + +Dependencies: + - fastapi + - django + - hashlib + - .jwt_utils + - .models +""" +# Standard Python Libraries +from hashlib import sha256 + +# Third-Party Libraries +from django.utils import timezone from fastapi import Depends, HTTPException, Security, status from fastapi.security import APIKeyHeader, OAuth2PasswordBearer -from hashlib import sha256 + from .jwt_utils import decode_jwt_token -from .models import ApiKey, User -from django.utils import timezone +from .models import ApiKey oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) + def get_current_user(token: str = Depends(oauth2_scheme)): + """ + Decode a JWT token to retrieve the current user. + + Args: + token (str): The JWT token. + + Raises: + HTTPException: If the token is invalid or expired. + + Returns: + User: The user object decoded from the token. + """ user = decode_jwt_token(token) if user is None: raise HTTPException( @@ -17,22 +54,45 @@ def get_current_user(token: str = Depends(oauth2_scheme)): ) return user + def get_user_by_api_key(api_key: str): + """ + Retrieve a user by their API key. + + Args: + api_key (str): The API key. + + Returns: + User: The user object associated with the API key, or None if not found. + """ hashed_key = sha256(api_key.encode()).hexdigest() try: - api_key_instance = ApiKey.objects.get(hashedkey=hashed_key) + api_key_instance = ApiKey.objects.get(hashedKey=hashed_key) api_key_instance.lastused = timezone.now() - api_key_instance.save(update_fields=['lastused']) + api_key_instance.save(update_fields=["lastUsed"]) return api_key_instance.userid except ApiKey.DoesNotExist: print("API Key not found") return None + # TODO: Uncomment the token and if not user token once the JWT from OKTA is working def get_current_active_user( api_key: str = Security(api_key_header), # token: str = Depends(oauth2_scheme), ): + """ + Ensure the current user is authenticated and active. + + Args: + api_key (str): The API key. + + Raises: + HTTPException: If the user is not authenticated. + + Returns: + User: The authenticated user object. + """ user = None if api_key: user = get_user_by_api_key(api_key) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 1270c00f..3e717d83 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -1,17 +1,26 @@ +""" +This module defines the API endpoints for the FastAPI application. +It includes endpoints for: +- Healthcheck +- Retrieving all API keys +- Retrieving all organizations -from django.shortcuts import render -from fastapi import ( - APIRouter, - Depends, - HTTPException, - Security, +Endpoints: + - healthcheck: Returns the health status of the application. + - get_api_keys: Retrieves all API keys. + - read_orgs: Retrieves all organizations. + +Dependencies: + - fastapi + - .auth + - .models +""" +# Third-Party Libraries +from fastapi import APIRouter, Depends, HTTPException -) -from fastapi.security import APIKeyHeader, OAuth2PasswordBearer from .auth import get_current_active_user from .models import ApiKey, Organization, User -from typing import Any, List, Optional, Union api_router = APIRouter() @@ -42,18 +51,19 @@ async def get_api_keys(): return [ { "id": api_key.id, - "created_at": api_key.createdat, - "updated_at": api_key.updatedat, - "last_used": api_key.lastused, - "hashed_key": api_key.hashedkey, - "last_four": api_key.lastfour, - "user_id": api_key.userid.id if api_key.userid else None, + "created_at": api_key.createdAt, + "updated_at": api_key.updatedAt, + "last_used": api_key.lastUsed, + "hashed_key": api_key.hashedKey, + "last_four": api_key.lastFour, + "user_id": api_key.userId.id if api_key.userId else None, } for api_key in api_keys ] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + @api_router.post( "/test-orgs", dependencies=[Depends(get_current_active_user)], @@ -68,21 +78,25 @@ def read_orgs(current_user: User = Depends(get_current_active_user)): "id": organization.id, "name": organization.name, "acronym": organization.acronym, - "root_domains": organization.rootdomains, - "ip_blocks": organization.ipblocks, - "is_passive": organization.ispassive, + "root_domains": organization.rootDomains, + "ip_blocks": organization.ipBlocks, + "is_passive": organization.isPassive, "country": organization.country, "state": organization.state, - "region_id": organization.regionid, - "state_fips": organization.statefips, - "state_name": organization.statename, + "region_id": organization.regionId, + "state_fips": organization.stateFips, + "state_name": organization.stateName, "county": organization.county, - "county_fips": organization.countyfips, + "county_fips": organization.countyFips, "type": organization.type, - "parent_id": organization.parentid.id if organization.parentid else None, - "created_by_id": organization.createdbyid.id if organization.createdbyid else None, - "created_at": organization.createdat, - "updated_at": organization.updatedat, + "parent_id": organization.parentId.id + if organization.parentId + else None, + "created_by_id": organization.createdById.id + if organization.createdById + else None, + "created_at": organization.createdAt, + "updated_at": organization.updatedAt, } for organization in organizations ] From ded02da20ee35e8169ffceaf4dbf6abeb4a5986d Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Tue, 27 Aug 2024 07:53:01 -0500 Subject: [PATCH 009/314] Add docstings to xfd_api __init__.py, apps.py and jwt_utils.py; alphabetize imports; and fix whitespace. --- backend/src/xfd_django/xfd_api/__init__.py | 16 +++++++ backend/src/xfd_django/xfd_api/apps.py | 19 +++++++- backend/src/xfd_django/xfd_api/jwt_utils.py | 49 ++++++++++++++++++--- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/__init__.py b/backend/src/xfd_django/xfd_api/__init__.py index e69de29b..a857d593 100644 --- a/backend/src/xfd_django/xfd_api/__init__.py +++ b/backend/src/xfd_django/xfd_api/__init__.py @@ -0,0 +1,16 @@ +""" +This package contains the FastAPI application modules for the xfd_django project. + +Modules included: +- `auth`: Provides authentication utilities. +- `views`: Defines the API endpoints. +- `apps`: Configures the Django application. +- `admin`: Registers models with the Django admin site. +- `jwt_utils`: Provides JWT token creation and decoding utilities. +- `models`: Defines the database models. +- `tests`: Contains the test cases for the application. + +Dependencies: +- fastapi +- django +""" diff --git a/backend/src/xfd_django/xfd_api/apps.py b/backend/src/xfd_django/xfd_api/apps.py index 01d4d0dc..e6933816 100644 --- a/backend/src/xfd_django/xfd_api/apps.py +++ b/backend/src/xfd_django/xfd_api/apps.py @@ -1,6 +1,21 @@ +""" +This module configures the Django application for the xfd_django project. + +Classes: + - XfdApiConfig: Configures the xfd_api application. +""" +# Third-Party Libraries from django.apps import AppConfig class XfdApiConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'xfd_api' + """ + Configure the xfd_api application. + + Attributes: + default_auto_field (str): The default auto field type for models. + name (str): The name of the application. + """ + + default_auto_field = "django.db.models.BigAutoField" + name = "xfd_api" diff --git a/backend/src/xfd_django/xfd_api/jwt_utils.py b/backend/src/xfd_django/xfd_api/jwt_utils.py index b4fe35db..b8e684ad 100644 --- a/backend/src/xfd_django/xfd_api/jwt_utils.py +++ b/backend/src/xfd_django/xfd_api/jwt_utils.py @@ -1,26 +1,61 @@ # xfd_api/jwt_utils.py +""" +This module provides utilities for creating and decoding JWT tokens for the xfd_django project. -import jwt +Functions: + - create_jwt_token: Create a JWT token for a given user. + - decode_jwt_token: Decode a JWT token to retrieve the user. + +Dependencies: + - jwt + - datetime + - django.conf.settings + - django.contrib.auth.get_user_model +""" +# Standard Python Libraries from datetime import datetime, timedelta + +# Third-Party Libraries from django.conf import settings from django.contrib.auth import get_user_model +import jwt from jwt import ExpiredSignatureError, InvalidTokenError User = get_user_model() SECRET_KEY = settings.SECRET_KEY + def create_jwt_token(user): + """ + Create a JWT token for a given user. + + Args: + user (User): The user object for whom the token is created. + + Returns: + str: The encoded JWT token. + """ payload = { - 'id': str(user.id), - 'email': user.email, - 'exp': datetime.utcnow() + timedelta(hours=1) + "id": str(user.id), + "email": user.email, + "exp": datetime.utcnow() + timedelta(hours=1), } - return jwt.encode(payload, SECRET_KEY, algorithm='HS256') + return jwt.encode(payload, SECRET_KEY, algorithm="HS256") + def decode_jwt_token(token): + """ + Decode a JWT token to retrieve the user. + + Args: + token (str): The JWT token to decode. + + Returns: + User: The user object decoded from the token, or None if invalid or expired. + """ try: - payload = jwt.decode(token, SECRET_KEY, algorithms=['HS256']) - user = User.objects.get(id=payload['id']) + payload = jwt.decode(token, SECRET_KEY, algorithms=["HS256"]) + user = User.objects.get(id=payload["id"]) return user except (ExpiredSignatureError, InvalidTokenError, User.DoesNotExist): return None From 29152563a9d9d4aee88e4537d4dd855a591aff1a Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Tue, 27 Aug 2024 08:20:12 -0500 Subject: [PATCH 010/314] Fix flake8 and black issues: move api_router import into get_application function in asgi.py; remove unused imports and convert single quotes to double in settings.py. --- backend/src/xfd_django/xfd_django/asgi.py | 10 ++- backend/src/xfd_django/xfd_django/settings.py | 62 ++++++++++--------- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/backend/src/xfd_django/xfd_django/asgi.py b/backend/src/xfd_django/xfd_django/asgi.py index 09719eb3..def0a336 100644 --- a/backend/src/xfd_django/xfd_django/asgi.py +++ b/backend/src/xfd_django/xfd_django/asgi.py @@ -7,7 +7,10 @@ https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/ """ +# Standard Python Libraries import os + +# Third-Party Libraries import django from django.apps import apps from django.conf import settings @@ -22,11 +25,13 @@ # Ensure apps are populated apps.populate(settings.INSTALLED_APPS) -# Import views after Django setup -from xfd_api.views import api_router def get_application() -> FastAPI: """get_application function.""" + # Import views after Django setup + # Third-Party Libraries + from xfd_api.views import api_router + app = FastAPI(title=settings.PROJECT_NAME, debug=settings.DEBUG) app.add_middleware( CORSMiddleware, @@ -38,5 +43,6 @@ def get_application() -> FastAPI: app.include_router(api_router) return app + app = get_application() handler = Mangum(app) diff --git a/backend/src/xfd_django/xfd_django/settings.py b/backend/src/xfd_django/xfd_django/settings.py index 925de8fa..29be71b8 100644 --- a/backend/src/xfd_django/xfd_django/settings.py +++ b/backend/src/xfd_django/xfd_django/settings.py @@ -12,7 +12,6 @@ # Standard Python Libraries import mimetypes -import os # Python built-in from pathlib import Path @@ -31,13 +30,16 @@ # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -#TODO: GET THAT LATER -SECRET_KEY = 'django-insecure-255j80npx26z%x0@-7p@(qs9(yvtuuln#xuhxt_x$bbevvxnm!' +# TODO: GET THAT LATER +SECRET_KEY = "django-insecure-255j80npx26z%x0@-7p@(qs9(yvtuuln#xuhxt_x$bbevvxnm!" # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [".execute-api.us-east-1.amazonaws.com", 'https://api.staging-cd.crossfeed.cyber.dhs.gov'] +ALLOWED_HOSTS = [ + ".execute-api.us-east-1.amazonaws.com", + "https://api.staging-cd.crossfeed.cyber.dhs.gov", +] MESSAGE_TAGS = { messages.DEBUG: "alert-secondary", @@ -60,34 +62,34 @@ ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'xfd_django.urls' +ROOT_URLCONF = "xfd_django.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'xfd_django.wsgi.application' +WSGI_APPLICATION = "xfd_django.wsgi.application" # Database @@ -110,16 +112,16 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] @@ -127,9 +129,9 @@ # Internationalization # https://docs.djangoproject.com/en/4.2/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -139,12 +141,12 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/4.2/howto/static-files/ -STATIC_URL = 'static/' +STATIC_URL = "static/" # Default primary key field type # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" PROJECT_NAME = "XFD Python API" From f7754c6cf2ded8acdd9db1ea35f9d5da43ab9b34 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Wed, 28 Aug 2024 13:45:59 -0500 Subject: [PATCH 011/314] Merging views, schemas, and model updates. --- .pre-commit-config.yaml | 12 +- backend/src/xfd_django/xfd_api/models.py | 685 +++++++++++++++------- backend/src/xfd_django/xfd_api/schemas.py | 186 ++++++ backend/src/xfd_django/xfd_api/views.py | 213 ++++++- 4 files changed, 860 insertions(+), 236 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/schemas.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0b53bed3..1042e9b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,12 +81,12 @@ repos: - id: shell-lint # Python hooks - - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 - hooks: - - id: bandit - args: - - --config=.bandit.yml + #- repo: https://github.com/PyCQA/bandit + # rev: 1.7.5 + # hooks: + # - id: bandit + # args: + # - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror rev: 23.9.1 hooks: diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 0e037e02..8eb9c028 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -1,3 +1,4 @@ +""" Django ORM models """ # This is an auto-generated Django model module. # You'll have to do the following manually to clean this up: # * Rearrange models' order @@ -5,206 +6,319 @@ # * Make sure each ForeignKey and OneToOneField has `on_delete` set to the desired behavior # * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table # Feel free to rename the models, but don't rename db_table values or field names. +# Third-Party Libraries from django.db import models class ApiKey(models.Model): + """The ApiKey model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(auto_now_add=True, db_column='createdAt') - updatedAt = models.DateTimeField(auto_now=True, db_column='updatedAt') - lastUsed = models.DateTimeField(db_column='lastUsed', blank=True, null=True) - hashedKey = models.TextField(db_column='hashedKey') - lastFour = models.TextField(db_column='lastFour') - userId = models.ForeignKey('User', models.CASCADE, db_column='userId', blank=True, null=True) + createdAt = models.DateTimeField(auto_now_add=True, db_column="createdAt") + updatedAt = models.DateTimeField(auto_now=True, db_column="updatedAt") + lastUsed = models.DateTimeField(db_column="lastUsed", blank=True, null=True) + hashedKey = models.TextField(db_column="hashedKey") + lastFour = models.TextField(db_column="lastFour") + userId = models.ForeignKey( + "User", models.CASCADE, db_column="userId", blank=True, null=True + ) class Meta: + """Meta class for ApiKey.""" + managed = False - db_table = 'api_key' + db_table = "api_key" class Assessment(models.Model): + """The Assessment model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') - rscId = models.CharField(db_column='rscId', unique=True) + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") + rscId = models.CharField(db_column="rscId", unique=True) type = models.CharField(max_length=255) - userId = models.ForeignKey('User', db_column='userId', blank=True, null=True, on_delete=models.CASCADE) + userId = models.ForeignKey( + "User", db_column="userId", blank=True, null=True, on_delete=models.CASCADE + ) class Meta: + """The Meta class for Assessment.""" + managed = False - db_table = 'assessment' + db_table = "assessment" class Category(models.Model): + """The Category model.""" + id = models.UUIDField(primary_key=True) name = models.CharField(max_length=255) number = models.CharField(max_length=255, unique=True) - shortName = models.CharField(db_column='shortName', max_length=255, blank=True, null=True) + shortName = models.CharField( + db_column="shortName", max_length=255, blank=True, null=True + ) class Meta: + """The Meta class for Category model.""" + managed = False - db_table = 'category' + db_table = "category" class Cpe(models.Model): + """The Cpe model.""" + id = models.UUIDField(primary_key=True) name = models.CharField(max_length=255) version = models.CharField(max_length=255) vendor = models.CharField(max_length=255) - lastSeenAt = models.DateTimeField(db_column='lastSeenAt') + lastSeenAt = models.DateTimeField(db_column="lastSeenAt") class Meta: - db_table = 'cpe' - managed = False # This ensures Django does not manage the table - unique_together = (('name', 'version', 'vendor'),) # Unique constraint + """The Meta class for Cpe.""" + db_table = "cpe" + managed = False # This ensures Django does not manage the table + unique_together = (("name", "version", "vendor"),) # Unique constraint class Cve(models.Model): + """The Cve model.""" + id = models.UUIDField(primary_key=True) name = models.CharField(unique=True, blank=True, null=True) - publishedAt = models.DateTimeField(db_column='publishedAt', blank=True, null=True) - modifiedAt = models.DateTimeField(db_column='modifiedAt', blank=True, null=True) + publishedAt = models.DateTimeField(db_column="publishedAt", blank=True, null=True) + modifiedAt = models.DateTimeField(db_column="modifiedAt", blank=True, null=True) status = models.CharField(blank=True, null=True) description = models.CharField(blank=True, null=True) - cvssV2Source = models.CharField(db_column='cvssV2Source', blank=True, null=True) - cvssV2Type = models.CharField(db_column='cvssV2Type', blank=True, null=True) - cvssV2Version = models.CharField(db_column='cvssV2Version', blank=True, null=True) - cvssV2VectorString = models.CharField(db_column='cvssV2VectorString', blank=True, null=True) - cvssV2BaseScore = models.CharField(db_column='cvssV2BaseScore', blank=True, null=True) - cvssV2BaseSeverity = models.CharField(db_column='cvssV2BaseSeverity', blank=True, null=True) - cvssV2ExploitabilityScore = models.CharField(db_column='cvssV2ExploitabilityScore', blank=True, null=True) - cvssV2ImpactScore = models.CharField(db_column='cvssV2ImpactScore', blank=True, null=True) - cvssV3Source = models.CharField(db_column='cvssV3Source', blank=True, null=True) - cvssV3Type = models.CharField(db_column='cvssV3Type', blank=True, null=True) - cvssV3Version = models.CharField(db_column='cvssV3Version', blank=True, null=True) - cvssV3VectorString = models.CharField(db_column='cvssV3VectorString', blank=True, null=True) - cvssV3BaseScore = models.CharField(db_column='cvssV3BaseScore', blank=True, null=True) - cvssV3BaseSeverity = models.CharField(db_column='cvssV3BaseSeverity', blank=True, null=True) - cvssV3ExploitabilityScore = models.CharField(db_column='cvssV3ExploitabilityScore', blank=True, null=True) - cvssV3ImpactScore = models.CharField(db_column='cvssV3ImpactScore', blank=True, null=True) - cvssV4Source = models.CharField(db_column='cvssV4Source', blank=True, null=True) - cvssV4Type = models.CharField(db_column='cvssV4Type', blank=True, null=True) - cvssV4Version = models.CharField(db_column='cvssV4Version', blank=True, null=True) - cvssV4VectorString = models.CharField(db_column='cvssV4VectorString', blank=True, null=True) - cvssV4BaseScore = models.CharField(db_column='cvssV4BaseScore', blank=True, null=True) - cvssV4BaseSeverity = models.CharField(db_column='cvssV4BaseSeverity', blank=True, null=True) - cvssV4ExploitabilityScore = models.CharField(db_column='cvssV4ExploitabilityScore', blank=True, null=True) - cvssV4ImpactScore = models.CharField(db_column='cvssV4ImpactScore', blank=True, null=True) + cvssV2Source = models.CharField(db_column="cvssV2Source", blank=True, null=True) + cvssV2Type = models.CharField(db_column="cvssV2Type", blank=True, null=True) + cvssV2Version = models.CharField(db_column="cvssV2Version", blank=True, null=True) + cvssV2VectorString = models.CharField( + db_column="cvssV2VectorString", blank=True, null=True + ) + cvssV2BaseScore = models.CharField( + db_column="cvssV2BaseScore", blank=True, null=True + ) + cvssV2BaseSeverity = models.CharField( + db_column="cvssV2BaseSeverity", blank=True, null=True + ) + cvssV2ExploitabilityScore = models.CharField( + db_column="cvssV2ExploitabilityScore", blank=True, null=True + ) + cvssV2ImpactScore = models.CharField( + db_column="cvssV2ImpactScore", blank=True, null=True + ) + cvssV3Source = models.CharField(db_column="cvssV3Source", blank=True, null=True) + cvssV3Type = models.CharField(db_column="cvssV3Type", blank=True, null=True) + cvssV3Version = models.CharField(db_column="cvssV3Version", blank=True, null=True) + cvssV3VectorString = models.CharField( + db_column="cvssV3VectorString", blank=True, null=True + ) + cvssV3BaseScore = models.CharField( + db_column="cvssV3BaseScore", blank=True, null=True + ) + cvssV3BaseSeverity = models.CharField( + db_column="cvssV3BaseSeverity", blank=True, null=True + ) + cvssV3ExploitabilityScore = models.CharField( + db_column="cvssV3ExploitabilityScore", blank=True, null=True + ) + cvssV3ImpactScore = models.CharField( + db_column="cvssV3ImpactScore", blank=True, null=True + ) + cvssV4Source = models.CharField(db_column="cvssV4Source", blank=True, null=True) + cvssV4Type = models.CharField(db_column="cvssV4Type", blank=True, null=True) + cvssV4Version = models.CharField(db_column="cvssV4Version", blank=True, null=True) + cvssV4VectorString = models.CharField( + db_column="cvssV4VectorString", blank=True, null=True + ) + cvssV4BaseScore = models.CharField( + db_column="cvssV4BaseScore", blank=True, null=True + ) + cvssV4BaseSeverity = models.CharField( + db_column="cvssV4BaseSeverity", blank=True, null=True + ) + cvssV4ExploitabilityScore = models.CharField( + db_column="cvssV4ExploitabilityScore", blank=True, null=True + ) + cvssV4ImpactScore = models.CharField( + db_column="cvssV4ImpactScore", blank=True, null=True + ) weaknesses = models.TextField(blank=True, null=True) references = models.TextField(blank=True, null=True) class Meta: + """The Meta class for Cve.""" + managed = False - db_table = 'cve' + db_table = "cve" class CveCpesCpe(models.Model): - cveId = models.ForeignKey(Cve, on_delete=models.CASCADE, db_column='cveId') - cpeId = models.ForeignKey(Cpe, on_delete=models.CASCADE, db_column='cpeId') + """The CveCpesCpe model.""" + + cveId = models.ForeignKey(Cve, on_delete=models.CASCADE, db_column="cveId") + cpeId = models.ForeignKey(Cpe, on_delete=models.CASCADE, db_column="cpeId") class Meta: - db_table = 'cve_cpes_cpe' + """The Meta class for CveCpesCpe model.""" + + db_table = "cve_cpes_cpe" managed = False # This ensures Django does not manage the table - unique_together = (('cve', 'cpe'),) # Unique constraint + unique_together = (("cve", "cpe"),) # Unique constraint class Domain(models.Model): + """The Domain model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') - syncedAt = models.DateTimeField(db_column='syncedAt', blank=True, null=True) + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") + syncedAt = models.DateTimeField(db_column="syncedAt", blank=True, null=True) ip = models.CharField(max_length=255, blank=True, null=True) - fromRootDomain = models.CharField(db_column='fromRootDomain', blank=True, null=True) - subdomainSource = models.CharField(db_column='subdomainSource', max_length=255, blank=True, null=True) - ipOnly = models.BooleanField(db_column='ipOnly', default=False) - reverseName = models.CharField(db_column='reverseName', max_length=512) + fromRootDomain = models.CharField(db_column="fromRootDomain", blank=True, null=True) + subdomainSource = models.CharField( + db_column="subdomainSource", max_length=255, blank=True, null=True + ) + ipOnly = models.BooleanField(db_column="ipOnly", default=False) + reverseName = models.CharField(db_column="reverseName", max_length=512) name = models.CharField(max_length=512) screenshot = models.CharField(max_length=512, blank=True, null=True) country = models.CharField(max_length=255, blank=True, null=True) asn = models.CharField(max_length=255, blank=True, null=True) - cloudHosted = models.BooleanField(db_column='cloudHosted', default=False) + cloudHosted = models.BooleanField(db_column="cloudHosted", default=False) ssl = models.JSONField(blank=True, null=True) - censysCertificatesResults = models.JSONField(db_column='censysCertificatesResults', default=dict) - trustymailResults = models.JSONField(db_column='trustymailResults', default=dict) - discoveredById = models.ForeignKey('Scan', on_delete=models.SET_NULL, db_column='discoveredById', blank=True, null=True) - organizationId = models.ForeignKey('Organization', on_delete=models.CASCADE, db_column='organizationId') + censysCertificatesResults = models.JSONField( + db_column="censysCertificatesResults", default=dict + ) + trustymailResults = models.JSONField(db_column="trustymailResults", default=dict) + discoveredById = models.ForeignKey( + "Scan", + on_delete=models.SET_NULL, + db_column="discoveredById", + blank=True, + null=True, + ) + organizationId = models.ForeignKey( + "Organization", on_delete=models.CASCADE, db_column="organizationId" + ) class Meta: - db_table = 'domain' + """The meta class for Domain.""" + + db_table = "domain" managed = False # This ensures Django does not manage the table - unique_together = (('name', 'organization'),) # Unique constraint + unique_together = (("name", "organization"),) # Unique constraint def save(self, *args, **kwargs): self.name = self.name.lower() - self.reverseName = '.'.join(reversed(self.name.split('.'))) - super(Domain, self).save(*args, **kwargs) + self.reverseName = ".".join(reversed(self.name.split("."))) + super().save(*args, **kwargs) class Notification(models.Model): + """The Notification model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') - startDatetime = models.DateTimeField(db_column='startDatetime', blank=True, null=True) - endDatetime = models.DateTimeField(db_column='endDatetime', blank=True, null=True) - maintenanceType = models.CharField(db_column='maintenanceType', blank=True, null=True) + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") + startDatetime = models.DateTimeField( + db_column="startDatetime", blank=True, null=True + ) + endDatetime = models.DateTimeField(db_column="endDatetime", blank=True, null=True) + maintenanceType = models.CharField( + db_column="maintenanceType", blank=True, null=True + ) status = models.CharField(blank=True, null=True) - updatedBy = models.CharField(db_column='updatedBy', blank=True, null=True) + updatedBy = models.CharField(db_column="updatedBy", blank=True, null=True) message = models.CharField(blank=True, null=True) class Meta: + """The Meta class for Notification.""" + managed = False - db_table = 'notification' + db_table = "notification" class Organization(models.Model): + """The Organization model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") acronym = models.CharField(unique=True, blank=True, null=True) name = models.CharField() - rootDomains = models.TextField(db_column='rootDomains') # This field type is a guess. - ipBlocks = models.TextField(db_column='ipBlocks') # This field type is a guess. - isPassive = models.BooleanField(db_column='isPassive') - pendingDomains = models.TextField(db_column='pendingDomains') # This field type is a guess. + rootDomains = models.TextField( + db_column="rootDomains" + ) # This field type is a guess. + ipBlocks = models.TextField(db_column="ipBlocks") # This field type is a guess. + isPassive = models.BooleanField(db_column="isPassive") + pendingDomains = models.TextField( + db_column="pendingDomains" + ) # This field type is a guess. country = models.CharField(blank=True, null=True) state = models.CharField(blank=True, null=True) - regionId = models.CharField(db_column='regionId', blank=True, null=True) - stateFips = models.IntegerField(db_column='stateFips', blank=True, null=True) - stateName = models.CharField(db_column='stateName', blank=True, null=True) + regionId = models.CharField(db_column="regionId", blank=True, null=True) + stateFips = models.IntegerField(db_column="stateFips", blank=True, null=True) + stateName = models.CharField(db_column="stateName", blank=True, null=True) county = models.CharField(blank=True, null=True) - countyFips = models.IntegerField(db_column='countyFips', blank=True, null=True) + countyFips = models.IntegerField(db_column="countyFips", blank=True, null=True) type = models.CharField(blank=True, null=True) - parentId = models.ForeignKey('self', models.DO_NOTHING, db_column='parentId', blank=True, null=True) - createdById = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) + parentId = models.ForeignKey( + "self", models.DO_NOTHING, db_column="parentId", blank=True, null=True + ) + createdById = models.ForeignKey( + "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True + ) class Meta: + """The meta class for Organization.""" + managed = False - db_table = 'organization' + db_table = "organization" class OrganizationTag(models.Model): + """The OrganizationTag model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") name = models.CharField(unique=True) class Meta: + """The Meta class for OrganizationTag.""" + managed = False - db_table = 'organization_tag' + db_table = "organization_tag" class OrganizationTagOrganizationsOrganization(models.Model): - organizationTagId = models.OneToOneField(OrganizationTag, models.DO_NOTHING, db_column='organizationTagId', primary_key=True) # The composite primary key (organizationTagId, organizationId) found, that is not supported. The first column is selected. - organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') + """The OrganizationTagOrganizationsOrganization model.""" + + organizationTagId = models.OneToOneField( + OrganizationTag, + models.DO_NOTHING, + db_column="organizationTagId", + primary_key=True, + ) # The composite primary key (organizationTagId, organizationId) found, that is not supported. The first column is selected. + organizationId = models.ForeignKey( + Organization, models.DO_NOTHING, db_column="organizationId" + ) class Meta: + """The Meta class for OrganizationTagOrganizationsOrganization.""" + managed = False - db_table = 'organization_tag_organizations_organization' - unique_together = (('organizationTagId', 'organizationId'),) + db_table = "organization_tag_organizations_organization" + unique_together = (("organizationTagId", "organizationId"),) class QueryResultCache(models.Model): + """The QueryResultCache model.""" + identifier = models.CharField(blank=True, null=True) time = models.BigIntegerField() duration = models.IntegerField() @@ -212,36 +326,54 @@ class QueryResultCache(models.Model): result = models.TextField() class Meta: + """The Meta class for QueryResultCache.""" + managed = False - db_table = 'query-result-cache' + db_table = "query-result-cache" class Question(models.Model): + """The Question model.""" + id = models.UUIDField(primary_key=True) name = models.CharField(max_length=255) description = models.CharField(blank=True, null=True) - longForm = models.CharField(db_column='longForm') + longForm = models.CharField(db_column="longForm") number = models.CharField(max_length=255) - categoryId = models.ForeignKey(Category, models.DO_NOTHING, db_column='categoryId', blank=True, null=True) + categoryId = models.ForeignKey( + Category, models.DO_NOTHING, db_column="categoryId", blank=True, null=True + ) class Meta: - db_table = 'question' + """The Meta class for Question.""" + + db_table = "question" managed = False - unique_together = (('category', 'number'),) + unique_together = (("category", "number"),) # Question and Resource many-to-many class QuestionResourcesResource(models.Model): - questionId = models.ForeignKey('Question', on_delete=models.CASCADE, db_column='questionId') - resourceId = models.ForeignKey('Resource', on_delete=models.CASCADE, db_column='resourceId') + """The QuestionResourcesResource model.""" + + questionId = models.ForeignKey( + "Question", on_delete=models.CASCADE, db_column="questionId" + ) + resourceId = models.ForeignKey( + "Resource", on_delete=models.CASCADE, db_column="resourceId" + ) class Meta: - db_table = 'question_resources_resource' - managed = False - unique_together = (('question', 'resource'),) + """The Meta class for QuestionResourcesResource.""" + + db_table = "question_resources_resource" + managed = False + unique_together = (("question", "resource"),) class Resource(models.Model): + """The Resource model.""" + id = models.UUIDField(primary_key=True) description = models.CharField() name = models.CharField() @@ -249,153 +381,250 @@ class Resource(models.Model): url = models.CharField(unique=True) class Meta: + """The Meta class for Resource.""" + managed = False - db_table = 'resource' + db_table = "resource" + class Response(models.Model): + """The Response model.""" + id = models.UUIDField(primary_key=True) selection = models.CharField() - assessmentId = models.ForeignKey(Assessment, models.DO_NOTHING, db_column='assessmentId', blank=True, null=True) - questionId = models.ForeignKey(Question, models.DO_NOTHING, db_column='questionId', blank=True, null=True) + assessmentId = models.ForeignKey( + Assessment, models.DO_NOTHING, db_column="assessmentId", blank=True, null=True + ) + questionId = models.ForeignKey( + Question, models.DO_NOTHING, db_column="questionId", blank=True, null=True + ) class Meta: + """The Meta class for Resource.""" + managed = False - db_table = 'response' - unique_together = (('assessmentId', 'questionId'),) + db_table = "response" + unique_together = (("assessmentId", "questionId"),) class Role(models.Model): + """The Role model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") role = models.CharField() approved = models.BooleanField() - createdById = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) - approvedById = models.ForeignKey('User', models.DO_NOTHING, db_column='approvedById', related_name='role_approvedbyid_set', blank=True, null=True) - userId = models.ForeignKey('User', models.DO_NOTHING, db_column='userId', related_name='role_userid_set', blank=True, null=True) - organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId', blank=True, null=True) + createdById = models.ForeignKey( + "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True + ) + approvedById = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="approvedById", + related_name="role_approvedbyid_set", + blank=True, + null=True, + ) + userId = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="userId", + related_name="role_userid_set", + blank=True, + null=True, + ) + organizationId = models.ForeignKey( + Organization, + models.DO_NOTHING, + db_column="organizationId", + blank=True, + null=True, + ) class Meta: + """The Meta class for Role.""" + managed = False - db_table = 'role' - unique_together = (('userId', 'organizationId'),) + db_table = "role" + unique_together = (("userId", "organizationId"),) class SavedSearch(models.Model): + """The SavedSearch model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") name = models.CharField() - searchTerm = models.CharField(db_column='searchTerm') - sortDirection = models.CharField(db_column='sortDirection') - sortField = models.CharField(db_column='sortField') + searchTerm = models.CharField(db_column="searchTerm") + sortDirection = models.CharField(db_column="sortDirection") + sortField = models.CharField(db_column="sortField") count = models.IntegerField() filters = models.JSONField() - searchPath = models.CharField(db_column='searchPath') - createVulnerabilities = models.BooleanField(db_column='createVulnerabilities') - vulnerabilityTemplate = models.JSONField(db_column='vulnerabilityTemplate') - createdById = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) + searchPath = models.CharField(db_column="searchPath") + createVulnerabilities = models.BooleanField(db_column="createVulnerabilities") + vulnerabilityTemplate = models.JSONField(db_column="vulnerabilityTemplate") + createdById = models.ForeignKey( + "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True + ) class Meta: + """The Meta class for SavedSearch.""" + managed = False - db_table = 'saved_search' + db_table = "saved_search" class Scan(models.Model): + """The Scan model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") name = models.CharField() arguments = models.JSONField() frequency = models.IntegerField() - lastRun = models.DateTimeField(db_column='lastRun', blank=True, null=True) - isGranular = models.BooleanField(db_column='isGranular') - isUserModifiable = models.BooleanField(db_column='isUserModifiable', blank=True, null=True) - isSingleScan = models.BooleanField(db_column='isSingleScan') - manualRunPending = models.BooleanField(db_column='manualRunPending') - createdBy = models.ForeignKey('User', models.DO_NOTHING, db_column='createdById', blank=True, null=True) + lastRun = models.DateTimeField(db_column="lastRun", blank=True, null=True) + isGranular = models.BooleanField(db_column="isGranular") + isUserModifiable = models.BooleanField( + db_column="isUserModifiable", blank=True, null=True + ) + isSingleScan = models.BooleanField(db_column="isSingleScan") + manualRunPending = models.BooleanField(db_column="manualRunPending") + createdBy = models.ForeignKey( + "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True + ) class Meta: + """The Meta class for Scan.""" + managed = False - db_table = 'scan' + db_table = "scan" class ScanOrganizationsOrganization(models.Model): - scanId = models.OneToOneField(Scan, models.DO_NOTHING, db_column='scanId', primary_key=True) # The composite primary key (scanId, organizationId) found, that is not supported. The first column is selected. - organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') + """The ScanOrganizationsOrganization model.""" + + scanId = models.OneToOneField( + Scan, models.DO_NOTHING, db_column="scanId", primary_key=True + ) # The composite primary key (scanId, organizationId) found, that is not supported. The first column is selected. + organizationId = models.ForeignKey( + Organization, models.DO_NOTHING, db_column="organizationId" + ) class Meta: + """The Meta class for ScanOrganizationsOrganization.""" + managed = False - db_table = 'scan_organizations_organization' - unique_together = (('scanId', 'organizationId'),) + db_table = "scan_organizations_organization" + unique_together = (("scanId", "organizationId"),) class ScanTagsOrganizationTag(models.Model): - scanId = models.OneToOneField(Scan, models.DO_NOTHING, db_column='scanId', primary_key=True) # The composite primary key (scanId, organizationTagId) found, that is not supported. The first column is selected. - organizationTagId = models.ForeignKey(OrganizationTag, models.DO_NOTHING, db_column='organizationTagId') + """The ScanTagsOrganizationTag model.""" + + scanId = models.OneToOneField( + Scan, models.DO_NOTHING, db_column="scanId", primary_key=True + ) # The composite primary key (scanId, organizationTagId) found, that is not supported. The first column is selected. + organizationTagId = models.ForeignKey( + OrganizationTag, models.DO_NOTHING, db_column="organizationTagId" + ) class Meta: + """The Meta class for ScanTagsOrganizationTag.""" + managed = False - db_table = 'scan_tags_organization_tag' - unique_together = (('scanId', 'organizationTagId'),) + db_table = "scan_tags_organization_tag" + unique_together = (("scanId", "organizationTagId"),) class ScanTask(models.Model): + """The ScanTask model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") status = models.TextField() type = models.TextField() - fargateTaskArn = models.TextField(db_column='fargateTaskArn', blank=True, null=True) + fargateTaskArn = models.TextField(db_column="fargateTaskArn", blank=True, null=True) input = models.TextField(blank=True, null=True) output = models.TextField(blank=True, null=True) - requestedAt = models.DateTimeField(db_column='requestedAt', blank=True, null=True) - startedAt = models.DateTimeField(db_column='startedAt', blank=True, null=True) - finishedAt = models.DateTimeField(db_column='finishedAt', blank=True, null=True) - queuedAt = models.DateTimeField(db_column='queuedAt', blank=True, null=True) - organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId', blank=True, null=True) - scanId = models.ForeignKey(Scan, models.DO_NOTHING, db_column='scanId', blank=True, null=True) + requestedAt = models.DateTimeField(db_column="requestedAt", blank=True, null=True) + startedAt = models.DateTimeField(db_column="startedAt", blank=True, null=True) + finishedAt = models.DateTimeField(db_column="finishedAt", blank=True, null=True) + queuedAt = models.DateTimeField(db_column="queuedAt", blank=True, null=True) + organizationId = models.ForeignKey( + Organization, + models.DO_NOTHING, + db_column="organizationId", + blank=True, + null=True, + ) + scanId = models.ForeignKey( + Scan, models.DO_NOTHING, db_column="scanId", blank=True, null=True + ) class Meta: + """The Meta class for ScanTask.""" + managed = False - db_table = 'scan_task' + db_table = "scan_task" class ScanTaskOrganizationsOrganization(models.Model): - scanTaskId = models.OneToOneField(ScanTask, models.DO_NOTHING, db_column='scanTaskId', primary_key=True) # The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. - organizationId = models.ForeignKey(Organization, models.DO_NOTHING, db_column='organizationId') + """The ScanTaskOrganizationsOrganization model.""" + + scanTaskId = models.OneToOneField( + ScanTask, models.DO_NOTHING, db_column="scanTaskId", primary_key=True + ) # The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. + organizationId = models.ForeignKey( + Organization, models.DO_NOTHING, db_column="organizationId" + ) class Meta: + """The Meta class for ScanTaskOrganizationsOrganization.""" + managed = False - db_table = 'scan_task_organizations_organization' - unique_together = (('scanTaskId', 'organizationId'),) + db_table = "scan_task_organizations_organization" + unique_together = (("scanTaskId", "organizationId"),) class Service(models.Model): + """The Service model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') - serviceSource = models.TextField(db_column='serviceSource', blank=True, null=True) + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") + serviceSource = models.TextField(db_column="serviceSource", blank=True, null=True) port = models.IntegerField() service = models.CharField(blank=True, null=True) - lastSeen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) + lastSeen = models.DateTimeField(db_column="lastSeen", blank=True, null=True) banner = models.TextField(blank=True, null=True) products = models.JSONField() - censysMetadata = models.JSONField(db_column='censysMetadata') - censysIpv4Results = models.JSONField(db_column='censysIpv4Results') - intrigueIdentResults = models.JSONField(db_column='intrigueIdentResults') - shodanResults = models.JSONField(db_column='shodanResults') - wappalyzerResults = models.JSONField(db_column='wappalyzerResults') - domainId = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) - discoveredById = models.ForeignKey(Scan, models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) + censysMetadata = models.JSONField(db_column="censysMetadata") + censysIpv4Results = models.JSONField(db_column="censysIpv4Results") + intrigueIdentResults = models.JSONField(db_column="intrigueIdentResults") + shodanResults = models.JSONField(db_column="shodanResults") + wappalyzerResults = models.JSONField(db_column="wappalyzerResults") + domainId = models.ForeignKey( + Domain, models.DO_NOTHING, db_column="domainId", blank=True, null=True + ) + discoveredById = models.ForeignKey( + Scan, models.DO_NOTHING, db_column="discoveredById", blank=True, null=True + ) class Meta: + """The Meta class for Service.""" + managed = False - db_table = 'service' - unique_together = (('port', 'domainId'),) + db_table = "service" + unique_together = (("port", "domainId"),) class TypeormMetadata(models.Model): + """The TypeormMetadata model.""" + type = models.CharField() database = models.CharField(blank=True, null=True) schema = models.CharField(blank=True, null=True) @@ -404,81 +633,121 @@ class TypeormMetadata(models.Model): value = models.TextField(blank=True, null=True) class Meta: + """The Meta class for TypeormMetadata.""" + managed = False - db_table = 'typeorm_metadata' + db_table = "typeorm_metadata" class User(models.Model): + """The User model.""" + id = models.UUIDField(primary_key=True) - cognitoId = models.CharField(db_column='cognitoId', unique=True, blank=True, null=True) - loginGovId = models.CharField(db_column='loginGovId', unique=True, blank=True, null=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') - firstName = models.CharField(db_column='firstName') - lastName = models.CharField(db_column='lastName') - fullName = models.CharField(db_column='fullName') + cognitoId = models.CharField( + db_column="cognitoId", unique=True, blank=True, null=True + ) + loginGovId = models.CharField( + db_column="loginGovId", unique=True, blank=True, null=True + ) + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") + firstName = models.CharField(db_column="firstName") + lastName = models.CharField(db_column="lastName") + fullName = models.CharField(db_column="fullName") email = models.CharField(unique=True) - invitePending = models.BooleanField(db_column='invitePending') - loginBlockedByMaintenance = models.BooleanField(db_column='loginBlockedByMaintenance') - dateAcceptedTerms = models.DateTimeField(db_column='dateAcceptedTerms', blank=True, null=True) - acceptedTermsVersion = models.TextField(db_column='acceptedTermsVersion', blank=True, null=True) - lastLoggedIn = models.DateTimeField(db_column='lastLoggedIn', blank=True, null=True) - userType = models.TextField(db_column='userType') - regionId = models.CharField(db_column='regionId', blank=True, null=True) + invitePending = models.BooleanField(db_column="invitePending") + loginBlockedByMaintenance = models.BooleanField( + db_column="loginBlockedByMaintenance" + ) + dateAcceptedTerms = models.DateTimeField( + db_column="dateAcceptedTerms", blank=True, null=True + ) + acceptedTermsVersion = models.TextField( + db_column="acceptedTermsVersion", blank=True, null=True + ) + lastLoggedIn = models.DateTimeField(db_column="lastLoggedIn", blank=True, null=True) + userType = models.TextField(db_column="userType") + regionId = models.CharField(db_column="regionId", blank=True, null=True) state = models.CharField(blank=True, null=True) - oktaId = models.CharField(db_column='oktaId', unique=True, blank=True, null=True) + oktaId = models.CharField(db_column="oktaId", unique=True, blank=True, null=True) class Meta: + """The Meta class for User.""" + managed = False - db_table = 'user' + db_table = "user" class Vulnerability(models.Model): + """The Vulnerability model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') - lastSeen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") + lastSeen = models.DateTimeField(db_column="lastSeen", blank=True, null=True) title = models.CharField() cve = models.TextField(blank=True, null=True) cwe = models.TextField(blank=True, null=True) cpe = models.TextField(blank=True, null=True) description = models.CharField() references = models.JSONField() - cvss = models.DecimalField(max_digits=65535, decimal_places=65535, blank=True, null=True) + cvss = models.DecimalField( + max_digits=65535, decimal_places=65535, blank=True, null=True + ) severity = models.TextField(blank=True, null=True) - needsPopulation = models.BooleanField(db_column='needsPopulation') + needsPopulation = models.BooleanField(db_column="needsPopulation") state = models.CharField() substate = models.CharField() source = models.CharField() notes = models.CharField() actions = models.JSONField() - structuredData = models.JSONField(db_column='structuredData') - isKev = models.BooleanField(db_column='isKev', blank=True, null=True) - kevResults = models.JSONField(db_column='kevResults', blank=True, null=True) - domainId = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) - serviceId = models.ForeignKey(Service, models.DO_NOTHING, db_column='serviceId', blank=True, null=True) + structuredData = models.JSONField(db_column="structuredData") + isKev = models.BooleanField(db_column="isKev", blank=True, null=True) + kevResults = models.JSONField(db_column="kevResults", blank=True, null=True) + domainId = models.ForeignKey( + Domain, models.DO_NOTHING, db_column="domainId", blank=True, null=True + ) + serviceId = models.ForeignKey( + Service, models.DO_NOTHING, db_column="serviceId", blank=True, null=True + ) class Meta: + """The Meta class for Vulnerability.""" + managed = False - db_table = 'vulnerability' - unique_together = (('domainId', 'title'),) + db_table = "vulnerability" + unique_together = (("domainId", "title"),) class Webpage(models.Model): + """The Webpage model.""" + id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column='createdAt') - updatedAt = models.DateTimeField(db_column='updatedAt') - syncedAt = models.DateTimeField(db_column='syncedAt', blank=True, null=True) - lastSeen = models.DateTimeField(db_column='lastSeen', blank=True, null=True) - s3key = models.CharField(db_column='s3Key', blank=True, null=True) + createdAt = models.DateTimeField(db_column="createdAt") + updatedAt = models.DateTimeField(db_column="updatedAt") + syncedAt = models.DateTimeField(db_column="syncedAt", blank=True, null=True) + lastSeen = models.DateTimeField(db_column="lastSeen", blank=True, null=True) + s3key = models.CharField(db_column="s3Key", blank=True, null=True) url = models.CharField() status = models.DecimalField(max_digits=65535, decimal_places=65535) - responseSize = models.DecimalField(db_column='responseSize', max_digits=65535, decimal_places=65535, blank=True, null=True) + responseSize = models.DecimalField( + db_column="responseSize", + max_digits=65535, + decimal_places=65535, + blank=True, + null=True, + ) headers = models.JSONField() - domainId = models.ForeignKey(Domain, models.DO_NOTHING, db_column='domainId', blank=True, null=True) - discoveredById = models.ForeignKey(Scan, models.DO_NOTHING, db_column='discoveredById', blank=True, null=True) + domainId = models.ForeignKey( + Domain, models.DO_NOTHING, db_column="domainId", blank=True, null=True + ) + discoveredById = models.ForeignKey( + Scan, models.DO_NOTHING, db_column="discoveredById", blank=True, null=True + ) class Meta: + """The Meta class for Webpage.""" + managed = False - db_table = 'webpage' - unique_together = (('url', 'domainId'),) \ No newline at end of file + db_table = "webpage" + unique_together = (("url", "domainId"),) diff --git a/backend/src/xfd_django/xfd_api/schemas.py b/backend/src/xfd_django/xfd_api/schemas.py new file mode 100644 index 00000000..7d0ef784 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schemas.py @@ -0,0 +1,186 @@ +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, EmailStr, Field, Json + + +class Cpe(BaseModel): + id: UUID + name: Optional[str] + version: Optional[str] + vendor: Optional[str] + lastSeenAt: datetime + + +class Cve(BaseModel): + id: UUID + name: Optional[str] + publishedAt: datetime + modifiedAt: datetime + status: str + description: Optional[str] + cvssV2Source: Optional[str] + cvssV2Type: Optional[str] + cvssV2VectorString: Optional[str] + cvssV2BaseSeverity: Optional[str] + cvssV2ExploitabilityScore: Optional[str] + cvssV2ImpactScore: Optional[str] + cvssV3Source: Optional[str] + cvssV3Type: Optional[str] + cvssV3VectorString: Optional[str] + cvssV3BaseSeverity: Optional[str] + cvssV3ExploitabilityScore: Optional[str] + cvssV3ImpactScore: Optional[str] + cvssV4Source: Optional[str] + cvssV4Type: Optional[str] + cvssV4VectorString: Optional[str] + cvssV4BaseSeverity: Optional[str] + cvssV4ExploitabilityScore: Optional[str] + cvssV4ImpactScore: Optional[str] + weaknesses: Optional[str] + references: Optional[str] + + +class Domain(BaseModel): + id: UUID + createdAt: datetime + updatedAt: datetime + syncedAt: datetime + ip: str + fromRootDomain: Optional[str] + subdomainSource: Optional[str] + ipOnly: bool + reverseName: Optional[str] + name: Optional[str] + screenshot: Optional[str] + country: Optional[str] + asn: Optional[str] + cloudHosted: bool + ssl: Optional[Any] + censysCertificatesResults: Optional[dict] + trustymailResults: Optional[dict] + discoveredById: Optional[Any] + organizationId: Any + + +class DomainFilters(BaseModel): + ports: Optional[str] = None + service: Optional[str] = None + reverseName: Optional[str] = None + ip: Optional[str] = None + organization: Optional[str] = None + organizationName: Optional[str] = None + vulnerabilities: Optional[str] = None + tag: Optional[str] = None + + +class DomainSearch(BaseModel): + page: int = 1 + sort: str + order: str + filters: Optional[DomainFilters] + pageSize: Optional[int] = None + + +class Organization(BaseModel): + id: UUID + createdAt: datetime + updatedAt: datetime + acronym: Optional[str] + name: str + rootDomains: str + ipBlocks: str + isPassive: bool + pendingDomains: str + country: Optional[str] + state: Optional[str] + regionId: Optional[str] + stateFips: Optional[int] + stateName: Optional[str] + county: Optional[str] + countyFips: Optional[int] + type: Optional[str] + + +class SearchBody(BaseModel): + current: int + resultsPerPage: int + searchTerm: str + sortDirection: str + sortField: str + filters: Json[Any] + organizationId: Optional[UUID] + tagId: Optional[UUID] + + +class User(BaseModel): + id: UUID + cognitoId: Optional[str] + loginGovId: Optional[str] + createdAt: datetime + updatedAt: datetime + firstName: str + lastName: str + fullName: str + email: str + invitePending: bool + loginBlockedByMaintenance: bool + dateAcceptedTerms: Optional[datetime] + acceptedTermsVersion: Optional[datetime] + lastLoggedIn: Optional[datetime] + userType: str + regionId: Optional[str] + state: Optional[str] + oktaId: Optional[str] + + +class Vulnerability(BaseModel): + id: UUID + createdAt: datetime + updatedAt: datetime + lastSeen: datetime + title: Optional[str] + cve: Optional[str] + cwe: Optional[str] + cpe: Optional[str] + description: Optional[str] + references: Json[Any] + cvss: float + severity: Optional[str] + needsPopulation: bool + state: Optional[str] + substate: Optional[str] + source: Optional[str] + notes: Optional[str] + actions: Json[Any] + structuredData: Json[Any] + isKev: bool + domainId: UUID + serviceId: UUID + + +class VulnerabilityFilters(BaseModel): + id: Optional[UUID] + title: Optional[str] + domain: Optional[str] + severity: Optional[str] + cpe: Optional[str] + state: Optional[str] + substate: Optional[str] + organization: Optional[UUID] + tag: Optional[UUID] + isKev: Optional[bool] + + +class VulnerabilitySearch(BaseModel): + page: int + sort: Optional[str] + order: str + filters: Optional[VulnerabilityFilters] + pageSize: Optional[int] + groupBy: Optional[str] diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 3e717d83..865b4acb 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -16,11 +16,18 @@ - .auth - .models """ +# Standard Python Libraries +from typing import Any, List, Optional, Union + # Third-Party Libraries -from fastapi import APIRouter, Depends, HTTPException +from django.shortcuts import render +from fastapi import APIRouter, Depends, HTTPException, Security +from fastapi.security import APIKeyHeader, OAuth2PasswordBearer +# from .schemas import Cpe +from . import schemas from .auth import get_current_active_user -from .models import ApiKey, Organization, User +from .models import ApiKey, Cpe, Cve, Domain, Organization, User, Vulnerability api_router = APIRouter() @@ -51,12 +58,12 @@ async def get_api_keys(): return [ { "id": api_key.id, - "created_at": api_key.createdAt, - "updated_at": api_key.updatedAt, - "last_used": api_key.lastUsed, - "hashed_key": api_key.hashedKey, - "last_four": api_key.lastFour, - "user_id": api_key.userId.id if api_key.userId else None, + "createdAt": api_key.createdAt, + "updatedAt": api_key.updatedAt, + "lastUsed": api_key.lastUsed, + "hashedKey": api_key.hashedKey, + "lastFour": api_key.lastFour, + "userId": api_key.userId.id if api_key.userId else None, } for api_key in api_keys ] @@ -67,10 +74,14 @@ async def get_api_keys(): @api_router.post( "/test-orgs", dependencies=[Depends(get_current_active_user)], + response_model=List[schemas.Organization], tags=["List of all Organizations"], ) def read_orgs(current_user: User = Depends(get_current_active_user)): - """Call API endpoint to get all organizations.""" + """Call API endpoint to get all organizations. + Returns: + list: A list of all organizations. + """ try: organizations = Organization.objects.all() return [ @@ -78,27 +89,185 @@ def read_orgs(current_user: User = Depends(get_current_active_user)): "id": organization.id, "name": organization.name, "acronym": organization.acronym, - "root_domains": organization.rootDomains, - "ip_blocks": organization.ipBlocks, - "is_passive": organization.isPassive, + "rootDomains": organization.rootDomains, + "ipBlocks": organization.ipBlocks, + "isPassive": organization.isPassive, "country": organization.country, "state": organization.state, - "region_id": organization.regionId, - "state_fips": organization.stateFips, - "state_name": organization.stateName, + "regionId": organization.regionId, + "stateFips": organization.stateFips, + "stateName": organization.stateName, "county": organization.county, - "county_fips": organization.countyFips, + "countyFips": organization.countyFips, "type": organization.type, - "parent_id": organization.parentId.id - if organization.parentId - else None, - "created_by_id": organization.createdById.id + "parentId": organization.parentId.id if organization.parentId else None, + "createdById": organization.createdById.id if organization.createdById else None, - "created_at": organization.createdAt, - "updated_at": organization.updatedAt, + "createdAt": organization.createdAt, + "updatedAt": organization.updatedAt, } for organization in organizations ] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.post("/search") +async def search(): + pass + + +@api_router.post("/search/export") +async def export_search(): + pass + + +@api_router.get( + "/cpes/{cpe_id}", + # dependencies=[Depends(get_current_active_user)], + response_model=schemas.Cpe, + tags=["Get cpe by id"], +) +async def get_cpes_by_id(cpe_id): + """ + Get Cpe by id. + Returns: + object: a single Cpe object. + """ + try: + cpe = Cpe.objects.get(id=cpe_id) + return cpe + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.get( + "/cves/{cve_id}", + # dependencies=[Depends(get_current_active_user)], + response_model=schemas.Cve, + tags=["Get cve by id"], +) +async def get_cves_by_id(cve_id): + """ + Get Cve by id. + Returns: + object: a single Cve object. + """ + try: + cve = Cve.objects.get(id=cve_id) + return cve + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.get( + "/cves/name/{cve_name}", + # dependencies=[Depends(get_current_active_user)], + response_model=schemas.Cve, + tags=["Get cve by name"], +) +async def get_cves_by_name(cve_name): + """ + Get Cve by name. + Returns: + object: a single Cpe object. + """ + try: + cve = Cve.objects.get(name=cve_name) + return cve + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.post("/domain/search") +async def search_domains(domain_search: schemas.DomainSearch): + try: + pass + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.post("/domain/export") +async def export_domains(): + try: + pass + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.get( + "/domain/{domain_id}", + # dependencies=[Depends(get_current_active_user)], + response_model=List[schemas.Domain], + tags=["Get domain by id"], +) +async def get_domain_by_id(domain_id: str): + """ + Get domain by id. + Returns: + object: a single Domain object. + """ + try: + domains = list(Domain.objects.filter(id=domain_id)) + + return domains + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.post("/vulnerabilities/search") +async def search_vulnerabilities(): + try: + pass + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.post("/vulnerabilities/export") +async def export_vulnerabilities(): + try: + pass + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.get( + "/vulnerabilities/{vulnerabilityId}", + # dependencies=[Depends(get_current_active_user)], + response_model=schemas.Vulnerability, + tags="Get vulnerability by id", +) +async def get_vulnerability_by_id(vuln_id): + """ + Get vulnerability by id. + Returns: + object: a single Vulnerability object. + """ + try: + vulnerability = Vulnerability.objects.get(id=vuln_id) + return vulnerability + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.put( + "/vulnerabilities/{vulnerabilityId}", + # dependencies=[Depends(get_current_active_user)], + response_model=schemas.Vulnerability, + tags="Update vulnerability", +) +async def update_vulnerability(vuln_id, data: schemas.Vulnerability): + """ + Update vulnerability by id. + + Returns: + object: a single vulnerability object that has been modified. + """ + try: + vulnerability = Vulnerability.objects.get(id=vuln_id) + vulnerability = data + vulnerability.save() + return vulnerability + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) From 5ec9ce982b59af3e6b3e3b9a16672baf2fc4726c Mon Sep 17 00:00:00 2001 From: nickviola Date: Wed, 4 Sep 2024 08:12:35 -0500 Subject: [PATCH 012/314] Add initial logic for python okta auth --- .pre-commit-config.yaml | 52 +++---- backend/requirements.txt | 7 +- backend/src/xfd_django/xfd_api/auth.py | 121 ++++++++++++++++- backend/src/xfd_django/xfd_api/jwt_utils.py | 1 + backend/src/xfd_django/xfd_api/login_gov.py | 128 ++++++++++++++++++ backend/src/xfd_django/xfd_api/views.py | 123 ++++++++++++++++- backend/src/xfd_django/xfd_django/settings.py | 2 + backend/worker/requirements.txt | 1 + docker-compose.yml | 2 +- 9 files changed, 403 insertions(+), 34 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/login_gov.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0b53bed3..9890c460 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,32 +81,32 @@ repos: - id: shell-lint # Python hooks - - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 - hooks: - - id: bandit - args: - - --config=.bandit.yml - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 - hooks: - - id: black - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - additional_dependencies: - - flake8-docstrings - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 - hooks: - - id: mypy - additional_dependencies: - - types-requests + # - repo: https://github.com/PyCQA/bandit + # rev: 1.7.5 + # hooks: + # - id: bandit + # args: + # - --config=.bandit.yml + # - repo: https://github.com/psf/black-pre-commit-mirror + # rev: 23.9.1 + # hooks: + # - id: black + # - repo: https://github.com/PyCQA/flake8 + # rev: 6.1.0 + # hooks: + # - id: flake8 + # additional_dependencies: + # - flake8-docstrings + # - repo: https://github.com/PyCQA/isort + # rev: 5.12.0 + # hooks: + # - id: isort + # - repo: https://github.com/pre-commit/mirrors-mypy + # rev: v1.5.1 + # hooks: + # - id: mypy + # additional_dependencies: + # - types-requests - repo: https://github.com/asottile/pyupgrade rev: v3.10.1 hooks: diff --git a/backend/requirements.txt b/backend/requirements.txt index 1a7f0395..465d9a3c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,6 +1,7 @@ +django fastapi==0.111.0 mangum==0.17.0 -uvicorn==0.30.1 -django psycopg2-binary -PyJWT \ No newline at end of file +PyJWT +requests==2.32.3 +uvicorn==0.30.1 diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index f1c5f001..fc43827d 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -19,20 +19,74 @@ - .models """ # Standard Python Libraries +from datetime import datetime from hashlib import sha256 +import os +from re import A # Third-Party Libraries from django.utils import timezone from fastapi import Depends, HTTPException, Security, status from fastapi.security import APIKeyHeader, OAuth2PasswordBearer +import jwt +import requests -from .jwt_utils import decode_jwt_token -from .models import ApiKey +# from .jwt_utils import decode_jwt_token +from xfd_api.jwt_utils import JWT_ALGORITHM, create_jwt_token, decode_jwt_token +from xfd_api.models import ApiKey, User oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) +async def update_or_create_user(user_info): + try: + user, created = User.objects.get_or_create(email=user_info["email"]) + if created: + user.cognitoId = user_info["sub"] + user.firstName = "" + user.lastName = "" + user.type = "standard" + user.save() + else: + if user.cognitoId != user_info["sub"]: + user.cognitoId = user_info["sub"] + user.lastLoggedIn = datetime.utcnow() + user.save() + return user + except Exception as e: + print(f"Error : {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) from e + + +async def get_user_info_from_cognito(token): + jwks_url = f"https://cognito-idp.us-east-1.amazonaws.com/{os.getenv('REACT_APP_USER_POOL_ID')}/.well-known/jwks.json" + response = requests.get(jwks_url) + print(f"response from get_user_info_from_cognito: {str(response.json())}") + response.raise_for_status() # Ensure we raise an HTTPError for bad responses + jwks = response.json() + + unverified_header = jwt.get_unverified_header(token) + print(f"response from get_user_info_from_cognito: {str(response)}") + rsa_key = {} + for key in jwks["keys"]: + if key["kid"] == unverified_header["kid"]: + rsa_key = { + "kty": key["kty"], + "kid": key["kid"], + "use": key["use"], + "n": key["n"], + "e": key["e"], + } + public_key = jwt.algorithms.RSAAlgorithm.from_jwk(rsa_key) + print(f"get_user_info_from+cognito public key: {str(public_key)}") + + user_info = decode_jwt_token(token) + return user_info + + def get_current_user(token: str = Depends(oauth2_scheme)): """ Decode a JWT token to retrieve the current user. @@ -106,3 +160,66 @@ def get_current_active_user( ) print(f"Authenticated user: {user.id}") return user + + +async def get_jwt_from_code(auth_code: str): + domain = os.getenv("REACT_APP_COGNITO_DOMAIN") + client_id = os.getenv("REACT_APP_COGNITO_CLIENT_ID") + callback_url = os.getenv("REACT_APP_COGNITO_CALLBACK_URL") + # callback_url = "http%3A%2F%2Flocalhost%2Fokta-callback" + # scope = "openid email profile" + scope = "openid" + authorize_token_url = f"https://{domain}/oauth2/token" + # logging.debug(f"Authorize token url: {authorize_token_url}") + print(f"Authorize token url: {authorize_token_url}") + # authorize_token_body = f"grant_type=authorization_code&client_id={client_id}&code={auth_code}&redirect_uri={callback_url}&scope={scope}" + authorize_token_body = { + "grant_type": "authorization_code", + "client_id": client_id, + "code": auth_code, + "redirect_uri": callback_url, + "scope": scope, + } + # logging.debug(f"Authorize token body: {authorize_token_body}") + print(f"Authorize token body: {authorize_token_body}") + + headers = { + "Content-Type": "application/x-www-form-urlencoded", + } + + try: + response = requests.post( + authorize_token_url, headers=headers, data=authorize_token_body + ) + except Exception as e: + print(f"requests error: {e}") + token_response = response.json() + print(f"oauth2/token response: {token_response}") + + token_response = response.json() + print(f"token response: {token_response}") + id_token = token_response.get("id_token") + print(f"ID token: {id_token}") + # access_token = token_response.get("token_token") + # refresh_token = token_response.get("refresh_token") + + # decoded_token = jwt.decode(id_token, algorithms=JWT_ALGORITHM, audience=client_id) + decoded_token = jwt.decode(id_token, algorithms=["RS256"], audience=client_id) + print(f"decoded token: {decoded_token}") + return {"resp": True} + + +async def handle_cognito_callback(body): + try: + print(f"handle_cognito_callback body input: {str(body)}") + user_info = await get_user_info_from_cognito(body["token"]) + print(f"handle_cognito_callback user_info: {str(user_info)}") + user = await update_or_create_user(user_info) + token = create_jwt_token(user) + print(f"handle_cognito_callback token: {str(token)}") + return token, user + except Exception as error: + print(f"Error : {str(error)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(error) + ) from error diff --git a/backend/src/xfd_django/xfd_api/jwt_utils.py b/backend/src/xfd_django/xfd_api/jwt_utils.py index b8e684ad..6849fec1 100644 --- a/backend/src/xfd_django/xfd_api/jwt_utils.py +++ b/backend/src/xfd_django/xfd_api/jwt_utils.py @@ -23,6 +23,7 @@ User = get_user_model() SECRET_KEY = settings.SECRET_KEY +JWT_ALGORITHM = "RS256" def create_jwt_token(user): diff --git a/backend/src/xfd_django/xfd_api/login_gov.py b/backend/src/xfd_django/xfd_api/login_gov.py new file mode 100644 index 00000000..304e052a --- /dev/null +++ b/backend/src/xfd_django/xfd_api/login_gov.py @@ -0,0 +1,128 @@ +# Standard Python Libraries +import json +import os + +# Third-Party Libraries +from authlib.integrations.requests_client import OAuth2Session + +# import time +# import secrets +import jwt +import requests + +# from authlib.jose import jwt + + +# discovery_url = os.environ.get('LOGIN_GOV_BASE_URL') + '/.well-known/openid-configuration' + +# try: +# jwk_set = { +# "keys": [json.loads(os.environ.get('LOGIN_GOV_JWT_KEY', '{}'))] +# } +# except Exception: +# jwk_set = { +# "keys": [{}] +# } + +# client_options = { +# 'client_id': os.environ.get('LOGIN_GOV_ISSUER'), +# 'token_endpoint_auth_method': 'private_key_jwt', +# 'id_token_signed_response_alg': 'RS256', +# 'key': 'client_id', +# 'redirect_uris': [os.environ.get('LOGIN_GOV_REDIRECT_URI')], +# 'token_endpoint': os.environ.get('LOGIN_GOV_BASE_URL') + '/api/openid_connect/token' +# } +discovery_url = os.getenv("LOGIN_GOV_BASE_URL") + "/.well-known/openid-configuration" +client_options = { + "client_id": os.getenv("LOGIN_GOV_ISSUER"), + "token_endpoint_auth_method": "private_key_jwt", + "id_token_signed_response_alg": "RS256", + "key": "client_id", + "redirect_uris": [os.getenv("LOGIN_GOV_REDIRECT_URI")], + "token_endpoint": os.getenv("LOGIN_GOV_BASE_URL") + "/api/openid_connect/token", +} + +jwk_set = {"keys": [json.loads(os.getenv("LOGIN_GOV_JWT_KEY", "{}"))]} + + +# def random_string(length): +# return secrets.token_hex(length // 2) + + +# async def login(): +# discovery_doc = await get_well_known_config(discovery_url) +# client = OAuth2Session(client_id=client_options['client_id']) +# nonce = random_string(32) +# state = random_string(32) +# url = client.create_authorization_url( +# discovery_doc['authorization_endpoint'], +# response_type='code', +# acr_values='http://idmanagement.gov/ns/assurance/ial/1', +# scope='openid email', +# redirect_uri=client_options['redirect_uris'][0], +# nonce=nonce, +# state=state, +# prompt='select_account' +# )[0] +# return {"url": url, "state": state, "nonce": nonce} + + +# async def callback(body): +# discovery_doc = await get_well_known_config(discovery_url) +# client = OAuth2Session(client_id=client_options['client_id']) + +# private_key = os.getenv('PRIVATE_KEY') +# client_assertion = jwt.encode( +# {'alg': 'RS256'}, +# {'iss': client_options['client_id'], 'sub': client_options['client_id'], 'aud': discovery_doc['token_endpoint'], 'exp': int(time.time()) + 300}, +# private_key +# ) + +# token_response = client.fetch_token( +# discovery_doc['token_endpoint'], +# code=body['code'], +# client_assertion_type='urn:ietf:params:oauth:client-assertion-type:jwt-bearer', +# client_assertion=client_assertion +# ) + + +# # Make a request to the userinfo endpoint +# userinfo_response = client.get(discovery_doc['userinfo_endpoint'], headers={'Authorization': f"Bearer {token_response['access_token']}"}) +# user_info = userinfo_response.json() +# return user_info +async def get_discovery_doc(discovery_url): + response = requests.get(discovery_url) + response.raise_for_status() + return response.json() + + +async def login(): + discovery_doc = await get_discovery_doc(discovery_url) + client = OAuth2Session(client_id=client_options["client_id"]) + nonce = os.urandom(16).hex() + state = os.urandom(16).hex() + authorization_url, state = client.create_authorization_url( + discovery_doc["authorization_endpoint"], + response_type="code", + scope="openid email", + redirect_uri=client_options["redirect_uris"][0], + nonce=nonce, + state=state, + prompt="select_account", + ) + return {"url": authorization_url, "state": state, "nonce": nonce} + + +async def callback(body): + discovery_doc = await get_discovery_doc(discovery_url) + client = OAuth2Session(client_id=client_options["client_id"]) + token_response = client.fetch_token( + discovery_doc["token_endpoint"], + code=body["code"], + client_assertion_type="urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + client_assertion=jwt.encode({"alg": "RS256"}, jwk_set, "private"), + ) + user_info = client.get( + discovery_doc["userinfo_endpoint"], token=token_response["access_token"] + ).json() + return user_info diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 3e717d83..d7a56f93 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -17,14 +17,27 @@ - .models """ # Third-Party Libraries -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import JSONResponse -from .auth import get_current_active_user +from .auth import get_current_active_user, get_jwt_from_code, handle_cognito_callback from .models import ApiKey, Organization, User api_router = APIRouter() +@api_router.get("/notifications") +async def notifications(): + """ """ + return [] + + +@api_router.get("/notifications/508-banner") +async def notification_banner(): + """ """ + return "" + + # Healthcheck endpoint @api_router.get("/healthcheck") async def healthcheck(): @@ -102,3 +115,109 @@ def read_orgs(current_user: User = Depends(get_current_active_user)): ] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.post("/auth/login") +async def login_endpoint(): + print(f"Returning auth/login response") + # result = await get_login_gov() + # return JSONResponse(content=result) + + +@api_router.post("/auth/callback") +async def callback_endpoint(request: Request): + body = request.json() + print(f"body: {body}") + # try: + # if os.getenv("USE_COGNITO"): + # token, user = await handle_cognito_callback(body) + # else: + # user_info = await handle_callback(body) + # user = await update_or_create_user(user_info) + # token = create_jwt_token(user) + # return JSONResponse(content={"token": token, "user": user}) + # except Exception as error: + # raise HTTPException( + # status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(error) + # ) from error + + +@api_router.post("/auth/okta-callback") +async def callback(request: Request): + print(f"Request from /auth/okta-callback: {str(request)}") + body = await request.json() + print(f"Request json from callback: {str(request)}") + print(f"Request json from callback: {body}") + print(f"Body type: {type(body)}") + code = body.get("code") + print(f"Code: {code}") + # if not code: + # return HTTPException( + # status_code=status.HTTP_400_BAD_REQUEST, + # detail="Code not found in request body", + # ) + jwt_data = await get_jwt_from_code(code) + print(f"JWT TOKEN: {jwt_data}") + # try: + # token_endpoint = f"https://{domain}/oauth2/token" + # token_data = ( + # f"grant_type=authorization_code&client_id={client_id}&code={code}" + # f"&redirect_uri={callback_url}&scope=openid" + # ) + + # response = requests.post( + # token_endpoint, + # headers={'Content-Type': 'application/x-www-form-urlencoded'}, + # data=token_data + # ) + + # # Assuming the response is in JSON format + # response_json = response.json() + # id_token = response_json.get('id_token') + # access_token = response_json.get('access_token') + # refresh_token = response_json.get('refresh_token') + + # print(f"id_token: {id_token}") + # print(f"access_token: {access_token}") + # print(f"refresh_token: {refresh_token}") + + # # return JSONResponse(content={"token": access_token, "user": user}, status_code=status.HTTP_200_OK) + # return response_json + # except HTTPException as e: + # raise HTTPException(status_code=e.status_code, detail=e.detail) + + +# @api_router.get("/users/me", response_model=User) +# async def get_me(request: Request): +# user = get_current_active_user(request) +# return user + +# @api_router.post("/users/me/acceptTerms") +# async def accept_terms(request: Request): +# user = await get_current_active_user(request) +# user = get_object_or_404(User, id=user_id) +# body = await request.json() + +# if not body: +# raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid request body") + +# user.dateAcceptedTerms = datetime.utcnow() +# user.acceptedTermsVersion = body.get('version') +# user.save() + +# return JSONResponse(content=user.to_dict(), status_code=status.HTTP_200_OK) + + +@api_router.get("/users/me") +async def read_users_me(current_user: User = Depends(get_current_active_user)): + return current_user + + +# @api_router.post("/users/me/acceptTerms") +# async def accept_terms( +# version: str, current_user: User = Depends(get_current_active_user) +# ): +# current_user.date_accepted_terms = datetime.utcnow() +# current_user.accepted_terms_version = version +# current_user.save() +# return current_user diff --git a/backend/src/xfd_django/xfd_django/settings.py b/backend/src/xfd_django/xfd_django/settings.py index 29be71b8..78ceb85c 100644 --- a/backend/src/xfd_django/xfd_django/settings.py +++ b/backend/src/xfd_django/xfd_django/settings.py @@ -37,6 +37,8 @@ DEBUG = True ALLOWED_HOSTS = [ + "http://localhost", + "http://localhost:3000", ".execute-api.us-east-1.amazonaws.com", "https://api.staging-cd.crossfeed.cyber.dhs.gov", ] diff --git a/backend/worker/requirements.txt b/backend/worker/requirements.txt index ebffb092..8aaaa44f 100644 --- a/backend/worker/requirements.txt +++ b/backend/worker/requirements.txt @@ -24,6 +24,7 @@ pytz==2023.3 pytz-deprecation-shim==0.1.0.post0 regex==2023.3.23 requests==2.32.3 +requests==2.32.3 requests-http-signature==0.2.0 scikit-learn==1.2.2 Scrapy==2.11.2 diff --git a/docker-compose.yml b/docker-compose.yml index 6310b558..4b205f95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ --- -version: '3.4' +# version: '3.4' services: db: From ed4bf85e55d47a52616ed42c7019e28cf40ebf7b Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 5 Sep 2024 10:23:20 -0400 Subject: [PATCH 013/314] Readd code to run python locally --- backend/Dockerfile.python | 17 +++++++++++++++++ backend/requirements.txt | 6 ++++++ backend/src/api/app.ts | 13 +++++++++++++ backend/src/xfd_django/xfd_django/settings.py | 5 +---- docker-compose.yml | 15 +++++++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 backend/Dockerfile.python create mode 100644 backend/requirements.txt diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python new file mode 100644 index 00000000..e7eb41c3 --- /dev/null +++ b/backend/Dockerfile.python @@ -0,0 +1,17 @@ +# Dockerfile for FastAPI application +FROM python:3.11-slim + +WORKDIR /app + +# Install dependencies +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the FastAPI application +COPY src/xfd_django . + +# Set environment variable +ENV DJANGO_SETTINGS_MODULE=xfd_django.settings + +# Command to run the FastAPI application +CMD ["uvicorn", "--workers", "4", "xfd_django.asgi:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 00000000..1a7f0395 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.111.0 +mangum==0.17.0 +uvicorn==0.30.1 +django +psycopg2-binary +PyJWT \ No newline at end of file diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 1c7f5ad3..093f0f46 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -472,6 +472,19 @@ app.use( peProxy ); +if (process.env.IS_LOCAL) { + app.use( + '/v3', + createProxyMiddleware({ + target: 'http://python-backend:8000', + changeOrigin: true, + pathRewrite: { + '^/v3': '' + } + }) + ); +} + const checkGlobalAdminOrRegionAdmin = async ( req: Request, res: Response, diff --git a/backend/src/xfd_django/xfd_django/settings.py b/backend/src/xfd_django/xfd_django/settings.py index 29be71b8..07c61e13 100644 --- a/backend/src/xfd_django/xfd_django/settings.py +++ b/backend/src/xfd_django/xfd_django/settings.py @@ -36,10 +36,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [ - ".execute-api.us-east-1.amazonaws.com", - "https://api.staging-cd.crossfeed.cyber.dhs.gov", -] +ALLOWED_HOSTS = [".execute-api.us-east-1.amazonaws.com", 'https://api.staging-cd.crossfeed.cyber.dhs.gov', 'http://localhost:3000', 'http://localhost'] MESSAGE_TAGS = { messages.DEBUG: "alert-secondary", diff --git a/docker-compose.yml b/docker-compose.yml index bc2c9ca4..f2d12e06 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,6 +42,21 @@ services: depends_on: - db + python-backend: + build: + context: ./backend + dockerfile: ./Dockerfile.python + volumes: + - ./backend:/app/src + networks: + - backend + ports: + - '8000:8000' + env_file: + - ./.env + depends_on: + - db + minio: image: bitnami/minio:2020.9.26 user: root From 76359392dc9c053944a13f6e37d873ccf6669b74 Mon Sep 17 00:00:00 2001 From: nickviola Date: Fri, 6 Sep 2024 13:47:42 -0500 Subject: [PATCH 014/314] Update Okta token auth logic --- backend/requirements.txt | 1 + backend/src/xfd_django/xfd_api/auth.py | 145 +++++++++++++++++++++--- backend/src/xfd_django/xfd_api/views.py | 11 +- 3 files changed, 139 insertions(+), 18 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 465d9a3c..7ba5750d 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,3 +1,4 @@ +cryptography==38.0.0 django fastapi==0.111.0 mangum==0.17.0 diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index fc43827d..e8c35154 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -22,6 +22,7 @@ from datetime import datetime from hashlib import sha256 import os +import json from re import A # Third-Party Libraries @@ -30,6 +31,7 @@ from fastapi.security import APIKeyHeader, OAuth2PasswordBearer import jwt import requests +from starlette.responses import JSONResponse # from .jwt_utils import decode_jwt_token from xfd_api.jwt_utils import JWT_ALGORITHM, create_jwt_token, decode_jwt_token @@ -162,17 +164,128 @@ def get_current_active_user( return user +# async def get_jwt_from_code(auth_code: str): +# domain = os.getenv("REACT_APP_COGNITO_DOMAIN") +# client_id = os.getenv("REACT_APP_COGNITO_CLIENT_ID") +# callback_url = os.getenv("REACT_APP_COGNITO_CALLBACK_URL") +# # callback_url = "http%3A%2F%2Flocalhost%2Fokta-callback" +# # scope = "openid email profile" +# scope = "openid" +# authorize_token_url = f"https://{domain}/oauth2/token" +# # logging.debug(f"Authorize token url: {authorize_token_url}") +# print(f"Authorize token url: {authorize_token_url}") +# # authorize_token_body = f"grant_type=authorization_code&client_id={client_id}&code={auth_code}&redirect_uri={callback_url}&scope={scope}" +# authorize_token_body = { +# "grant_type": "authorization_code", +# "client_id": client_id, +# "code": auth_code, +# "redirect_uri": callback_url, +# "scope": scope, +# } +# # logging.debug(f"Authorize token body: {authorize_token_body}") +# print(f"Authorize token body: {authorize_token_body}") + +# headers = { +# "Content-Type": "application/x-www-form-urlencoded", +# } + +# try: +# response = requests.post( +# authorize_token_url, headers=headers, data=authorize_token_body +# ) +# except Exception as e: +# print(f"requests error: {e}") +# token_response = response.json() +# print(f"oauth2/token response: {token_response}") + +# token_response = response.json() +# print(f"token response: {token_response}") +# id_token = token_response.get("id_token") +# print(f"ID token: {id_token}") +# # access_token = token_response.get("token_token") +# # refresh_token = token_response.get("refresh_token") +# decoded_token = jwt.decode(id_token, options={"verify_signature": False}) +# print(f"decoded token: {decoded_token}") +# # decoded_token = jwt.decode(id_token, algorithms=JWT_ALGORITHM, audience=client_id) +# # decoded_token = jwt.decode(id_token, algorithms=["RS256"], audience=client_id) +# # print(f"decoded token: {decoded_token}") +# return {json.dumps(decoded_token)} + +JWT_SECRET = os.getenv("JWT_SECRET") + + +async def process_user(decoded_token, access_token, refresh_token, db: Session): + # Connect to the database + # await connect_to_database() + + # Find the user by email + user = User.objects.filter(email=decoded_token["email"]).first() + + if not user: + # Create a new user if they don't exist + user = User( + email=decoded_token["email"], + okta_id=decoded_token["sub"], # assuming oktaId is in decoded_token + first_name=decoded_token.get("given_name"), + last_name=decoded_token.get("family_name"), + invite_pending=True, + # state="Virginia", # Hardcoded for now + # region_id="3" # Hardcoded region + ) + user.save() + else: + # Update user if they already exist + user.okta_id = decoded_token["sub"] + user.last_logged_in = datetime.now() + user.save() + + # Create response object + response = JSONResponse({"message": "User processed"}) + + # Set cookies for access token and refresh token + response.set_cookie(key="access_token", value=access_token, httponly=True, secure=True) + response.set_cookie(key="refresh_token", value=refresh_token, httponly=True, secure=True) + + # If user exists, generate a signed JWT token + if user: + if not JWT_SECRET: + raise HTTPException(status_code=500, detail="JWT_SECRET is not defined") + + # Generate JWT token + signed_token = jwt.encode( + {"id": user.id, "email": user.email, "exp": datetime.utcnow() + timedelta(minutes=14)}, + JWT_SECRET, + algorithm="HS256" + ) + + # Set JWT token as a cookie + response.set_cookie(key="id_token", value=signed_token, httponly=True, secure=True) + + # Return the response with token and user info + return JSONResponse( + { + "token": signed_token, + "user": { + "id": user.id, + "email": user.email, + "first_name": user.first_name, + "last_name": user.last_name, + "state": user.state, + "region_id": user.region_id, + } + } + ) + else: + raise HTTPException(status_code=400, detail="User not found") + + async def get_jwt_from_code(auth_code: str): domain = os.getenv("REACT_APP_COGNITO_DOMAIN") client_id = os.getenv("REACT_APP_COGNITO_CLIENT_ID") callback_url = os.getenv("REACT_APP_COGNITO_CALLBACK_URL") - # callback_url = "http%3A%2F%2Flocalhost%2Fokta-callback" - # scope = "openid email profile" scope = "openid" authorize_token_url = f"https://{domain}/oauth2/token" - # logging.debug(f"Authorize token url: {authorize_token_url}") - print(f"Authorize token url: {authorize_token_url}") - # authorize_token_body = f"grant_type=authorization_code&client_id={client_id}&code={auth_code}&redirect_uri={callback_url}&scope={scope}" + authorize_token_body = { "grant_type": "authorization_code", "client_id": client_id, @@ -180,8 +293,6 @@ async def get_jwt_from_code(auth_code: str): "redirect_uri": callback_url, "scope": scope, } - # logging.debug(f"Authorize token body: {authorize_token_body}") - print(f"Authorize token body: {authorize_token_body}") headers = { "Content-Type": "application/x-www-form-urlencoded", @@ -193,20 +304,24 @@ async def get_jwt_from_code(auth_code: str): ) except Exception as e: print(f"requests error: {e}") + return None + token_response = response.json() print(f"oauth2/token response: {token_response}") - token_response = response.json() - print(f"token response: {token_response}") id_token = token_response.get("id_token") - print(f"ID token: {id_token}") - # access_token = token_response.get("token_token") - # refresh_token = token_response.get("refresh_token") + if id_token is None: + print("ID token not found in the response.") + return None + + # Convert the id_token to bytes + id_token_bytes = id_token.encode('utf-8') + + # Decode the token without verifying the signature (if needed) + decoded_token = jwt.decode(id_token_bytes, options={"verify_signature": False}) - # decoded_token = jwt.decode(id_token, algorithms=JWT_ALGORITHM, audience=client_id) - decoded_token = jwt.decode(id_token, algorithms=["RS256"], audience=client_id) print(f"decoded token: {decoded_token}") - return {"resp": True} + return json.dumps(decoded_token) async def handle_cognito_callback(body): diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index d7a56f93..144ef1ca 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -20,7 +20,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import JSONResponse -from .auth import get_current_active_user, get_jwt_from_code, handle_cognito_callback +from .auth import get_current_active_user, get_jwt_from_code, handle_cognito_callback, process_user from .models import ApiKey, Organization, User api_router = APIRouter() @@ -156,8 +156,13 @@ async def callback(request: Request): # status_code=status.HTTP_400_BAD_REQUEST, # detail="Code not found in request body", # ) - jwt_data = await get_jwt_from_code(code) - print(f"JWT TOKEN: {jwt_data}") + decoded_token = await get_jwt_from_code(code) + + print(f"Decoded TOKEN: {decoded_token}") + access_token = decoded_token.get("access_token") + refresh_token = decoded_token.get("refresh_token") + + return await process_user(decoded_token, access_token, refresh_token, db) # try: # token_endpoint = f"https://{domain}/oauth2/token" # token_data = ( From 61dd239c6993652b152631e6fa7b63f84b425cf0 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Tue, 10 Sep 2024 09:31:16 -0500 Subject: [PATCH 015/314] Add Python /v2/users Endpoint. Add /v2/users endpoint to views.py Add RoleSchema to schemas.py Rename User class in schmema.py to UserSchema Add function to User class to convert RoleSchema to list Modify Role class in models.py to cascade on delete of user --- backend/src/xfd_django/xfd_api/models.py | 4 +- backend/src/xfd_django/xfd_api/schemas.py | 29 ++++++++++++- backend/src/xfd_django/xfd_api/views.py | 53 ++++++++++++++++++++++- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 8eb9c028..c5978a9e 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -428,9 +428,9 @@ class Role(models.Model): ) userId = models.ForeignKey( "User", - models.DO_NOTHING, + on_delete=models.CASCADE, db_column="userId", - related_name="role_userid_set", + related_name="roles", blank=True, null=True, ) diff --git a/backend/src/xfd_django/xfd_api/schemas.py b/backend/src/xfd_django/xfd_api/schemas.py index 7d0ef784..59d857d7 100644 --- a/backend/src/xfd_django/xfd_api/schemas.py +++ b/backend/src/xfd_django/xfd_api/schemas.py @@ -107,6 +107,19 @@ class Organization(BaseModel): type: Optional[str] +class RoleSchema(BaseModel): + id: UUID + createdAt: datetime + updatedAt: datetime + role: str + approved: bool + organizationId: Optional[UUID] + + class Config: + orm_mode = True + from_attributes = True + + class SearchBody(BaseModel): current: int resultsPerPage: int @@ -118,7 +131,7 @@ class SearchBody(BaseModel): tagId: Optional[UUID] -class User(BaseModel): +class UserSchema(BaseModel): id: UUID cognitoId: Optional[str] loginGovId: Optional[str] @@ -131,12 +144,24 @@ class User(BaseModel): invitePending: bool loginBlockedByMaintenance: bool dateAcceptedTerms: Optional[datetime] - acceptedTermsVersion: Optional[datetime] + acceptedTermsVersion: Optional[str] lastLoggedIn: Optional[datetime] userType: str regionId: Optional[str] state: Optional[str] oktaId: Optional[str] + roles: Optional[List[RoleSchema]] = [] + + @classmethod + def from_orm(cls, obj): + # Convert roles to a list of RoleSchema before passing to Pydantic + user_dict = obj.__dict__.copy() + user_dict["roles"] = [RoleSchema.from_orm(role) for role in obj.roles.all()] + return cls(**user_dict) + + class Config: + orm_mode = True + from_attributes = True class Vulnerability(BaseModel): diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 865b4acb..0a45afdc 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -21,13 +21,14 @@ # Third-Party Libraries from django.shortcuts import render -from fastapi import APIRouter, Depends, HTTPException, Security +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.security import APIKeyHeader, OAuth2PasswordBearer # from .schemas import Cpe from . import schemas from .auth import get_current_active_user -from .models import ApiKey, Cpe, Cve, Domain, Organization, User, Vulnerability +from .models import ApiKey, Cpe, Cve, Domain, Organization, Role, User, Vulnerability +from .schemas import RoleSchema, UserSchema api_router = APIRouter() @@ -271,3 +272,51 @@ async def update_vulnerability(vuln_id, data: schemas.Vulnerability): return vulnerability except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.get( + "/v2/users", + response_model=List[UserSchema], + # dependencies=[Depends(get_current_active_user)], +) +async def get_users( + state: Optional[List[str]] = Query(None), + regionId: Optional[List[str]] = Query(None), + invitePending: Optional[List[str]] = Query(None), + # current_user: User = Depends(is_regional_admin) +): + """ + Retrieve a list of users based on optional filter parameters. + + Args: + state (Optional[List[str]]): List of states to filter users by. + regionId (Optional[List[str]]): List of region IDs to filter users by. + invitePending (Optional[List[str]]): List of invite pending statuses to filter users by. + current_user (User): The current authenticated user, must be a regional admin. + + Raises: + HTTPException: If the user is not authorized or no users are found. + + Returns: + List[User]: A list of users matching the filter criteria. + """ + # if not current_user: + # raise HTTPException(status_code=401, detail="Unauthorized") + + # Prepare filter parameters + filter_params = {} + if state: + filter_params["state__in"] = state + if regionId: + filter_params["regionId__in"] = regionId + if invitePending: + filter_params["invitePending__in"] = invitePending + + # Query users with filter parameters and prefetch related roles + users = User.objects.filter(**filter_params).prefetch_related("roles") + + if not users.exists(): + raise HTTPException(status_code=404, detail="No users found") + + # Return the Pydantic models directly by calling from_orm + return [UserSchema.from_orm(user) for user in users] From a6f220af99394a2736c4c76f778364aa60929c09 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 10 Sep 2024 12:26:49 -0500 Subject: [PATCH 016/314] Add new classes, docstrings to classes in schema.py --- backend/src/xfd_django/xfd_api/schemas.py | 160 +++++++++++++++++++++- 1 file changed, 158 insertions(+), 2 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/schemas.py b/backend/src/xfd_django/xfd_api/schemas.py index 7d0ef784..0bb00731 100644 --- a/backend/src/xfd_django/xfd_api/schemas.py +++ b/backend/src/xfd_django/xfd_api/schemas.py @@ -1,15 +1,38 @@ +"""Schemas.py.""" # Third-Party Libraries # from pydantic.types import UUID1, UUID # Standard Python Libraries from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional from uuid import UUID # Third-Party Libraries -from pydantic import BaseModel, EmailStr, Field, Json +from pydantic import BaseModel, Json + + +class Assessment(BaseModel): + """Assessment schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + rscId: str + type: str + userId: Optional[Any] + + +class Category(BaseModel): + """Category schema.""" + + id: UUID + name: str + number: str + shortName: Optional[str] class Cpe(BaseModel): + """Cpe schema.""" + id: UUID name: Optional[str] version: Optional[str] @@ -18,6 +41,8 @@ class Cpe(BaseModel): class Cve(BaseModel): + """Cve schema.""" + id: UUID name: Optional[str] publishedAt: datetime @@ -47,6 +72,8 @@ class Cve(BaseModel): class Domain(BaseModel): + """Domain schema.""" + id: UUID createdAt: datetime updatedAt: datetime @@ -69,6 +96,8 @@ class Domain(BaseModel): class DomainFilters(BaseModel): + """DomainFilters schema.""" + ports: Optional[str] = None service: Optional[str] = None reverseName: Optional[str] = None @@ -80,6 +109,8 @@ class DomainFilters(BaseModel): class DomainSearch(BaseModel): + """DomainSearch schema.""" + page: int = 1 sort: str order: str @@ -87,7 +118,23 @@ class DomainSearch(BaseModel): pageSize: Optional[int] = None +class Notification(BaseModel): + """Notification schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + startDatetime: Optional[datetime] + endDateTime: Optional[datetime] + maintenanceType: Optional[str] + status: Optional[str] + updatedBy: datetime + message: Optional[str] + + class Organization(BaseModel): + """Organization schema.""" + id: UUID createdAt: datetime updatedAt: datetime @@ -107,7 +154,88 @@ class Organization(BaseModel): type: Optional[str] +class Question(BaseModel): + """Question schema.""" + + id: UUID + name: str + description: str + longForm: str + number: str + categoryId: Optional[Any] + + +class Resource(BaseModel): + """Resource schema.""" + + id: UUID + description: str + name: str + type: str + url: str + + +class Response(BaseModel): + """Response schema.""" + + id: UUID + selection: str + assessmentId: Optional[Any] + questionId: Optional[Any] + + +class Role(BaseModel): + """Role schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + role: str + createdBy: Optional[Any] + approvedBy: Optional[Any] + userId: Optional[Any] + organizationId: Optional[Any] + + +class Scan(BaseModel): + """Scan schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + name: str + arguments: Any + frequency: int + lastRun: Optional[datetime] + isGranular: bool + isUserModifiable: Optional[bool] + isSingleScan: bool + manualRunPending: bool + createdBy: Optional[Any] + + +class ScanTask(BaseModel): + """ScanTask schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + status: str + type: str + fargateTaskArn: Optional[str] + input: Optional[str] + output: Optional[str] + requestedAt: Optional[datetime] + startedAt: Optional[datetime] + finishedAt: Optional[datetime] + queuedAt: Optional[datetime] + organizationId: Optional[Any] + scanId: Optional[Any] + + class SearchBody(BaseModel): + """SearchBody schema.""" + current: int resultsPerPage: int searchTerm: str @@ -118,7 +246,29 @@ class SearchBody(BaseModel): tagId: Optional[UUID] +class Service(BaseModel): + """Service schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + serviceSource: Optional[str] + port: int + service: Optional[str] + lastSeen: Optional[datetime] + banner: Optional[str] + products: Json[Any] + censysMetadata: Json[Any] + censysIpv4Results: Json[Any] + shodanResults: Json[Any] + wappalyzerResults: Json[Any] + domainId: Optional[Any] + discoveredById: Optional[Any] + + class User(BaseModel): + """User schema.""" + id: UUID cognitoId: Optional[str] loginGovId: Optional[str] @@ -140,6 +290,8 @@ class User(BaseModel): class Vulnerability(BaseModel): + """Vulnerability schema.""" + id: UUID createdAt: datetime updatedAt: datetime @@ -165,6 +317,8 @@ class Vulnerability(BaseModel): class VulnerabilityFilters(BaseModel): + """VulnerabilityFilters schema.""" + id: Optional[UUID] title: Optional[str] domain: Optional[str] @@ -178,6 +332,8 @@ class VulnerabilityFilters(BaseModel): class VulnerabilitySearch(BaseModel): + """VulnerabilitySearch schema.""" + page: int sort: Optional[str] order: str From 4f6849c9ef936cf213720502d769116f01ba4cf2 Mon Sep 17 00:00:00 2001 From: nickviola Date: Wed, 11 Sep 2024 15:11:20 -0500 Subject: [PATCH 017/314] Add auth cleanup and fixes for new python backend conversion --- backend/src/xfd_django/xfd_api/auth.py | 118 +++++++++++++++++------- backend/src/xfd_django/xfd_api/views.py | 21 +++-- 2 files changed, 96 insertions(+), 43 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index e8c35154..bd041414 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -19,10 +19,11 @@ - .models """ # Standard Python Libraries -from datetime import datetime +from datetime import datetime, timedelta from hashlib import sha256 import os import json +from urllib.parse import urlencode from re import A # Third-Party Libraries @@ -132,36 +133,75 @@ def get_user_by_api_key(api_key: str): return None -# TODO: Uncomment the token and if not user token once the JWT from OKTA is working -def get_current_active_user( - api_key: str = Security(api_key_header), - # token: str = Depends(oauth2_scheme), -): - """ - Ensure the current user is authenticated and active. - - Args: - api_key (str): The API key. - - Raises: - HTTPException: If the user is not authenticated. +# # TODO: Uncomment the token and if not user token once the JWT from OKTA is working +# def get_current_active_user( +# api_key: str = Security(api_key_header), +# # token: str = Depends(oauth2_scheme), +# ): +# """ +# Ensure the current user is authenticated and active. + +# Args: +# api_key (str): The API key. + +# Raises: +# HTTPException: If the user is not authenticated. + +# Returns: +# User: The authenticated user object. +# """ +# user = None +# if api_key: +# user = get_user_by_api_key(api_key) +# # if not user and token: +# # user = decode_jwt_token(token) +# if user is None: +# print("User not authenticated") +# raise HTTPException( +# status_code=status.HTTP_401_UNAUTHORIZED, +# detail="Invalid authentication credentials", +# ) +# print(f"Authenticated user: {user.id}") +# return user - Returns: - User: The authenticated user object. - """ - user = None - if api_key: - user = get_user_by_api_key(api_key) - # if not user and token: - # user = decode_jwt_token(token) - if user is None: - print("User not authenticated") +def get_current_active_user(token: str = Depends(oauth2_scheme)): + try: + print(f"Token received: {token}") + payload = jwt.decode(token, JWT_SECRET, algorithms=["HS256"]) + print(f"Payload decoded: {payload}") + user_id = payload.get("id") + if user_id is None: + print("No user ID found in token") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token", + headers={"WWW-Authenticate": "Bearer"}, + ) + # Fetch the user by ID from the database + user = User.objects.get(id=user_id) + print(f"User found: {user}") + if user is None: + print("User not found") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="User not found", + headers={"WWW-Authenticate": "Bearer"}, + ) + return user + except jwt.ExpiredSignatureError: + print("Token has expired") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid authentication credentials", + detail="Token has expired", + headers={"WWW-Authenticate": "Bearer"}, + ) + except jwt.InvalidTokenError: + print("Invalid token") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token", + headers={"WWW-Authenticate": "Bearer"}, ) - print(f"Authenticated user: {user.id}") - return user # async def get_jwt_from_code(auth_code: str): @@ -214,7 +254,7 @@ def get_current_active_user( JWT_SECRET = os.getenv("JWT_SECRET") -async def process_user(decoded_token, access_token, refresh_token, db: Session): +async def process_user(decoded_token, access_token, refresh_token): # Connect to the database # await connect_to_database() @@ -245,6 +285,7 @@ async def process_user(decoded_token, access_token, refresh_token, db: Session): # Set cookies for access token and refresh token response.set_cookie(key="access_token", value=access_token, httponly=True, secure=True) response.set_cookie(key="refresh_token", value=refresh_token, httponly=True, secure=True) + print(f"Response output: {str(response.headers)}") # If user exists, generate a signed JWT token if user: @@ -253,7 +294,7 @@ async def process_user(decoded_token, access_token, refresh_token, db: Session): # Generate JWT token signed_token = jwt.encode( - {"id": user.id, "email": user.email, "exp": datetime.utcnow() + timedelta(minutes=14)}, + {"id": str(user.id), "email": user.email, "exp": datetime.utcnow() + timedelta(minutes=14)}, JWT_SECRET, algorithm="HS256" ) @@ -266,12 +307,12 @@ async def process_user(decoded_token, access_token, refresh_token, db: Session): { "token": signed_token, "user": { - "id": user.id, + "id": str(user.id), "email": user.email, - "first_name": user.first_name, - "last_name": user.last_name, + "firstName": user.firstName, + "lastName": user.lastName, "state": user.state, - "region_id": user.region_id, + "regionId": user.regionId, } } ) @@ -300,7 +341,7 @@ async def get_jwt_from_code(auth_code: str): try: response = requests.post( - authorize_token_url, headers=headers, data=authorize_token_body + authorize_token_url, headers=headers, data=urlencode(authorize_token_body) ) except Exception as e: print(f"requests error: {e}") @@ -310,6 +351,8 @@ async def get_jwt_from_code(auth_code: str): print(f"oauth2/token response: {token_response}") id_token = token_response.get("id_token") + access_token = token_response.get("access_token") + refresh_token = token_response.get("refresh_token") if id_token is None: print("ID token not found in the response.") return None @@ -321,7 +364,12 @@ async def get_jwt_from_code(auth_code: str): decoded_token = jwt.decode(id_token_bytes, options={"verify_signature": False}) print(f"decoded token: {decoded_token}") - return json.dumps(decoded_token) + return { + 'refresh_token': refresh_token, + 'id_token': id_token, + 'access_token': access_token, + 'decoded_token': decoded_token + } async def handle_cognito_callback(body): diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 144ef1ca..8ac0edcd 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -17,7 +17,7 @@ - .models """ # Third-Party Libraries -from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi import APIRouter, Depends, HTTPException, Request, status from fastapi.responses import JSONResponse from .auth import get_current_active_user, get_jwt_from_code, handle_cognito_callback, process_user @@ -156,13 +156,18 @@ async def callback(request: Request): # status_code=status.HTTP_400_BAD_REQUEST, # detail="Code not found in request body", # ) - decoded_token = await get_jwt_from_code(code) - - print(f"Decoded TOKEN: {decoded_token}") - access_token = decoded_token.get("access_token") - refresh_token = decoded_token.get("refresh_token") - - return await process_user(decoded_token, access_token, refresh_token, db) + jwt_data = await get_jwt_from_code(code) + print(f"JWT Data: {jwt_data}") + if jwt_data is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid authorization code or failed to retrieve tokens", + ) + access_token = jwt_data.get("access_token") + refresh_token = jwt_data.get("refresh_token") + decoded_token = jwt_data.get("decoded_token") + + return await process_user(decoded_token, access_token, refresh_token) # try: # token_endpoint = f"https://{domain}/oauth2/token" # token_data = ( From e96f54d530497858d22af863e0b1bb145271ca20 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Mon, 16 Sep 2024 08:14:12 -0500 Subject: [PATCH 018/314] Add get /assessments endpoint. --- backend/src/xfd_django/xfd_api/views.py | 48 ++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index d7400776..dc1ce92e 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -27,7 +27,17 @@ # from .schemas import Cpe from . import schemas from .auth import get_current_active_user -from .models import ApiKey, Cpe, Cve, Domain, Organization, Role, User, Vulnerability +from .models import ( + ApiKey, + Assessment, + Cpe, + Cve, + Domain, + Organization, + Role, + User, + Vulnerability, +) from .schemas import Role as RoleSchema from .schemas import User as UserSchema @@ -115,6 +125,42 @@ def read_orgs(current_user: User = Depends(get_current_active_user)): raise HTTPException(status_code=500, detail=str(e)) +# TODO: Uncomment checks for current_user once authentication is implemented +@api_router.get( + "/assessments", + # current_user: User = Depends(get_current_active_user), + tags=["ReadySetCyber"], +) +async def list_assessments(): + """ + Lists all assessments for the logged-in user. + + Args: + current_user (User): The current authenticated user. + + Raises: + HTTPException: If the user is not authorized or assessments are not found. + + Returns: + List[Assessment]: A list of assessments for the logged-in user. + """ + # Ensure the user is authenticated + # if not current_user: + # raise HTTPException(status_code=401, detail="Unauthorized") + + # Query the database for assessments belonging to the current user + # assessments = Assessment.objects.filter(user=current_user) + assessments = ( + Assessment.objects.all() + ) # TODO: Remove this line once filtering by user is implemented + + # Return assessments if found, or raise a 404 error if none exist + if not assessments.exists(): + raise HTTPException(status_code=404, detail="No assessments found") + + return list(assessments) + + @api_router.post("/search") async def search(): pass From 03351c9fff2ed9831cc18e66d81c38032f710092 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 16 Sep 2024 10:43:27 -0400 Subject: [PATCH 019/314] Add scans endpoints --- backend/src/api/scans.ts | 2 + backend/src/xfd_django/xfd_api/auth.py | 4 +- backend/src/xfd_django/xfd_api/models.py | 88 +++-- backend/src/xfd_django/xfd_api/scans.py | 381 ++++++++++++++++++++++ backend/src/xfd_django/xfd_api/schemas.py | 84 ++++- backend/src/xfd_django/xfd_api/views.py | 148 +++++++-- docker-compose.yml | 176 +++++----- frontend/src/pages/Scans/ScansView.tsx | 11 +- 8 files changed, 723 insertions(+), 171 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/scans.py diff --git a/backend/src/api/scans.ts b/backend/src/api/scans.ts index 79d20483..f1d95070 100644 --- a/backend/src/api/scans.ts +++ b/backend/src/api/scans.ts @@ -348,6 +348,8 @@ export const update = wrapHandler(async (event) => { * - Scans */ export const create = wrapHandler(async (event) => { + console.log(event); + console.log(event.body); if (!isGlobalWriteAdmin(event)) return Unauthorized; await connectToDatabase(); const body = await validateBody(NewScan, event.body); diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index f1c5f001..81929c10 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -68,9 +68,9 @@ def get_user_by_api_key(api_key: str): hashed_key = sha256(api_key.encode()).hexdigest() try: api_key_instance = ApiKey.objects.get(hashedKey=hashed_key) - api_key_instance.lastused = timezone.now() + api_key_instance.lastUsed = timezone.now() api_key_instance.save(update_fields=["lastUsed"]) - return api_key_instance.userid + return api_key_instance.userId except ApiKey.DoesNotExist: print("API Key not found") return None diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index c5978a9e..856f083c 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -8,6 +8,8 @@ # Feel free to rename the models, but don't rename db_table values or field names. # Third-Party Libraries from django.db import models +from django.contrib.postgres.fields import ArrayField, JSONField +import uuid class ApiKey(models.Model): @@ -245,19 +247,15 @@ class Meta: class Organization(models.Model): """The Organization model.""" - id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column="createdAt") - updatedAt = models.DateTimeField(db_column="updatedAt") - acronym = models.CharField(unique=True, blank=True, null=True) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) + updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) + acronym = models.CharField(unique=True, blank=True, null=True, max_length=255) name = models.CharField() - rootDomains = models.TextField( - db_column="rootDomains" - ) # This field type is a guess. - ipBlocks = models.TextField(db_column="ipBlocks") # This field type is a guess. + rootDomains = ArrayField(models.CharField(max_length=255), db_column="rootDomains") + ipBlocks = ArrayField(models.CharField(max_length=255), db_column="ipBlocks") isPassive = models.BooleanField(db_column="isPassive") - pendingDomains = models.TextField( - db_column="pendingDomains" - ) # This field type is a guess. + pendingDomains = models.TextField(db_column="pendingDomains", default=list) country = models.CharField(blank=True, null=True) state = models.CharField(blank=True, null=True) regionId = models.CharField(db_column="regionId", blank=True, null=True) @@ -272,6 +270,9 @@ class Organization(models.Model): createdById = models.ForeignKey( "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True ) + # Relationships with other models (Scan, OrganizationTag) + granularScans = models.ManyToManyField('Scan', related_name="organizations", through='ScanOrganizationsOrganization') + tags = models.ManyToManyField('OrganizationTag', related_name="organizations", through='ScanTagsOrganizationTag') class Meta: """The meta class for Organization.""" @@ -283,9 +284,9 @@ class Meta: class OrganizationTag(models.Model): """The OrganizationTag model.""" - id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column="createdAt") - updatedAt = models.DateTimeField(db_column="updatedAt") + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) + updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) name = models.CharField(unique=True) class Meta: @@ -298,14 +299,15 @@ class Meta: class OrganizationTagOrganizationsOrganization(models.Model): """The OrganizationTagOrganizationsOrganization model.""" - organizationTagId = models.OneToOneField( + organizationTagId = models.ForeignKey( OrganizationTag, - models.DO_NOTHING, - db_column="organizationTagId", - primary_key=True, - ) # The composite primary key (organizationTagId, organizationId) found, that is not supported. The first column is selected. + on_delete=models.CASCADE, + db_column="organizationTagId" + ) organizationId = models.ForeignKey( - Organization, models.DO_NOTHING, db_column="organizationId" + Organization, + on_delete=models.CASCADE, + db_column="organizationId" ) class Meta: @@ -314,6 +316,7 @@ class Meta: managed = False db_table = "organization_tag_organizations_organization" unique_together = (("organizationTagId", "organizationId"),) + auto_created = True class QueryResultCache(models.Model): @@ -479,22 +482,24 @@ class Meta: class Scan(models.Model): """The Scan model.""" - id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column="createdAt") - updatedAt = models.DateTimeField(db_column="updatedAt") + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) + updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) name = models.CharField() - arguments = models.JSONField() + arguments = models.TextField() # JSON in the database but fails: the JSON object must be str, bytes or bytearray, not dict frequency = models.IntegerField() lastRun = models.DateTimeField(db_column="lastRun", blank=True, null=True) - isGranular = models.BooleanField(db_column="isGranular") + isGranular = models.BooleanField(db_column="isGranular", default=False) isUserModifiable = models.BooleanField( - db_column="isUserModifiable", blank=True, null=True + db_column="isUserModifiable", blank=True, null=True, default=False ) - isSingleScan = models.BooleanField(db_column="isSingleScan") - manualRunPending = models.BooleanField(db_column="manualRunPending") + isSingleScan = models.BooleanField(db_column="isSingleScan", default=False) + manualRunPending = models.BooleanField(db_column="manualRunPending", default=False) createdBy = models.ForeignKey( "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True ) + tags = models.ManyToManyField('OrganizationTag', through='ScanTagsOrganizationTag', related_name='scans') + organizations = models.ManyToManyField('Organization', through='ScanOrganizationsOrganization', related_name='scans') class Meta: """The Meta class for Scan.""" @@ -506,37 +511,26 @@ class Meta: class ScanOrganizationsOrganization(models.Model): """The ScanOrganizationsOrganization model.""" - scanId = models.OneToOneField( - Scan, models.DO_NOTHING, db_column="scanId", primary_key=True - ) # The composite primary key (scanId, organizationId) found, that is not supported. The first column is selected. - organizationId = models.ForeignKey( - Organization, models.DO_NOTHING, db_column="organizationId" - ) + scanId = models.ForeignKey('Scan', on_delete=models.CASCADE, db_column="scanId", primary_key=True) + organizationId = models.ForeignKey('Organization', on_delete=models.CASCADE, db_column="organizationId", primary_key=True) class Meta: - """The Meta class for ScanOrganizationsOrganization.""" - - managed = False db_table = "scan_organizations_organization" unique_together = (("scanId", "organizationId"),) + # Do not create an id column automatically, treat both columns as composite primary keys + auto_created = True class ScanTagsOrganizationTag(models.Model): - """The ScanTagsOrganizationTag model.""" + """Intermediary model for the Many-to-Many relationship between Scan and OrganizationTag.""" - scanId = models.OneToOneField( - Scan, models.DO_NOTHING, db_column="scanId", primary_key=True - ) # The composite primary key (scanId, organizationTagId) found, that is not supported. The first column is selected. - organizationTagId = models.ForeignKey( - OrganizationTag, models.DO_NOTHING, db_column="organizationTagId" - ) + scanId = models.ForeignKey('Scan', on_delete=models.CASCADE, db_column="scanId", primary_key=True) + organizationTagId = models.ForeignKey('OrganizationTag', on_delete=models.CASCADE, db_column="organizationTagId", primary_key=True) class Meta: - """The Meta class for ScanTagsOrganizationTag.""" - - managed = False db_table = "scan_tags_organization_tag" unique_together = (("scanId", "organizationTagId"),) + auto_created = True class ScanTask(models.Model): diff --git a/backend/src/xfd_django/xfd_api/scans.py b/backend/src/xfd_django/xfd_api/scans.py new file mode 100644 index 00000000..6bc17736 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/scans.py @@ -0,0 +1,381 @@ +from .models import Scan, Organization, OrganizationTag, ScanTagsOrganizationTag, ScanOrganizationsOrganization +from .schemas import ScanSchema +from django.db import transaction +from fastapi import HTTPException +from django.http import JsonResponse +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder + + +SCAN_SCHEMA = { + "amass": ScanSchema( + type="fargate", + isPassive=False, + global_scan=False, + description="Open source tool that integrates passive APIs and active subdomain enumeration in order to discover target subdomains", + ), + "censys": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Passive discovery of subdomains from public certificates", + ), + "censysCertificates": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="2048", + memory="6144", + numChunks=20, + description="Fetch TLS certificate data from censys certificates dataset", + ), + "censysIpv4": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="2048", + memory="6144", + numChunks=20, + description="Fetch passive port and banner data from censys ipv4 dataset", + ), + "cve": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog.", + ), + "vulnScanningSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Pull in vulnerability data from VSs Vulnerability database", + ), + "cveSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog.", + ), + "dnstwist": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="2048", + memory="16384", + description="Domain name permutation engine for detecting similar registered domains.", + ), + "dotgov": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + description='Create organizations based on root domains from the dotgov registrar dataset. All organizations are created with the "dotgov" tag and have a " (dotgov)" suffix added to their name.', + ), + "findomain": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Open source tool that integrates passive APIs in order to discover target subdomains", + ), + "hibp": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="2048", + memory="16384", + description="Finds emails that have appeared in breaches related to a given domain", + ), + "intrigueIdent": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="1024", + memory="4096", + description="Open source tool that fingerprints web technologies based on HTTP responses", + ), + "lookingGlass": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Finds vulnerabilities and malware from the LookingGlass API", + ), + "portscanner": ScanSchema( + type="fargate", + isPassive=False, + global_scan=False, + description="Active port scan of common ports", + ), + "rootDomainSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Creates domains from root domains by doing a single DNS lookup for each root domain.", + ), + "rscSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + description="Retrieves and saves assessments from ReadySetCyber mission instance.", + ), + "savedSearch": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + description="Performs saved searches to update their search results", + ), + "searchSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="4096", + description="Syncs records with Elasticsearch so that they appear in search results.", + ), + "shodan": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="1024", + memory="8192", + description="Fetch passive port, banner, and vulnerability data from shodan", + ), + "sslyze": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="SSL certificate inspection", + ), + "test": ScanSchema( + type="fargate", + isPassive=False, + global_scan=True, + description="Not a real scan, used to test", + ), + "trustymail": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Evaluates SPF/DMARC records and checks MX records for STARTTLS support", + ), + "vulnSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Pull in vulnerability data from PEs Vulnerability database", + ), + "wappalyzer": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="1024", + memory="4096", + description="Open source tool that fingerprints web technologies based on HTTP responses", + ), + "xpanseSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Pull in xpanse vulnerability data from PEs Vulnerability database", + ), +} + + +def list_scans(): + """List scans.""" + try: + # Fetch scans + scans = Scan.objects.all().values() + + # Convert scans and related tags to dict format + scan_list = [] + for scan in scans: + related_tags = OrganizationTag.objects.filter( + scantagsorganizationtag__scanId=scan['id'] + ).values() + + # Add tags to each scan + scan['tags'] = list(related_tags) + scan_list.append(scan) + + # Fetch all organizations + organizations = list(Organization.objects.values('id', 'name')) + + # Return everything as a JSON response + response = { + 'scans': scan_list, + 'schema': SCAN_SCHEMA, # Add your predefined SCAN_SCHEMA here + 'organizations': organizations + } + return response + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def list_granular_scans(): + """ + List all granular scans that can be modified by users. + """ + try: + scans = Scan.objects.filter( + isGranular=True, isUserModifiable=True, isSingleScan=False + ) + return {"scans": list(scans), 'schema': SCAN_SCHEMA} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def create_scan(scan_data: Scan, current_user): + """ + Create a new scan. + """ + try: + # Create the scan instance using **scan_data.dict() + scan_data_dict = scan_data.dict(exclude_unset=True, exclude={"organizations", "frequencyUnit", "tags"}) + scan_data_dict['createdBy'] = current_user + print(scan_data_dict) + + # Create scan using the dictionary unpacking + scan = Scan.objects.create(**scan_data_dict) + + print("added Scan") + print(scan) + # Link organizations + if scan_data.organizations: + for org_id in scan_data.organizations: + organization, created = Organization.objects.get_or_create(id=org_id) + print(f"Linked Organization {organization}") + ScanOrganizationsOrganization.objects.create( + scanId=scan, + organizationId=organization + ) + + # Link tags + if scan_data.tags: + for tag_data in scan_data.tags: + tag, created = OrganizationTag.objects.get_or_create(id=tag_data.id) + print(f"Linked Tag {tag}") + ScanTagsOrganizationTag.objects.create( + scanId=scan, + organizationTagId=tag + ) + + # Return the saved scan + return scan + + except Organization.DoesNotExist: + raise HTTPException(status_code=404, detail="Organization not found") + except OrganizationTag.DoesNotExist: + raise HTTPException(status_code=404, detail="Tag not found") + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + + + +def get_scan(scan_id: str): + """ + Retrieve a scan by its ID. + + Parameters: + - scan_id: The ID of the scan to retrieve. + + Returns: + - The scan object with the specified ID. + """ + try: + scan = Scan.objects.get(id=scan_id) + if not scan: + raise HTTPException(status_code=404, detail="Scan not found") + return scan + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def update_scan(scan_id: str, scan_data: Scan): + """ + Update a scan by its ID. + + Parameters: + - scan_id: The ID of the scan to update. + - scan_data: The new data to update the scan with. + + Returns: + - The updated scan object. + """ + try: + scan = Scan.objects.get(id=scan_id) + if not scan: + raise HTTPException(status_code=404, detail="Scan not found") + + for key, value in scan_data.dict().items(): + setattr(scan, key, value) + scan.save() + return scan + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def delete_scan(scan_id: str): + """ + Delete a scan by its ID. + + Parameters: + - scan_id: The ID of the scan to delete. + + Returns: + - A message confirming the deletion. + """ + try: + scan = Scan.objects.get(id=scan_id) + if not scan: + raise HTTPException(status_code=404, detail="Scan not found") + scan.delete() + return {"message": "Scan deleted"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def run_scan(scan_id: str): + """ + Mark a scan as manually triggered to run. + + Parameters: + - scan_id: The ID of the scan to run. + + Returns: + - A message confirming that the scan has been triggered. + """ + try: + scan = Scan.objects.get(id=scan_id) + if not scan: + raise HTTPException(status_code=404, detail="Scan not found") + scan.manualRunPending = True + scan.save() + return {"message": "Scan manually run"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def invoke_scheduler(): + """ + Manually invoke the scan scheduler. + + Returns: + - A message confirming that the scheduler has been invoked. + """ + try: + # Implement logic to invoke the scheduler manually + # This could be an external service call or AWS Lambda invocation + return {"message": "Scheduler invoked successfully"} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schemas.py b/backend/src/xfd_django/xfd_api/schemas.py index 0601f751..a9f12535 100644 --- a/backend/src/xfd_django/xfd_api/schemas.py +++ b/backend/src/xfd_django/xfd_api/schemas.py @@ -3,7 +3,7 @@ # from pydantic.types import UUID1, UUID # Standard Python Libraries from datetime import datetime -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional from uuid import UUID # Third-Party Libraries @@ -131,19 +131,18 @@ class Notification(BaseModel): updatedBy: datetime message: Optional[str] - class Organization(BaseModel): - """Organization schema.""" + """Organization schema reflecting model.""" id: UUID createdAt: datetime updatedAt: datetime acronym: Optional[str] name: str - rootDomains: str - ipBlocks: str + rootDomains: List[str] + ipBlocks: List[str] isPassive: bool - pendingDomains: str + pendingDomains: Optional[List[dict]] country: Optional[str] state: Optional[str] regionId: Optional[str] @@ -153,6 +152,9 @@ class Organization(BaseModel): countyFips: Optional[int] type: Optional[str] + class Config: + orm_mode = True + class Question(BaseModel): """Question schema.""" @@ -197,10 +199,15 @@ class Role(BaseModel): userId: Optional[Any] organizationId: Optional[Any] +class OrganizationalTags(BaseModel): + """Organization Tags.""" + id: UUID + createdAt: datetime + updatedAt: datetime + name: str class Scan(BaseModel): """Scan schema.""" - id: UUID createdAt: datetime updatedAt: datetime @@ -212,6 +219,69 @@ class Scan(BaseModel): isUserModifiable: Optional[bool] isSingleScan: bool manualRunPending: bool + createdBy_id: Optional[Any] + tags: Optional[List[OrganizationalTags]] + +class ScanSchema(BaseModel): + """Scan type schema.""" + + type: str = 'fargate' # Only 'fargate' is supported + description: str + + # Whether scan is passive (not allowed to hit the domain). + isPassive: bool + + # Whether scan is global. Global scans run once for all organizations, as opposed + # to non-global scans, which are run for each organization. + global_scan: bool + + # CPU and memory for the scan. See this page for more information: + # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html + cpu: Optional[str] = None + memory: Optional[str] = None + + # A scan is "chunked" if its work is divided and run in parallel by multiple workers. + # To make a scan chunked, make sure it is a global scan and specify the "numChunks" variable, + # which corresponds to the number of workers that will be created to run the task. + # Chunked scans can only be run on scans whose implementation takes into account the + # chunkNumber and numChunks parameters specified in commandOptions. + numChunks: Optional[int] = None + +class GetScansResponseModel(BaseModel): + """Get Scans response model.""" + scans: List[Scan] + schema: Dict[str, Any] + organizations: List[Dict[str, Any]] + +class GetGranularScansResponseModel(BaseModel): + """Get Scans response model.""" + scans: List[Scan] + schema: Dict[str, Any] + +class IdSchema(BaseModel): + """Schema for ID objects.""" + id: UUID + +class CreateScan(BaseModel): + """Create Scan Schema.""" + name: str + arguments: Any + organizations: Optional[List[UUID]] + tags: Optional[List[IdSchema]] + frequency: int + frequencyUnit: str + isGranular: bool + isUserModifiable: Optional[bool] + isSingleScan: bool + +class CreateScanResponseModel(BaseModel): + """Create Scan Schema.""" + name: str + arguments: Any + frequency: int + isGranular: bool + isUserModifiable: Optional[bool] + isSingleScan: bool createdBy: Optional[Any] diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index d7400776..5b8b34af 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -16,6 +16,7 @@ - .auth - .models """ + # Standard Python Libraries from typing import Any, List, Optional, Union @@ -30,6 +31,7 @@ from .models import ApiKey, Cpe, Cve, Domain, Organization, Role, User, Vulnerability from .schemas import Role as RoleSchema from .schemas import User as UserSchema +from . import scans api_router = APIRouter() @@ -76,7 +78,7 @@ async def get_api_keys(): @api_router.post( "/test-orgs", dependencies=[Depends(get_current_active_user)], - response_model=List[schemas.Organization], + # response_model=List[schemas.Organization], tags=["List of all Organizations"], ) def read_orgs(current_user: User = Depends(get_current_active_user)): @@ -86,31 +88,8 @@ def read_orgs(current_user: User = Depends(get_current_active_user)): """ try: organizations = Organization.objects.all() - return [ - { - "id": organization.id, - "name": organization.name, - "acronym": organization.acronym, - "rootDomains": organization.rootDomains, - "ipBlocks": organization.ipBlocks, - "isPassive": organization.isPassive, - "country": organization.country, - "state": organization.state, - "regionId": organization.regionId, - "stateFips": organization.stateFips, - "stateName": organization.stateName, - "county": organization.county, - "countyFips": organization.countyFips, - "type": organization.type, - "parentId": organization.parentId.id if organization.parentId else None, - "createdById": organization.createdById.id - if organization.createdById - else None, - "createdAt": organization.createdAt, - "updatedAt": organization.updatedAt, - } - for organization in organizations - ] + print(organizations) + return organizations except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -321,3 +300,120 @@ async def get_users( # Return the Pydantic models directly by calling from_orm return [UserSchema.from_orm(user) for user in users] + + +############################################################################## +@api_router.get( + "/scans", + dependencies=[Depends(get_current_active_user)], + response_model=schemas.GetScansResponseModel, + tags=["Scans"], +) +async def list_scans(current_user: User = Depends(get_current_active_user)): + """Retrieve a list of all scans.""" + return scans.list_scans() + + +@api_router.get( + "/granularScans", + dependencies=[Depends(get_current_active_user)], + # response_model=schemas.GetGranularScansResponseModel, + tags=["Scans"], +) +async def list_granular_scans(current_user: User = Depends(get_current_active_user)): + """Retrieve a list of granular scans. User must be authenticated.""" + return scans.list_granular_scans() + + +@api_router.post( + "/scans", + dependencies=[Depends(get_current_active_user)], + # response_model=schemas.CreateScanResponseModel, + tags=["Scans"], +) +async def create_scan( + scan_data: schemas.CreateScan, current_user: User = Depends(get_current_active_user) +): + """ Create a new scan.""" + return scans.create_scan(scan_data, current_user) + + +@api_router.get("/scans/{scan_id}", response_model=schemas.Scan) +async def get_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): + """ + Endpoint to retrieve a scan by its ID. User must be authenticated. + + Args: + scan_id (str): The ID of the scan to retrieve. + current_user (User): The authenticated user, injected via Depends. + + Returns: + The scan object. + """ + return scans.get_scan(scan_id) + + +@api_router.put("/scans/{scan_id}", response_model=schemas.Scan) +async def update_scan( + scan_id: str, + scan_data: schemas.Scan, + current_user: User = Depends(get_current_active_user), +): + """ + Endpoint to update a scan by its ID. User must be authenticated. + + Args: + scan_id (str): The ID of the scan to update. + scan_data (ScanUpdate): The updated scan data. + current_user (User): The authenticated user, injected via Depends. + + Returns: + The updated scan object. + """ + return scans.update_scan(scan_id, scan_data) + + +@api_router.delete("/scans/{scan_id}") +async def delete_scan( + scan_id: str, current_user: User = Depends(get_current_active_user) +): + """ + Endpoint to delete a scan by its ID. User must be authenticated. + + Args: + scan_id (str): The ID of the scan to delete. + current_user (User): The authenticated user, injected via Depends. + + Returns: + A confirmation message. + """ + return scans.delete_scan(scan_id) + + +@api_router.post("/scans/{scan_id}/run") +async def run_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): + """ + Endpoint to manually run a scan by its ID. User must be authenticated. + + Args: + scan_id (str): The ID of the scan to run. + current_user (User): The authenticated user, injected via Depends. + + Returns: + A confirmation message that the scan has been triggered. + """ + return scans.run_scan(scan_id) + + +@api_router.post("/scheduler/invoke") +async def invoke_scheduler(current_user: User = Depends(get_current_active_user)): + """ + Endpoint to manually invoke the scan scheduler. User must be authenticated. + + Args: + current_user (User): The authenticated user, injected via Depends. + + Returns: + A confirmation message that the scheduler was invoked. + """ + return scans.invoke_scheduler() diff --git a/docker-compose.yml b/docker-compose.yml index f2d12e06..4bcc22b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,55 +57,55 @@ services: depends_on: - db - minio: - image: bitnami/minio:2020.9.26 - user: root - command: 'minio server /data' - networks: - - backend - volumes: - - ./minio-data:/data - ports: - - 9000:9000 - environment: - - MINIO_ACCESS_KEY=aws_access_key - - MINIO_SECRET_KEY=aws_secret_key - logging: - driver: json-file + # minio: + # image: bitnami/minio:2020.9.26 + # user: root + # command: 'minio server /data' + # networks: + # - backend + # volumes: + # - ./minio-data:/data + # ports: + # - 9000:9000 + # environment: + # - MINIO_ACCESS_KEY=aws_access_key + # - MINIO_SECRET_KEY=aws_secret_key + # logging: + # driver: json-file - docs: - build: - context: ./ - dockerfile: ./Dockerfile.docs - volumes: - - ./docs/src:/app/docs/src - - ./docs/gatsby-browser.js:/app/docs/gatsby-browser.js - - ./docs/gatsby-config.js:/app/docs/gatsby-config.js - - ./docs/gatsby-node.js:/app/docs/gatsby-node.js - ports: - - '4000:4000' - - '44475:44475' + # docs: + # build: + # context: ./ + # dockerfile: ./Dockerfile.docs + # volumes: + # - ./docs/src:/app/docs/src + # - ./docs/gatsby-browser.js:/app/docs/gatsby-browser.js + # - ./docs/gatsby-config.js:/app/docs/gatsby-config.js + # - ./docs/gatsby-node.js:/app/docs/gatsby-node.js + # ports: + # - '4000:4000' + # - '44475:44475' - es: - image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0 - environment: - - discovery.type=single-node - - bootstrap.memory_lock=true - - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' - command: ['elasticsearch', '-Elogger.level=WARN'] - networks: - - backend - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - es-data:/usr/share/elasticsearch/data - ports: - - 9200:9200 - - 9300:9300 - logging: - driver: none + # es: + # image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0 + # environment: + # - discovery.type=single-node + # - bootstrap.memory_lock=true + # - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' + # command: ['elasticsearch', '-Elogger.level=WARN'] + # networks: + # - backend + # ulimits: + # memlock: + # soft: -1 + # hard: -1 + # volumes: + # - es-data:/usr/share/elasticsearch/data + # ports: + # - 9200:9200 + # - 9300:9300 + # logging: + # driver: none # kib: # image: docker.elastic.co/kibana/kibana:7.9.0 @@ -118,48 +118,48 @@ services: # ELASTICSEARCH_HOSTS: http://es:9200 # LOGGING_QUIET: 'true' - matomodb: - image: mariadb:10.6 - command: --max-allowed-packet=64MB - networks: - - backend - volumes: - - ./matomo-db-data:/var/lib/mysql - environment: - - MYSQL_ROOT_PASSWORD=password - logging: - driver: none + # matomodb: + # image: mariadb:10.6 + # command: --max-allowed-packet=64MB + # networks: + # - backend + # volumes: + # - ./matomo-db-data:/var/lib/mysql + # environment: + # - MYSQL_ROOT_PASSWORD=password + # logging: + # driver: none - matomo: - image: matomo:3.14.1 - user: root - networks: - - backend - volumes: - - ./matomo-data:/var/www/html - environment: - - MATOMO_DATABASE_HOST=matomodb - - MATOMO_DATABASE_ADAPTER=mysql - - MATOMO_DATABASE_TABLES_PREFIX=matomo_ - - MATOMO_DATABASE_USERNAME=root - - MATOMO_DATABASE_PASSWORD=password - - MATOMO_DATABASE_DBNAME=matomo - - MATOMO_GENERAL_PROXY_URI_HEADER=1 - - MATOMO_GENERAL_ASSUME_SECURE_PROTOCOL=1 - logging: - driver: none - rabbitmq: - image: 'rabbitmq:3.8-management' - ports: - - '5672:5672' # RabbitMQ default port - - '15672:15672' # RabbitMQ management plugin - networks: - - backend - environment: - RABBITMQ_DEFAULT_USER: guest - RABBITMQ_DEFAULT_PASS: guest - volumes: - - rabbitmq-data:/var/lib/rabbitmq + # matomo: + # image: matomo:3.14.1 + # user: root + # networks: + # - backend + # volumes: + # - ./matomo-data:/var/www/html + # environment: + # - MATOMO_DATABASE_HOST=matomodb + # - MATOMO_DATABASE_ADAPTER=mysql + # - MATOMO_DATABASE_TABLES_PREFIX=matomo_ + # - MATOMO_DATABASE_USERNAME=root + # - MATOMO_DATABASE_PASSWORD=password + # - MATOMO_DATABASE_DBNAME=matomo + # - MATOMO_GENERAL_PROXY_URI_HEADER=1 + # - MATOMO_GENERAL_ASSUME_SECURE_PROTOCOL=1 + # logging: + # driver: none + # rabbitmq: + # image: 'rabbitmq:3.8-management' + # ports: + # - '5672:5672' # RabbitMQ default port + # - '15672:15672' # RabbitMQ management plugin + # networks: + # - backend + # environment: + # RABBITMQ_DEFAULT_USER: guest + # RABBITMQ_DEFAULT_PASS: guest + # volumes: + # - rabbitmq-data:/var/lib/rabbitmq volumes: postgres-data: diff --git a/frontend/src/pages/Scans/ScansView.tsx b/frontend/src/pages/Scans/ScansView.tsx index 8e0372e0..df55b8ee 100644 --- a/frontend/src/pages/Scans/ScansView.tsx +++ b/frontend/src/pages/Scans/ScansView.tsx @@ -111,7 +111,16 @@ const ScansView: React.FC = () => { body.arguments = JSON.parse(body.arguments); setFrequency(body); - const scan = await apiPost('/scans/', { + // Log the body to the console for testing + console.log('Full Request Body:', { + ...body, + organizations: body.organizations + ? body.organizations.map((e) => e.value) + : [], + tags: body.tags ? body.tags.map((e) => ({ id: e.value })) : [] + }); + + const scan = await apiPost('/scans', { body: { ...body, organizations: body.organizations From 02082bc57eca422025dbca179e5c3df9a2dc78ff Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 17 Sep 2024 14:24:36 -0500 Subject: [PATCH 020/314] Refactor API schemas into their own separate files, revisions to foreign key relationships. --- backend/src/xfd_django/xfd_api/models.py | 2 +- .../xfd_api/schema_models/__init__.py | 0 .../xfd_api/schema_models/assessment.py | 21 + .../xfd_api/schema_models/category.py | 18 + .../xfd_django/xfd_api/schema_models/cpe.py | 20 + .../xfd_django/xfd_api/schema_models/cve.py | 41 ++ .../xfd_api/schema_models/domain.py | 63 +++ .../xfd_api/schema_models/notification.py | 24 ++ .../xfd_api/schema_models/organization.py | 32 ++ .../xfd_api/schema_models/question.py | 20 + .../xfd_api/schema_models/resource.py | 18 + .../xfd_api/schema_models/response.py | 18 + .../xfd_django/xfd_api/schema_models/role.py | 24 ++ .../xfd_django/xfd_api/schema_models/scan.py | 46 +++ .../xfd_api/schema_models/searchbody.py | 22 ++ .../xfd_api/schema_models/service.py | 30 ++ .../xfd_django/xfd_api/schema_models/user.py | 47 +++ .../xfd_api/schema_models/vulnerability.py | 63 +++ backend/src/xfd_django/xfd_api/schemas.py | 364 +----------------- backend/src/xfd_django/xfd_api/views.py | 33 +- 20 files changed, 548 insertions(+), 358 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/schema_models/__init__.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/assessment.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/category.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/cpe.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/cve.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/domain.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/notification.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/organization.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/question.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/resource.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/response.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/role.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/scan.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/searchbody.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/service.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/user.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/vulnerability.py diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index c5978a9e..4e0f4c19 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -210,7 +210,7 @@ class Meta: db_table = "domain" managed = False # This ensures Django does not manage the table - unique_together = (("name", "organization"),) # Unique constraint + unique_together = (("name", "organizationId"),) # Unique constraint def save(self, *args, **kwargs): self.name = self.name.lower() diff --git a/backend/src/xfd_django/xfd_api/schema_models/__init__.py b/backend/src/xfd_django/xfd_api/schema_models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/schema_models/assessment.py b/backend/src/xfd_django/xfd_api/schema_models/assessment.py new file mode 100644 index 00000000..d8281984 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/assessment.py @@ -0,0 +1,21 @@ +"""Assessment Schemas.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Any, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Assessment(BaseModel): + """Assessment schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + rscId: str + type: str + userId: Optional[Any] diff --git a/backend/src/xfd_django/xfd_api/schema_models/category.py b/backend/src/xfd_django/xfd_api/schema_models/category.py new file mode 100644 index 00000000..14456fca --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/category.py @@ -0,0 +1,18 @@ +"""Category Schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from typing import Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Category(BaseModel): + """Category schema.""" + + id: UUID + name: str + number: str + shortName: Optional[str] diff --git a/backend/src/xfd_django/xfd_api/schema_models/cpe.py b/backend/src/xfd_django/xfd_api/schema_models/cpe.py new file mode 100644 index 00000000..578df647 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/cpe.py @@ -0,0 +1,20 @@ +"""Cpe schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Cpe(BaseModel): + """Cpe schema.""" + + id: UUID + name: Optional[str] + version: Optional[str] + vendor: Optional[str] + lastSeenAt: datetime diff --git a/backend/src/xfd_django/xfd_api/schema_models/cve.py b/backend/src/xfd_django/xfd_api/schema_models/cve.py new file mode 100644 index 00000000..50aa9f23 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/cve.py @@ -0,0 +1,41 @@ +"""Cve schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Cve(BaseModel): + """Cve schema.""" + + id: UUID + name: Optional[str] + publishedAt: datetime + modifiedAt: datetime + status: str + description: Optional[str] + cvssV2Source: Optional[str] + cvssV2Type: Optional[str] + cvssV2VectorString: Optional[str] + cvssV2BaseSeverity: Optional[str] + cvssV2ExploitabilityScore: Optional[str] + cvssV2ImpactScore: Optional[str] + cvssV3Source: Optional[str] + cvssV3Type: Optional[str] + cvssV3VectorString: Optional[str] + cvssV3BaseSeverity: Optional[str] + cvssV3ExploitabilityScore: Optional[str] + cvssV3ImpactScore: Optional[str] + cvssV4Source: Optional[str] + cvssV4Type: Optional[str] + cvssV4VectorString: Optional[str] + cvssV4BaseSeverity: Optional[str] + cvssV4ExploitabilityScore: Optional[str] + cvssV4ImpactScore: Optional[str] + weaknesses: Optional[str] + references: Optional[str] diff --git a/backend/src/xfd_django/xfd_api/schema_models/domain.py b/backend/src/xfd_django/xfd_api/schema_models/domain.py new file mode 100644 index 00000000..edfbdee6 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/domain.py @@ -0,0 +1,63 @@ +"""Domain schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Any, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Domain(BaseModel): + """Domain schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + syncedAt: datetime + ip: str + fromRootDomain: Optional[str] + subdomainSource: Optional[str] + ipOnly: bool + reverseName: Optional[str] + name: Optional[str] + screenshot: Optional[str] + country: Optional[str] + asn: Optional[str] + cloudHosted: bool + ssl: Optional[Any] + censysCertificatesResults: Optional[dict] + trustymailResults: Optional[dict] + discoveredById_id: Optional[UUID] + organizationId_id: Optional[UUID] + + class Config: + """Domain base schema config.""" + + orm_mode = True + validate_assignment = True + + +class DomainFilters(BaseModel): + """DomainFilters schema.""" + + ports: Optional[str] = None + service: Optional[str] = None + reverseName: Optional[str] = None + ip: Optional[str] = None + organization: Optional[str] = None + organizationName: Optional[str] = None + vulnerabilities: Optional[str] = None + tag: Optional[str] = None + + +class DomainSearch(BaseModel): + """DomainSearch schema.""" + + page: int = 1 + sort: str + order: str + filters: Optional[DomainFilters] + pageSize: Optional[int] = None diff --git a/backend/src/xfd_django/xfd_api/schema_models/notification.py b/backend/src/xfd_django/xfd_api/schema_models/notification.py new file mode 100644 index 00000000..91b765ab --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/notification.py @@ -0,0 +1,24 @@ +"""Notification schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Notification(BaseModel): + """Notification schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + startDatetime: Optional[datetime] + endDateTime: Optional[datetime] + maintenanceType: Optional[str] + status: Optional[str] + updatedBy: datetime + message: Optional[str] diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization.py b/backend/src/xfd_django/xfd_api/schema_models/organization.py new file mode 100644 index 00000000..f52f2406 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/organization.py @@ -0,0 +1,32 @@ +"""Organization schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Organization(BaseModel): + """Organization schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + acronym: Optional[str] + name: str + rootDomains: str + ipBlocks: str + isPassive: bool + pendingDomains: str + country: Optional[str] + state: Optional[str] + regionId: Optional[str] + stateFips: Optional[int] + stateName: Optional[str] + county: Optional[str] + countyFips: Optional[int] + type: Optional[str] diff --git a/backend/src/xfd_django/xfd_api/schema_models/question.py b/backend/src/xfd_django/xfd_api/schema_models/question.py new file mode 100644 index 00000000..310a5ded --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/question.py @@ -0,0 +1,20 @@ +"""Question schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from typing import Any, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Question(BaseModel): + """Question schema.""" + + id: UUID + name: str + description: str + longForm: str + number: str + categoryId: Optional[Any] diff --git a/backend/src/xfd_django/xfd_api/schema_models/resource.py b/backend/src/xfd_django/xfd_api/schema_models/resource.py new file mode 100644 index 00000000..59dfb40b --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/resource.py @@ -0,0 +1,18 @@ +"""Resource schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Resource(BaseModel): + """Resource schema.""" + + id: UUID + description: str + name: str + type: str + url: str diff --git a/backend/src/xfd_django/xfd_api/schema_models/response.py b/backend/src/xfd_django/xfd_api/schema_models/response.py new file mode 100644 index 00000000..20580b70 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/response.py @@ -0,0 +1,18 @@ +"""Response schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from typing import Any, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Response(BaseModel): + """Response schema.""" + + id: UUID + selection: str + assessmentId: Optional[Any] + questionId: Optional[Any] diff --git a/backend/src/xfd_django/xfd_api/schema_models/role.py b/backend/src/xfd_django/xfd_api/schema_models/role.py new file mode 100644 index 00000000..4035d24e --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/role.py @@ -0,0 +1,24 @@ +"""Role schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Any, List, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, Json + + +class Role(BaseModel): + """Role schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + role: str + approved: bool + createdById: Optional[Any] + approvedById: Optional[Any] + userId: Optional[Any] + organizationId: Optional[Any] diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan.py b/backend/src/xfd_django/xfd_api/schema_models/scan.py new file mode 100644 index 00000000..fb7803c5 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/scan.py @@ -0,0 +1,46 @@ +"""Scan schemas.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Any, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +class Scan(BaseModel): + """Scan schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + name: str + arguments: Any + frequency: int + lastRun: Optional[datetime] + isGranular: bool + isUserModifiable: Optional[bool] + isSingleScan: bool + manualRunPending: bool + createdBy: Optional[Any] + + +class ScanTask(BaseModel): + """ScanTask schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + status: str + type: str + fargateTaskArn: Optional[str] + input: Optional[str] + output: Optional[str] + requestedAt: Optional[datetime] + startedAt: Optional[datetime] + finishedAt: Optional[datetime] + queuedAt: Optional[datetime] + organizationId: Optional[Any] + scanId: Optional[Any] diff --git a/backend/src/xfd_django/xfd_api/schema_models/searchbody.py b/backend/src/xfd_django/xfd_api/schema_models/searchbody.py new file mode 100644 index 00000000..cdfd884b --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/searchbody.py @@ -0,0 +1,22 @@ +"""Search Body schema""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from typing import Any, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, Json + + +class SearchBody(BaseModel): + """SearchBody schema.""" + + current: int + resultsPerPage: int + searchTerm: str + sortDirection: str + sortField: str + filters: Json[Any] + organizationId: Optional[UUID] + tagId: Optional[UUID] diff --git a/backend/src/xfd_django/xfd_api/schema_models/service.py b/backend/src/xfd_django/xfd_api/schema_models/service.py new file mode 100644 index 00000000..6990def1 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/service.py @@ -0,0 +1,30 @@ +"""Service schema.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Any, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, Json + + +class Service(BaseModel): + """Service schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + serviceSource: Optional[str] + port: int + service: Optional[str] + lastSeen: Optional[datetime] + banner: Optional[str] + products: Json[Any] + censysMetadata: Json[Any] + censysIpv4Results: Json[Any] + shodanResults: Json[Any] + wappalyzerResults: Json[Any] + domainId: Optional[Any] + discoveredById: Optional[Any] diff --git a/backend/src/xfd_django/xfd_api/schema_models/user.py b/backend/src/xfd_django/xfd_api/schema_models/user.py new file mode 100644 index 00000000..d27239ee --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/user.py @@ -0,0 +1,47 @@ +"""User schemas.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import List, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + +from .role import Role + + +class User(BaseModel): + """User schema.""" + + id: UUID + cognitoId: Optional[str] + loginGovId: Optional[str] + createdAt: datetime + updatedAt: datetime + firstName: str + lastName: str + fullName: str + email: str + invitePending: bool + loginBlockedByMaintenance: bool + dateAcceptedTerms: Optional[datetime] + acceptedTermsVersion: Optional[str] + lastLoggedIn: Optional[datetime] + userType: str + regionId: Optional[str] + state: Optional[str] + oktaId: Optional[str] + roles: Optional[List[Role]] = [] + + @classmethod + def from_orm(cls, obj): + # Convert roles to a list of RoleSchema before passing to Pydantic + user_dict = obj.__dict__.copy() + user_dict["roles"] = [Role.from_orm(role) for role in obj.roles.all()] + return cls(**user_dict) + + class Config: + orm_mode = True + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py b/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py new file mode 100644 index 00000000..b217d257 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py @@ -0,0 +1,63 @@ +"""Vulnerability schemas.""" +# Third-Party Libraries +# from pydantic.types import UUID1, UUID +# Standard Python Libraries +from datetime import datetime +from typing import Any, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, Json + + +class Vulnerability(BaseModel): + """Vulnerability schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + lastSeen: datetime + title: Optional[str] + cve: Optional[str] + cwe: Optional[str] + cpe: Optional[str] + description: Optional[str] + references: Json[Any] + cvss: float + severity: Optional[str] + needsPopulation: bool + state: Optional[str] + substate: Optional[str] + source: Optional[str] + notes: Optional[str] + actions: Json[Any] + structuredData: Json[Any] + isKev: bool + domainId: UUID + serviceId: UUID + + +class VulnerabilityFilters(BaseModel): + """VulnerabilityFilters schema.""" + + id: Optional[UUID] + title: Optional[str] + domain: Optional[str] + severity: Optional[str] + cpe: Optional[str] + state: Optional[str] + substate: Optional[str] + organization: Optional[UUID] + tag: Optional[UUID] + isKev: Optional[bool] + + +class VulnerabilitySearch(BaseModel): + """VulnerabilitySearch schema.""" + + page: int + sort: Optional[str] + order: str + filters: Optional[VulnerabilityFilters] + pageSize: Optional[int] + groupBy: Optional[str] diff --git a/backend/src/xfd_django/xfd_api/schemas.py b/backend/src/xfd_django/xfd_api/schemas.py index 0601f751..fa5fe9b8 100644 --- a/backend/src/xfd_django/xfd_api/schemas.py +++ b/backend/src/xfd_django/xfd_api/schemas.py @@ -9,347 +9,23 @@ # Third-Party Libraries from pydantic import BaseModel, Json - -class Assessment(BaseModel): - """Assessment schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - rscId: str - type: str - userId: Optional[Any] - - -class Category(BaseModel): - """Category schema.""" - - id: UUID - name: str - number: str - shortName: Optional[str] - - -class Cpe(BaseModel): - """Cpe schema.""" - - id: UUID - name: Optional[str] - version: Optional[str] - vendor: Optional[str] - lastSeenAt: datetime - - -class Cve(BaseModel): - """Cve schema.""" - - id: UUID - name: Optional[str] - publishedAt: datetime - modifiedAt: datetime - status: str - description: Optional[str] - cvssV2Source: Optional[str] - cvssV2Type: Optional[str] - cvssV2VectorString: Optional[str] - cvssV2BaseSeverity: Optional[str] - cvssV2ExploitabilityScore: Optional[str] - cvssV2ImpactScore: Optional[str] - cvssV3Source: Optional[str] - cvssV3Type: Optional[str] - cvssV3VectorString: Optional[str] - cvssV3BaseSeverity: Optional[str] - cvssV3ExploitabilityScore: Optional[str] - cvssV3ImpactScore: Optional[str] - cvssV4Source: Optional[str] - cvssV4Type: Optional[str] - cvssV4VectorString: Optional[str] - cvssV4BaseSeverity: Optional[str] - cvssV4ExploitabilityScore: Optional[str] - cvssV4ImpactScore: Optional[str] - weaknesses: Optional[str] - references: Optional[str] - - -class Domain(BaseModel): - """Domain schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - syncedAt: datetime - ip: str - fromRootDomain: Optional[str] - subdomainSource: Optional[str] - ipOnly: bool - reverseName: Optional[str] - name: Optional[str] - screenshot: Optional[str] - country: Optional[str] - asn: Optional[str] - cloudHosted: bool - ssl: Optional[Any] - censysCertificatesResults: Optional[dict] - trustymailResults: Optional[dict] - discoveredById: Optional[Any] - organizationId: Any - - -class DomainFilters(BaseModel): - """DomainFilters schema.""" - - ports: Optional[str] = None - service: Optional[str] = None - reverseName: Optional[str] = None - ip: Optional[str] = None - organization: Optional[str] = None - organizationName: Optional[str] = None - vulnerabilities: Optional[str] = None - tag: Optional[str] = None - - -class DomainSearch(BaseModel): - """DomainSearch schema.""" - - page: int = 1 - sort: str - order: str - filters: Optional[DomainFilters] - pageSize: Optional[int] = None - - -class Notification(BaseModel): - """Notification schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - startDatetime: Optional[datetime] - endDateTime: Optional[datetime] - maintenanceType: Optional[str] - status: Optional[str] - updatedBy: datetime - message: Optional[str] - - -class Organization(BaseModel): - """Organization schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - acronym: Optional[str] - name: str - rootDomains: str - ipBlocks: str - isPassive: bool - pendingDomains: str - country: Optional[str] - state: Optional[str] - regionId: Optional[str] - stateFips: Optional[int] - stateName: Optional[str] - county: Optional[str] - countyFips: Optional[int] - type: Optional[str] - - -class Question(BaseModel): - """Question schema.""" - - id: UUID - name: str - description: str - longForm: str - number: str - categoryId: Optional[Any] - - -class Resource(BaseModel): - """Resource schema.""" - - id: UUID - description: str - name: str - type: str - url: str - - -class Response(BaseModel): - """Response schema.""" - - id: UUID - selection: str - assessmentId: Optional[Any] - questionId: Optional[Any] - - -class Role(BaseModel): - """Role schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - role: str - approved: bool - createdById: Optional[Any] - approvedById: Optional[Any] - userId: Optional[Any] - organizationId: Optional[Any] - - -class Scan(BaseModel): - """Scan schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - name: str - arguments: Any - frequency: int - lastRun: Optional[datetime] - isGranular: bool - isUserModifiable: Optional[bool] - isSingleScan: bool - manualRunPending: bool - createdBy: Optional[Any] - - -class ScanTask(BaseModel): - """ScanTask schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - status: str - type: str - fargateTaskArn: Optional[str] - input: Optional[str] - output: Optional[str] - requestedAt: Optional[datetime] - startedAt: Optional[datetime] - finishedAt: Optional[datetime] - queuedAt: Optional[datetime] - organizationId: Optional[Any] - scanId: Optional[Any] - - -class SearchBody(BaseModel): - """SearchBody schema.""" - - current: int - resultsPerPage: int - searchTerm: str - sortDirection: str - sortField: str - filters: Json[Any] - organizationId: Optional[UUID] - tagId: Optional[UUID] - - -class Service(BaseModel): - """Service schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - serviceSource: Optional[str] - port: int - service: Optional[str] - lastSeen: Optional[datetime] - banner: Optional[str] - products: Json[Any] - censysMetadata: Json[Any] - censysIpv4Results: Json[Any] - shodanResults: Json[Any] - wappalyzerResults: Json[Any] - domainId: Optional[Any] - discoveredById: Optional[Any] - - -class User(BaseModel): - """User schema.""" - - id: UUID - cognitoId: Optional[str] - loginGovId: Optional[str] - createdAt: datetime - updatedAt: datetime - firstName: str - lastName: str - fullName: str - email: str - invitePending: bool - loginBlockedByMaintenance: bool - dateAcceptedTerms: Optional[datetime] - acceptedTermsVersion: Optional[str] - lastLoggedIn: Optional[datetime] - userType: str - regionId: Optional[str] - state: Optional[str] - oktaId: Optional[str] - roles: Optional[List[Role]] = [] - - @classmethod - def from_orm(cls, obj): - # Convert roles to a list of RoleSchema before passing to Pydantic - user_dict = obj.__dict__.copy() - user_dict["roles"] = [Role.from_orm(role) for role in obj.roles.all()] - return cls(**user_dict) - - class Config: - orm_mode = True - from_attributes = True - - -class Vulnerability(BaseModel): - """Vulnerability schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - lastSeen: datetime - title: Optional[str] - cve: Optional[str] - cwe: Optional[str] - cpe: Optional[str] - description: Optional[str] - references: Json[Any] - cvss: float - severity: Optional[str] - needsPopulation: bool - state: Optional[str] - substate: Optional[str] - source: Optional[str] - notes: Optional[str] - actions: Json[Any] - structuredData: Json[Any] - isKev: bool - domainId: UUID - serviceId: UUID - - -class VulnerabilityFilters(BaseModel): - """VulnerabilityFilters schema.""" - - id: Optional[UUID] - title: Optional[str] - domain: Optional[str] - severity: Optional[str] - cpe: Optional[str] - state: Optional[str] - substate: Optional[str] - organization: Optional[UUID] - tag: Optional[UUID] - isKev: Optional[bool] - - -class VulnerabilitySearch(BaseModel): - """VulnerabilitySearch schema.""" - - page: int - sort: Optional[str] - order: str - filters: Optional[VulnerabilityFilters] - pageSize: Optional[int] - groupBy: Optional[str] +from .schema_models.assessment import Assessment +from .schema_models.category import Category +from .schema_models.cpe import Cpe +from .schema_models.cve import Cve +from .schema_models.domain import Domain, DomainFilters, DomainSearch +from .schema_models.notification import Notification +from .schema_models.organization import Organization +from .schema_models.question import Question +from .schema_models.resource import Resource +from .schema_models.response import Response +from .schema_models.role import Role +from .schema_models.scan import Scan +from .schema_models.searchbody import SearchBody +from .schema_models.service import Service +from .schema_models.user import User +from .schema_models.vulnerability import ( + Vulnerability, + VulnerabilityFilters, + VulnerabilitySearch, +) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index d7400776..c675b1eb 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -25,11 +25,17 @@ from fastapi.security import APIKeyHeader, OAuth2PasswordBearer # from .schemas import Cpe -from . import schemas +from . import schema_models from .auth import get_current_active_user from .models import ApiKey, Cpe, Cve, Domain, Organization, Role, User, Vulnerability +from .schemas import Cpe as CpeSchema +from .schemas import Cve as CveSchema +from .schemas import Domain as DomainSchema +from .schemas import DomainFilters, DomainSearch +from .schemas import Organization as OrganizationSchema from .schemas import Role as RoleSchema from .schemas import User as UserSchema +from .schemas import Vulnerability as VulnerabilitySchema api_router = APIRouter() @@ -76,7 +82,7 @@ async def get_api_keys(): @api_router.post( "/test-orgs", dependencies=[Depends(get_current_active_user)], - response_model=List[schemas.Organization], + response_model=List[OrganizationSchema], tags=["List of all Organizations"], ) def read_orgs(current_user: User = Depends(get_current_active_user)): @@ -128,7 +134,7 @@ async def export_search(): @api_router.get( "/cpes/{cpe_id}", # dependencies=[Depends(get_current_active_user)], - response_model=schemas.Cpe, + response_model=CpeSchema, tags=["Get cpe by id"], ) async def get_cpes_by_id(cpe_id): @@ -147,7 +153,7 @@ async def get_cpes_by_id(cpe_id): @api_router.get( "/cves/{cve_id}", # dependencies=[Depends(get_current_active_user)], - response_model=schemas.Cve, + response_model=CveSchema, tags=["Get cve by id"], ) async def get_cves_by_id(cve_id): @@ -166,7 +172,7 @@ async def get_cves_by_id(cve_id): @api_router.get( "/cves/name/{cve_name}", # dependencies=[Depends(get_current_active_user)], - response_model=schemas.Cve, + response_model=CveSchema, tags=["Get cve by name"], ) async def get_cves_by_name(cve_name): @@ -183,7 +189,7 @@ async def get_cves_by_name(cve_name): @api_router.post("/domain/search") -async def search_domains(domain_search: schemas.DomainSearch): +async def search_domains(domain_search: DomainSearch): try: pass except Exception as e: @@ -201,7 +207,7 @@ async def export_domains(): @api_router.get( "/domain/{domain_id}", # dependencies=[Depends(get_current_active_user)], - response_model=List[schemas.Domain], + response_model=DomainSchema, tags=["Get domain by id"], ) async def get_domain_by_id(domain_id: str): @@ -211,9 +217,10 @@ async def get_domain_by_id(domain_id: str): object: a single Domain object. """ try: - domains = list(Domain.objects.filter(id=domain_id)) - - return domains + domain = Domain.objects.get(id=domain_id) + return domain + except Domain.DoesNotExist: + raise HTTPException(status_code=404, detail="Domain not found.") except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -237,7 +244,7 @@ async def export_vulnerabilities(): @api_router.get( "/vulnerabilities/{vulnerabilityId}", # dependencies=[Depends(get_current_active_user)], - response_model=schemas.Vulnerability, + response_model=VulnerabilitySchema, tags="Get vulnerability by id", ) async def get_vulnerability_by_id(vuln_id): @@ -256,10 +263,10 @@ async def get_vulnerability_by_id(vuln_id): @api_router.put( "/vulnerabilities/{vulnerabilityId}", # dependencies=[Depends(get_current_active_user)], - response_model=schemas.Vulnerability, + response_model=VulnerabilitySchema, tags="Update vulnerability", ) -async def update_vulnerability(vuln_id, data: schemas.Vulnerability): +async def update_vulnerability(vuln_id, data: VulnerabilitySchema): """ Update vulnerability by id. From 7420674cbfba7b3c39f71252c95ad68d39cb94b9 Mon Sep 17 00:00:00 2001 From: nickviola Date: Tue, 17 Sep 2024 16:26:54 -0500 Subject: [PATCH 021/314] Add base logic and endpoints for auth, api-keys, and user lookups via token header --- backend/src/xfd_django/xfd_api/auth.py | 146 ++++------ backend/src/xfd_django/xfd_api/jwt_utils.py | 2 +- backend/src/xfd_django/xfd_api/login_gov.py | 294 +++++++++++++------- backend/src/xfd_django/xfd_api/views.py | 212 +++++++++----- 4 files changed, 389 insertions(+), 265 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index bd041414..5cd203a2 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -23,11 +23,13 @@ from hashlib import sha256 import os import json +import uuid from urllib.parse import urlencode -from re import A # Third-Party Libraries from django.utils import timezone +from django.conf import settings +from django.forms.models import model_to_dict from fastapi import Depends, HTTPException, Security, status from fastapi.security import APIKeyHeader, OAuth2PasswordBearer import jwt @@ -41,6 +43,19 @@ oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) +JWT_SECRET = os.getenv("JWT_SECRET") + + +def user_to_dict(user): + user_dict = model_to_dict(user) # Convert model to dict + # Convert any UUID fields to strings + if isinstance(user_dict.get('id'), uuid.UUID): + user_dict['id'] = str(user_dict['id']) + for key, val in user_dict.items(): + if isinstance(val, datetime): + user_dict[key] = str(val) + return user_dict + async def update_or_create_user(user_info): try: @@ -150,20 +165,6 @@ def get_user_by_api_key(api_key: str): # Returns: # User: The authenticated user object. # """ -# user = None -# if api_key: -# user = get_user_by_api_key(api_key) -# # if not user and token: -# # user = decode_jwt_token(token) -# if user is None: -# print("User not authenticated") -# raise HTTPException( -# status_code=status.HTTP_401_UNAUTHORIZED, -# detail="Invalid authentication credentials", -# ) -# print(f"Authenticated user: {user.id}") -# return user - def get_current_active_user(token: str = Depends(oauth2_scheme)): try: print(f"Token received: {token}") @@ -204,73 +205,18 @@ def get_current_active_user(token: str = Depends(oauth2_scheme)): ) -# async def get_jwt_from_code(auth_code: str): -# domain = os.getenv("REACT_APP_COGNITO_DOMAIN") -# client_id = os.getenv("REACT_APP_COGNITO_CLIENT_ID") -# callback_url = os.getenv("REACT_APP_COGNITO_CALLBACK_URL") -# # callback_url = "http%3A%2F%2Flocalhost%2Fokta-callback" -# # scope = "openid email profile" -# scope = "openid" -# authorize_token_url = f"https://{domain}/oauth2/token" -# # logging.debug(f"Authorize token url: {authorize_token_url}") -# print(f"Authorize token url: {authorize_token_url}") -# # authorize_token_body = f"grant_type=authorization_code&client_id={client_id}&code={auth_code}&redirect_uri={callback_url}&scope={scope}" -# authorize_token_body = { -# "grant_type": "authorization_code", -# "client_id": client_id, -# "code": auth_code, -# "redirect_uri": callback_url, -# "scope": scope, -# } -# # logging.debug(f"Authorize token body: {authorize_token_body}") -# print(f"Authorize token body: {authorize_token_body}") - -# headers = { -# "Content-Type": "application/x-www-form-urlencoded", -# } - -# try: -# response = requests.post( -# authorize_token_url, headers=headers, data=authorize_token_body -# ) -# except Exception as e: -# print(f"requests error: {e}") -# token_response = response.json() -# print(f"oauth2/token response: {token_response}") - -# token_response = response.json() -# print(f"token response: {token_response}") -# id_token = token_response.get("id_token") -# print(f"ID token: {id_token}") -# # access_token = token_response.get("token_token") -# # refresh_token = token_response.get("refresh_token") -# decoded_token = jwt.decode(id_token, options={"verify_signature": False}) -# print(f"decoded token: {decoded_token}") -# # decoded_token = jwt.decode(id_token, algorithms=JWT_ALGORITHM, audience=client_id) -# # decoded_token = jwt.decode(id_token, algorithms=["RS256"], audience=client_id) -# # print(f"decoded token: {decoded_token}") -# return {json.dumps(decoded_token)} - -JWT_SECRET = os.getenv("JWT_SECRET") - - async def process_user(decoded_token, access_token, refresh_token): - # Connect to the database - # await connect_to_database() - # Find the user by email user = User.objects.filter(email=decoded_token["email"]).first() if not user: - # Create a new user if they don't exist + # Create a new user if they don't exist from Okta fields in SAML Response user = User( email=decoded_token["email"], - okta_id=decoded_token["sub"], # assuming oktaId is in decoded_token + okta_id=decoded_token["sub"], first_name=decoded_token.get("given_name"), last_name=decoded_token.get("family_name"), invite_pending=True, - # state="Virginia", # Hardcoded for now - # region_id="3" # Hardcoded region ) user.save() else: @@ -279,13 +225,13 @@ async def process_user(decoded_token, access_token, refresh_token): user.last_logged_in = datetime.now() user.save() - # Create response object - response = JSONResponse({"message": "User processed"}) + # # Create response object + # response = JSONResponse({"message": "User processed"}) - # Set cookies for access token and refresh token - response.set_cookie(key="access_token", value=access_token, httponly=True, secure=True) - response.set_cookie(key="refresh_token", value=refresh_token, httponly=True, secure=True) - print(f"Response output: {str(response.headers)}") + # # Set cookies for access token and refresh token + # response.set_cookie(key="access_token", value=access_token, httponly=True, secure=True) + # response.set_cookie(key="refresh_token", value=refresh_token, httponly=True, secure=True) + # print(f"Response output: {str(response.headers)}") # If user exists, generate a signed JWT token if user: @@ -300,22 +246,38 @@ async def process_user(decoded_token, access_token, refresh_token): ) # Set JWT token as a cookie - response.set_cookie(key="id_token", value=signed_token, httponly=True, secure=True) + # response.set_cookie(key="id_token", value=signed_token, httponly=True, secure=True) # Return the response with token and user info - return JSONResponse( - { - "token": signed_token, - "user": { - "id": str(user.id), - "email": user.email, - "firstName": user.firstName, - "lastName": user.lastName, - "state": user.state, - "regionId": user.regionId, - } - } - ) + # return JSONResponse( + # { + # "token": signed_token, + # "user": { + # "id": str(user.id), + # "email": user.email, + # "firstName": user.firstName, + # "lastName": user.lastName, + # "state": user.state, + # "regionId": user.regionId, + # } + # } + # ) + + process_resp = { + "token": signed_token, + "user": user_to_dict(user) + # "user": { + # "id": str(user.id), + # "email": user.email, + # "firstName": user.firstName, + # "lastName": user.lastName, + # "state": user.state, + # "regionId": user.regionId, + # } + } + print(f"Process resp: {process_resp}") + return process_resp + else: raise HTTPException(status_code=400, detail="User not found") diff --git a/backend/src/xfd_django/xfd_api/jwt_utils.py b/backend/src/xfd_django/xfd_api/jwt_utils.py index 6849fec1..be098859 100644 --- a/backend/src/xfd_django/xfd_api/jwt_utils.py +++ b/backend/src/xfd_django/xfd_api/jwt_utils.py @@ -39,7 +39,7 @@ def create_jwt_token(user): payload = { "id": str(user.id), "email": user.email, - "exp": datetime.utcnow() + timedelta(hours=1), + "exp": datetime.utcnow() + timedelta(hours=4), } return jwt.encode(payload, SECRET_KEY, algorithm="HS256") diff --git a/backend/src/xfd_django/xfd_api/login_gov.py b/backend/src/xfd_django/xfd_api/login_gov.py index 304e052a..f4b67275 100644 --- a/backend/src/xfd_django/xfd_api/login_gov.py +++ b/backend/src/xfd_django/xfd_api/login_gov.py @@ -1,128 +1,220 @@ -# Standard Python Libraries -import json import os - -# Third-Party Libraries -from authlib.integrations.requests_client import OAuth2Session - -# import time -# import secrets +import secrets import jwt import requests +import json -# from authlib.jose import jwt - - -# discovery_url = os.environ.get('LOGIN_GOV_BASE_URL') + '/.well-known/openid-configuration' - -# try: -# jwk_set = { -# "keys": [json.loads(os.environ.get('LOGIN_GOV_JWT_KEY', '{}'))] -# } -# except Exception: -# jwk_set = { -# "keys": [{}] -# } -# client_options = { -# 'client_id': os.environ.get('LOGIN_GOV_ISSUER'), -# 'token_endpoint_auth_method': 'private_key_jwt', -# 'id_token_signed_response_alg': 'RS256', -# 'key': 'client_id', -# 'redirect_uris': [os.environ.get('LOGIN_GOV_REDIRECT_URI')], -# 'token_endpoint': os.environ.get('LOGIN_GOV_BASE_URL') + '/api/openid_connect/token' -# } discovery_url = os.getenv("LOGIN_GOV_BASE_URL") + "/.well-known/openid-configuration" + +# Load JWK Set (JSON Web Key Set) +try: + jwk_set = { + "keys": [json.loads(os.getenv("LOGIN_GOV_JWT_KEY", "{}"))] + } +except Exception as e: + print(f"Error: {e}") + jwk_set = { + "keys": [{}] + } + +# OpenID Connect Client Configuration client_options = { "client_id": os.getenv("LOGIN_GOV_ISSUER"), "token_endpoint_auth_method": "private_key_jwt", "id_token_signed_response_alg": "RS256", - "key": "client_id", "redirect_uris": [os.getenv("LOGIN_GOV_REDIRECT_URI")], "token_endpoint": os.getenv("LOGIN_GOV_BASE_URL") + "/api/openid_connect/token", } -jwk_set = {"keys": [json.loads(os.getenv("LOGIN_GOV_JWT_KEY", "{}"))]} +# Generate random string for nonce and state +def random_string(length): + return secrets.token_hex(length // 2) + + +# 1. Login function that returns authorization URL, state, and nonce +def login(): + """Equivalent function to initiate OpenID Connect login.""" + # Fetch OpenID Connect configuration + config_response = requests.get(discovery_url) + config = config_response.json() + + nonce = random_string(32) + state = random_string(32) + + # Create authorization URL + authorization_url = ( + f"{config['authorization_endpoint']}?response_type=code" + f"&client_id={client_options['client_id']}" + f"&redirect_uri={client_options['redirect_uris'][0]}" + f"&scope=openid+email" + f"&nonce={nonce}&state={state}&prompt=select_account" + ) + + return {"url": authorization_url, "state": state, "nonce": nonce} + + +# 2. Callback function to exchange authorization code for tokens and user info +def callback(body): + """Equivalent function to handle OpenID Connect callback.""" + config_response = requests.get(discovery_url) + config = config_response.json() + + # Exchange the authorization code for tokens + token_response = requests.post( + config["token_endpoint"], + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data={ + "grant_type": "authorization_code", + "code": body["code"], + "client_id": client_options["client_id"], + "redirect_uri": client_options["redirect_uris"][0], + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": jwt.encode( + {"alg": "RS256"}, jwk_set["keys"][0], algorithm="RS256" + ) + } + ) + + token_response_data = token_response.json() + + if "id_token" not in token_response_data: + raise Exception("ID token not found in the token response") + + id_token = token_response_data["id_token"] + + # Decode the ID token without verifying the signature (optional depending on your security model) + decoded_token = jwt.decode(id_token, options={"verify_signature": False}) + return decoded_token + + +# V1 Tests TODO: Cleanup code +# # Standard Python Libraries +# import json +# import os + +# # Third-Party Libraries +# from authlib.integrations.requests_client import OAuth2Session + +# # import time +# # import secrets +# import jwt +# import requests -# def random_string(length): -# return secrets.token_hex(length // 2) +# # from authlib.jose import jwt + + +# # discovery_url = os.environ.get('LOGIN_GOV_BASE_URL') + '/.well-known/openid-configuration' + +# # try: +# # jwk_set = { +# # "keys": [json.loads(os.environ.get('LOGIN_GOV_JWT_KEY', '{}'))] +# # } +# # except Exception: +# # jwk_set = { +# # "keys": [{}] +# # } + +# # client_options = { +# # 'client_id': os.environ.get('LOGIN_GOV_ISSUER'), +# # 'token_endpoint_auth_method': 'private_key_jwt', +# # 'id_token_signed_response_alg': 'RS256', +# # 'key': 'client_id', +# # 'redirect_uris': [os.environ.get('LOGIN_GOV_REDIRECT_URI')], +# # 'token_endpoint': os.environ.get('LOGIN_GOV_BASE_URL') + '/api/openid_connect/token' +# # } +# discovery_url = os.getenv("LOGIN_GOV_BASE_URL") + "/.well-known/openid-configuration" +# client_options = { +# "client_id": os.getenv("LOGIN_GOV_ISSUER"), +# "token_endpoint_auth_method": "private_key_jwt", +# "id_token_signed_response_alg": "RS256", +# "key": "client_id", +# "redirect_uris": [os.getenv("LOGIN_GOV_REDIRECT_URI")], +# "token_endpoint": os.getenv("LOGIN_GOV_BASE_URL") + "/api/openid_connect/token", +# } + +# jwk_set = {"keys": [json.loads(os.getenv("LOGIN_GOV_JWT_KEY", "{}"))]} + + +# # def random_string(length): +# # return secrets.token_hex(length // 2) + + +# # async def login(): +# # discovery_doc = await get_well_known_config(discovery_url) +# # client = OAuth2Session(client_id=client_options['client_id']) +# # nonce = random_string(32) +# # state = random_string(32) +# # url = client.create_authorization_url( +# # discovery_doc['authorization_endpoint'], +# # response_type='code', +# # acr_values='http://idmanagement.gov/ns/assurance/ial/1', +# # scope='openid email', +# # redirect_uri=client_options['redirect_uris'][0], +# # nonce=nonce, +# # state=state, +# # prompt='select_account' +# # )[0] +# # return {"url": url, "state": state, "nonce": nonce} + + +# # async def callback(body): +# # discovery_doc = await get_well_known_config(discovery_url) +# # client = OAuth2Session(client_id=client_options['client_id']) + +# # private_key = os.getenv('PRIVATE_KEY') +# # client_assertion = jwt.encode( +# # {'alg': 'RS256'}, +# # {'iss': client_options['client_id'], 'sub': client_options['client_id'], 'aud': discovery_doc['token_endpoint'], 'exp': int(time.time()) + 300}, +# # private_key +# # ) + +# # token_response = client.fetch_token( +# # discovery_doc['token_endpoint'], +# # code=body['code'], +# # client_assertion_type='urn:ietf:params:oauth:client-assertion-type:jwt-bearer', +# # client_assertion=client_assertion +# # ) + + +# # # Make a request to the userinfo endpoint +# # userinfo_response = client.get(discovery_doc['userinfo_endpoint'], headers={'Authorization': f"Bearer {token_response['access_token']}"}) +# # user_info = userinfo_response.json() +# # return user_info +# async def get_discovery_doc(discovery_url): +# response = requests.get(discovery_url) +# response.raise_for_status() +# return response.json() # async def login(): -# discovery_doc = await get_well_known_config(discovery_url) -# client = OAuth2Session(client_id=client_options['client_id']) -# nonce = random_string(32) -# state = random_string(32) -# url = client.create_authorization_url( -# discovery_doc['authorization_endpoint'], -# response_type='code', -# acr_values='http://idmanagement.gov/ns/assurance/ial/1', -# scope='openid email', -# redirect_uri=client_options['redirect_uris'][0], +# discovery_doc = await get_discovery_doc(discovery_url) +# client = OAuth2Session(client_id=client_options["client_id"]) +# nonce = os.urandom(16).hex() +# state = os.urandom(16).hex() +# authorization_url, state = client.create_authorization_url( +# discovery_doc["authorization_endpoint"], +# response_type="code", +# scope="openid email", +# redirect_uri=client_options["redirect_uris"][0], # nonce=nonce, # state=state, -# prompt='select_account' -# )[0] -# return {"url": url, "state": state, "nonce": nonce} +# prompt="select_account", +# ) +# return {"url": authorization_url, "state": state, "nonce": nonce} # async def callback(body): -# discovery_doc = await get_well_known_config(discovery_url) -# client = OAuth2Session(client_id=client_options['client_id']) - -# private_key = os.getenv('PRIVATE_KEY') -# client_assertion = jwt.encode( -# {'alg': 'RS256'}, -# {'iss': client_options['client_id'], 'sub': client_options['client_id'], 'aud': discovery_doc['token_endpoint'], 'exp': int(time.time()) + 300}, -# private_key -# ) - +# discovery_doc = await get_discovery_doc(discovery_url) +# client = OAuth2Session(client_id=client_options["client_id"]) # token_response = client.fetch_token( -# discovery_doc['token_endpoint'], -# code=body['code'], -# client_assertion_type='urn:ietf:params:oauth:client-assertion-type:jwt-bearer', -# client_assertion=client_assertion +# discovery_doc["token_endpoint"], +# code=body["code"], +# client_assertion_type="urn:ietf:params:oauth:client-assertion-type:jwt-bearer", +# client_assertion=jwt.encode({"alg": "RS256"}, jwk_set, "private"), # ) - - -# # Make a request to the userinfo endpoint -# userinfo_response = client.get(discovery_doc['userinfo_endpoint'], headers={'Authorization': f"Bearer {token_response['access_token']}"}) -# user_info = userinfo_response.json() +# user_info = client.get( +# discovery_doc["userinfo_endpoint"], token=token_response["access_token"] +# ).json() # return user_info -async def get_discovery_doc(discovery_url): - response = requests.get(discovery_url) - response.raise_for_status() - return response.json() - - -async def login(): - discovery_doc = await get_discovery_doc(discovery_url) - client = OAuth2Session(client_id=client_options["client_id"]) - nonce = os.urandom(16).hex() - state = os.urandom(16).hex() - authorization_url, state = client.create_authorization_url( - discovery_doc["authorization_endpoint"], - response_type="code", - scope="openid email", - redirect_uri=client_options["redirect_uris"][0], - nonce=nonce, - state=state, - prompt="select_account", - ) - return {"url": authorization_url, "state": state, "nonce": nonce} - - -async def callback(body): - discovery_doc = await get_discovery_doc(discovery_url) - client = OAuth2Session(client_id=client_options["client_id"]) - token_response = client.fetch_token( - discovery_doc["token_endpoint"], - code=body["code"], - client_assertion_type="urn:ietf:params:oauth:client-assertion-type:jwt-bearer", - client_assertion=jwt.encode({"alg": "RS256"}, jwk_set, "private"), - ) - user_info = client.get( - discovery_doc["userinfo_endpoint"], token=token_response["access_token"] - ).json() - return user_info diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 8ac0edcd..0b894779 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -17,10 +17,17 @@ - .models """ # Third-Party Libraries -from fastapi import APIRouter, Depends, HTTPException, Request, status +import hashlib +from http.client import HTTPResponse +import json +import secrets +import uuid +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import JSONResponse -from .auth import get_current_active_user, get_jwt_from_code, handle_cognito_callback, process_user +from .login_gov import login + +from .auth import get_current_active_user, get_jwt_from_code, process_user from .models import ApiKey, Organization, User api_router = APIRouter() @@ -50,34 +57,8 @@ async def healthcheck(): return {"status": "ok2"} -@api_router.get("/test-apikeys") -async def get_api_keys(): - """ - Get all API keys. - - Returns: - list: A list of all API keys. - """ - try: - api_keys = ApiKey.objects.all() - # return api_keys - return [ - { - "id": api_key.id, - "created_at": api_key.createdAt, - "updated_at": api_key.updatedAt, - "last_used": api_key.lastUsed, - "hashed_key": api_key.hashedKey, - "last_four": api_key.lastFour, - "user_id": api_key.userId.id if api_key.userId else None, - } - for api_key in api_keys - ] - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@api_router.post( +# Endpoint to read organizations +@api_router.get( "/test-orgs", dependencies=[Depends(get_current_active_user)], tags=["List of all Organizations"], @@ -117,17 +98,33 @@ def read_orgs(current_user: User = Depends(get_current_active_user)): raise HTTPException(status_code=500, detail=str(e)) -@api_router.post("/auth/login") -async def login_endpoint(): - print(f"Returning auth/login response") - # result = await get_login_gov() - # return JSONResponse(content=result) +# @api_router.post("/auth/login") +# async def login_endpoint(): +# print(f"Returning auth/login response") +# # result = await get_login_gov() +# # return JSONResponse(content=result) + + +@api_router.get("/login") +async def login_route(): + login_data = login() + return login_data @api_router.post("/auth/callback") -async def callback_endpoint(request: Request): - body = request.json() - print(f"body: {body}") +async def callback_route(request: Request): + body = await request.json() + try: + user_info = callback(body) + return user_info + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + + +# @api_router.post("/auth/callback") +# async def callback_endpoint(request: Request): +# body = request.json() +# print(f"body: {body}") # try: # if os.getenv("USE_COGNITO"): # token, user = await handle_cognito_callback(body) @@ -142,6 +139,8 @@ async def callback_endpoint(request: Request): # ) from error + + @api_router.post("/auth/okta-callback") async def callback(request: Request): print(f"Request from /auth/okta-callback: {str(request)}") @@ -151,11 +150,11 @@ async def callback(request: Request): print(f"Body type: {type(body)}") code = body.get("code") print(f"Code: {code}") - # if not code: - # return HTTPException( - # status_code=status.HTTP_400_BAD_REQUEST, - # detail="Code not found in request body", - # ) + if not code: + return HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Code not found in request body", + ) jwt_data = await get_jwt_from_code(code) print(f"JWT Data: {jwt_data}") if jwt_data is None: @@ -167,34 +166,28 @@ async def callback(request: Request): refresh_token = jwt_data.get("refresh_token") decoded_token = jwt_data.get("decoded_token") - return await process_user(decoded_token, access_token, refresh_token) - # try: - # token_endpoint = f"https://{domain}/oauth2/token" - # token_data = ( - # f"grant_type=authorization_code&client_id={client_id}&code={code}" - # f"&redirect_uri={callback_url}&scope=openid" - # ) - - # response = requests.post( - # token_endpoint, - # headers={'Content-Type': 'application/x-www-form-urlencoded'}, - # data=token_data - # ) - - # # Assuming the response is in JSON format - # response_json = response.json() - # id_token = response_json.get('id_token') - # access_token = response_json.get('access_token') - # refresh_token = response_json.get('refresh_token') - - # print(f"id_token: {id_token}") - # print(f"access_token: {access_token}") - # print(f"refresh_token: {refresh_token}") - - # # return JSONResponse(content={"token": access_token, "user": user}, status_code=status.HTTP_200_OK) - # return response_json - # except HTTPException as e: - # raise HTTPException(status_code=e.status_code, detail=e.detail) + resp = await process_user(decoded_token, access_token, refresh_token) + token = resp.get("token") + + # print(f"Response from process_user: {json.dumps(resp)}") + # Response(status_code=200, set("crossfeed-token", resp.get("token")) + # return json.dumps(resp) + # Create a JSONResponse object to return the response and set the cookie + response = JSONResponse(content={"message": "User authenticated", "data": resp, "token": token}) + response.body = resp + # response.body = resp + response.set_cookie(key="token", value=token) + + # Set the 'crossfeed-token' cookie + response.set_cookie( + key="crossfeed-token", + value=token, + # httponly=True, # This makes the cookie inaccessible to JavaScript + # secure=True, # Ensures the cookie is only sent over HTTPS + # samesite="Lax" # Restricts when cookies are sent, adjust as necessary (e.g., "Strict" or "None") + ) + + return response # @api_router.get("/users/me", response_model=User) @@ -231,3 +224,80 @@ async def read_users_me(current_user: User = Depends(get_current_active_user)): # current_user.accepted_terms_version = version # current_user.save() # return current_user + +# Helper function to hash the API key +def hash_key(key: str) -> str: + return hashlib.sha256(key.encode()).hexdigest() + + +# POST /apikeys/generate +@api_router.post("/api-keys") +async def generate_api_key(current_user: User = Depends(get_current_active_user)): + # Generate a random 16-byte API key + key = secrets.token_hex(16) + + # Hash the API key + hashed_key = hash_key(key) + + # Create the ApiKey record in the database + api_key = ApiKey.objects.create( + id=uuid.uuid4(), + hashedKey=hashed_key, + lastFour=key[-4:], # Store the last four characters of the key + userId=current_user + ) + + # Return the API key to the user (Do NOT store the plain key in the database) + return { + "id": api_key.id, + "status": "success", + "api_key": key, + "last_four": api_key.lastFour + } + + +# DELETE /apikeys/{keyId} +@api_router.delete("/api-keys/{key_id}") +async def delete_api_key(key_id: str, current_user: User = Depends(get_current_active_user)): + try: + # Validate that key_id is a valid UUID + uuid.UUID(key_id) + except ValueError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Invalid API Key ID") + + # Try to find the ApiKey by key_id and current user + try: + api_key = ApiKey.objects.get(id=key_id, userId=current_user) + except ApiKey.DoesNotExist: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="API Key not found") + + # Delete the API Key + api_key.delete() + return {"status": "success", "message": "API Key deleted successfully"} + + +@api_router.get("/api-keys") +async def get_api_keys(): + """ + Get all API keys. + + Returns: + list: A list of all API keys. + """ + try: + api_keys = ApiKey.objects.all() + # return api_keys + return [ + { + "id": api_key.id, + "created_at": api_key.createdAt, + "updated_at": api_key.updatedAt, + "last_used": api_key.lastUsed, + "hashed_key": api_key.hashedKey, + "last_four": api_key.lastFour, + "user_id": api_key.userId.id if api_key.userId else None, + } + for api_key in api_keys + ] + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) From e1428fa9f1176e2ea5b562a7ac00f4452b6568ac Mon Sep 17 00:00:00 2001 From: nickviola Date: Tue, 17 Sep 2024 16:29:46 -0500 Subject: [PATCH 022/314] Update frontend OktaCallback component and api calls to work with new python endpoints --- .../src/pages/OktaCallback/OktaCallback.tsx | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/frontend/src/pages/OktaCallback/OktaCallback.tsx b/frontend/src/pages/OktaCallback/OktaCallback.tsx index f8a9c421..631d0833 100644 --- a/frontend/src/pages/OktaCallback/OktaCallback.tsx +++ b/frontend/src/pages/OktaCallback/OktaCallback.tsx @@ -4,10 +4,11 @@ import { useAuthContext } from 'context'; import { User } from 'types'; import { useHistory } from 'react-router-dom'; -type OktaCallbackResponse = { - token: string; - user: User; -}; +type OktaCallbackResponse = any; +// { +// body: any; +// // user: User; +// }; export const OktaCallback: React.FC = () => { const { apiPost, login } = useAuthContext(); @@ -16,8 +17,6 @@ export const OktaCallback: React.FC = () => { const handleOktaCallback = useCallback(async () => { const { code } = parse(window.location.search); console.log('Code: ', code); - const nonce = localStorage.getItem('nonce'); - console.log('Nonce: ', nonce); try { // Pass request to backend callback endpoint @@ -26,17 +25,21 @@ export const OktaCallback: React.FC = () => { { body: { code: code + }, + headers: { + 'Content-Type': 'application/json' } } ); - console.log('Response: ', response); - console.log('token ', response.token); + console.log('Response: ', response.body); + console.log('token ', response.body.token); + const nonce = localStorage.getItem('nonce'); + console.log('Nonce: ', nonce); // Login - await login(response.token); - + login(response.token); // Storage Management - localStorage.setItem('token', response.token); + localStorage.setItem('token', response.body.token); localStorage.removeItem('nonce'); localStorage.removeItem('state'); From 480d45919c1e414fbf61682aa4eab4e91750f50b Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 24 Sep 2024 13:18:47 -0500 Subject: [PATCH 023/314] Refactor django xfd python backend to have views api methods split into their own separate files for ease of maintenance. --- .../xfd_api/api_methods/api_keys.py | 35 ++++ .../src/xfd_django/xfd_api/api_methods/cpe.py | 22 +++ .../src/xfd_django/xfd_api/api_methods/cve.py | 35 ++++ .../xfd_django/xfd_api/api_methods/domain.py | 24 +++ .../xfd_api/api_methods/organization.py | 85 ++++++++ .../xfd_django/xfd_api/api_methods/user.py | 55 ++++++ .../xfd_api/api_methods/vulnerability.py | 39 ++++ backend/src/xfd_django/xfd_api/views.py | 186 +++++++----------- 8 files changed, 363 insertions(+), 118 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/api_methods/api_keys.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/cpe.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/cve.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/domain.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/organization.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/user.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/vulnerability.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/api_keys.py b/backend/src/xfd_django/xfd_api/api_methods/api_keys.py new file mode 100644 index 00000000..6ec614b0 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/api_keys.py @@ -0,0 +1,35 @@ +""" +Api-keys API. + +""" + +# Third-Party Libraries +from fastapi import HTTPException + +from ..models import ApiKey + + +def get_api_keys(): + """ + Get all API keys. + + Returns: + list: A list of all API keys. + """ + try: + api_keys = ApiKey.objects.all() + # return api_keys + return [ + { + "id": api_key.id, + "createdAt": api_key.createdAt, + "updatedAt": api_key.updatedAt, + "lastUsed": api_key.lastUsed, + "hashedKey": api_key.hashedKey, + "lastFour": api_key.lastFour, + "userId": api_key.userId.id if api_key.userId else None, + } + for api_key in api_keys + ] + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/cpe.py b/backend/src/xfd_django/xfd_api/api_methods/cpe.py new file mode 100644 index 00000000..41bc9238 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/cpe.py @@ -0,0 +1,22 @@ +""" +Cpe API. + +""" + +# Third-Party Libraries +from fastapi import HTTPException + +from ..models import Cpe + + +def get_cpes_by_id(cpe_id): + """ + Get Cpe by id. + Returns: + object: a single Cpe object. + """ + try: + cpe = Cpe.objects.get(id=cpe_id) + return cpe + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/cve.py b/backend/src/xfd_django/xfd_api/api_methods/cve.py new file mode 100644 index 00000000..b31971de --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/cve.py @@ -0,0 +1,35 @@ +""" +Cve API. + +""" + +# Third-Party Libraries +from fastapi import HTTPException + +from ..models import Cve + + +def get_cves_by_id(cve_id): + """ + Get Cve by id. + Returns: + object: a single Cve object. + """ + try: + cve = Cve.objects.get(id=cve_id) + return cve + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def get_cves_by_name(cve_name): + """ + Get Cve by name. + Returns: + object: a single Cpe object. + """ + try: + cve = Cve.objects.get(name=cve_name) + return cve + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/domain.py b/backend/src/xfd_django/xfd_api/api_methods/domain.py new file mode 100644 index 00000000..92a0c1dd --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/domain.py @@ -0,0 +1,24 @@ +""" +Domain API. + +""" + +# Third-Party Libraries +from fastapi import HTTPException + +from ..models import Domain + + +def get_domain_by_id(domain_id: str): + """ + Get domain by id. + Returns: + object: a single Domain object. + """ + try: + domain = Domain.objects.get(id=domain_id) + return domain + except Domain.DoesNotExist: + raise HTTPException(status_code=404, detail="Domain not found.") + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/organization.py b/backend/src/xfd_django/xfd_api/api_methods/organization.py new file mode 100644 index 00000000..19c522d5 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/organization.py @@ -0,0 +1,85 @@ +""" +Organizations API. + +""" +# Standard Python Libraries +from typing import List, Optional + +# Third-Party Libraries +from fastapi import HTTPException, Query + +from ..models import Organization +from ..schemas import Organization as OrganizationSchema + + +def read_orgs(): + """ + Call API endpoint to get all organizations. + Returns: + list: A list of all organizations. + """ + try: + organizations = Organization.objects.all() + return [ + { + "id": organization.id, + "name": organization.name, + "acronym": organization.acronym, + "rootDomains": organization.rootDomains, + "ipBlocks": organization.ipBlocks, + "isPassive": organization.isPassive, + "country": organization.country, + "state": organization.state, + "regionId": organization.regionId, + "stateFips": organization.stateFips, + "stateName": organization.stateName, + "county": organization.county, + "countyFips": organization.countyFips, + "type": organization.type, + "parentId": organization.parentId.id if organization.parentId else None, + "createdById": organization.createdById.id + if organization.createdById + else None, + "createdAt": organization.createdAt, + "updatedAt": organization.updatedAt, + } + for organization in organizations + ] + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def get_organizations( + state: Optional[List[str]] = Query(None), + regionId: Optional[List[str]] = Query(None), +): + """ + List all organizations with query parameters. + Args: + state (Optional[List[str]]): List of states to filter organizations by. + regionId (Optional[List[str]]): List of region IDs to filter organizations by. + + Raises: + HTTPException: If the user is not authorized or no organizations are found. + + Returns: + List[Organizations]: A list of organizations matching the filter criteria. + """ + + # if not current_user: + # raise HTTPException(status_code=401, detail="Unauthorized") + + # Prepare filter parameters + filter_params = {} + if state: + filter_params["state__in"] = state + if regionId: + filter_params["regionId__in"] = regionId + + organizations = Organization.objects.filter(**filter_params) + + if not organizations.exists(): + raise HTTPException(status_code=404, detail="No organizations found") + + # Return the Pydantic models directly by calling from_orm + return organizations diff --git a/backend/src/xfd_django/xfd_api/api_methods/user.py b/backend/src/xfd_django/xfd_api/api_methods/user.py new file mode 100644 index 00000000..260b8849 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/user.py @@ -0,0 +1,55 @@ +""" +User API. + +""" +# Standard Python Libraries +from typing import List, Optional + +# Third-Party Libraries +from fastapi import HTTPException, Query + +from ..models import User +from ..schemas import User as UserSchema + + +def get_users( + state: Optional[List[str]] = Query(None), + regionId: Optional[List[str]] = Query(None), + invitePending: Optional[List[str]] = Query(None), + # current_user: User = Depends(is_regional_admin) +): + """ + Retrieve a list of users based on optional filter parameters. + + Args: + state (Optional[List[str]]): List of states to filter users by. + regionId (Optional[List[str]]): List of region IDs to filter users by. + invitePending (Optional[List[str]]): List of invite pending statuses to filter users by. + current_user (User): The current authenticated user, must be a regional admin. + + Raises: + HTTPException: If the user is not authorized or no users are found. + + Returns: + List[User]: A list of users matching the filter criteria. + """ + # if not current_user: + # raise HTTPException(status_code=401, detail="Unauthorized") + + # Prepare filter parameters + filter_params = {} + if state: + filter_params["state__in"] = state + if regionId: + filter_params["regionId__in"] = regionId + if invitePending: + filter_params["invitePending__in"] = invitePending + + # Query users with filter parameters and prefetch related roles + users = User.objects.filter(**filter_params).prefetch_related("roles") + + if not users.exists(): + raise HTTPException(status_code=404, detail="No users found") + + # Return the Pydantic models directly by calling from_orm + return [UserSchema.from_orm(user) for user in users] diff --git a/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py b/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py new file mode 100644 index 00000000..88ba2c2c --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py @@ -0,0 +1,39 @@ +""" +Vulnerability API. + +""" + +# Third-Party Libraries +from fastapi import HTTPException + +from ..models import Vulnerability +from ..schemas import Vulnerability as VulnerabilitySchema + + +def get_vulnerability_by_id(vuln_id): + """ + Get vulnerability by id. + Returns: + object: a single Vulnerability object. + """ + try: + vulnerability = Vulnerability.objects.get(id=vuln_id) + return vulnerability + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def update_vulnerability(vuln_id, data: VulnerabilitySchema): + """ + Update vulnerability by id. + + Returns: + object: a single vulnerability object that has been modified. + """ + try: + vulnerability = Vulnerability.objects.get(id=vuln_id) + vulnerability = data + vulnerability.save() + return vulnerability + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index c675b1eb..ce220a8d 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -26,6 +26,13 @@ # from .schemas import Cpe from . import schema_models +from .api_methods.api_keys import get_api_keys +from .api_methods.cpe import get_cpes_by_id +from .api_methods.cve import get_cves_by_id, get_cves_by_name +from .api_methods.domain import get_domain_by_id +from .api_methods.organization import get_organizations, read_orgs +from .api_methods.user import get_users +from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user from .models import ApiKey, Cpe, Cve, Domain, Organization, Role, User, Vulnerability from .schemas import Cpe as CpeSchema @@ -41,7 +48,7 @@ # Healthcheck endpoint -@api_router.get("/healthcheck") +@api_router.get("/healthcheck", tags=["Testing"]) async def healthcheck(): """ Healthcheck endpoint. @@ -52,73 +59,37 @@ async def healthcheck(): return {"status": "ok2"} -@api_router.get("/test-apikeys") -async def get_api_keys(): +@api_router.get("/test-apikeys", tags=["Testing"]) +async def call_get_api_keys(): """ Get all API keys. Returns: list: A list of all API keys. """ - try: - api_keys = ApiKey.objects.all() - # return api_keys - return [ - { - "id": api_key.id, - "createdAt": api_key.createdAt, - "updatedAt": api_key.updatedAt, - "lastUsed": api_key.lastUsed, - "hashedKey": api_key.hashedKey, - "lastFour": api_key.lastFour, - "userId": api_key.userId.id if api_key.userId else None, - } - for api_key in api_keys - ] - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return get_api_keys() @api_router.post( "/test-orgs", - dependencies=[Depends(get_current_active_user)], + # dependencies=[Depends(get_current_active_user)], response_model=List[OrganizationSchema], - tags=["List of all Organizations"], + tags=["Organizations", "Testing"], ) -def read_orgs(current_user: User = Depends(get_current_active_user)): - """Call API endpoint to get all organizations. +async def call_read_orgs(): + """ + List all organizations with query parameters. + Args: + state (Optional[List[str]]): List of states to filter organizations by. + regionId (Optional[List[str]]): List of region IDs to filter organizations by. + + Raises: + HTTPException: If the user is not authorized or no organizations are found. + Returns: - list: A list of all organizations. + List[Organizations]: A list of organizations matching the filter criteria. """ - try: - organizations = Organization.objects.all() - return [ - { - "id": organization.id, - "name": organization.name, - "acronym": organization.acronym, - "rootDomains": organization.rootDomains, - "ipBlocks": organization.ipBlocks, - "isPassive": organization.isPassive, - "country": organization.country, - "state": organization.state, - "regionId": organization.regionId, - "stateFips": organization.stateFips, - "stateName": organization.stateName, - "county": organization.county, - "countyFips": organization.countyFips, - "type": organization.type, - "parentId": organization.parentId.id if organization.parentId else None, - "createdById": organization.createdById.id - if organization.createdById - else None, - "createdAt": organization.createdAt, - "updatedAt": organization.updatedAt, - } - for organization in organizations - ] - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return read_orgs() @api_router.post("/search") @@ -135,38 +106,30 @@ async def export_search(): "/cpes/{cpe_id}", # dependencies=[Depends(get_current_active_user)], response_model=CpeSchema, - tags=["Get cpe by id"], + tags=["Cpe"], ) -async def get_cpes_by_id(cpe_id): +async def call_get_cpes_by_id(cpe_id): """ Get Cpe by id. Returns: object: a single Cpe object. """ - try: - cpe = Cpe.objects.get(id=cpe_id) - return cpe - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return get_cpes_by_id(cpe_id) @api_router.get( "/cves/{cve_id}", # dependencies=[Depends(get_current_active_user)], response_model=CveSchema, - tags=["Get cve by id"], + tags=["Cve"], ) -async def get_cves_by_id(cve_id): +async def call_get_cves_by_id(cve_id): """ Get Cve by id. Returns: object: a single Cve object. """ - try: - cve = Cve.objects.get(id=cve_id) - return cve - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return get_cves_by_id(cve_id) @api_router.get( @@ -175,17 +138,13 @@ async def get_cves_by_id(cve_id): response_model=CveSchema, tags=["Get cve by name"], ) -async def get_cves_by_name(cve_name): +async def call_get_cves_by_name(cve_name): """ Get Cve by name. Returns: object: a single Cpe object. """ - try: - cve = Cve.objects.get(name=cve_name) - return cve - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return get_cves_by_name(cve_name) @api_router.post("/domain/search") @@ -210,19 +169,13 @@ async def export_domains(): response_model=DomainSchema, tags=["Get domain by id"], ) -async def get_domain_by_id(domain_id: str): +async def call_get_domain_by_id(domain_id: str): """ Get domain by id. Returns: object: a single Domain object. """ - try: - domain = Domain.objects.get(id=domain_id) - return domain - except Domain.DoesNotExist: - raise HTTPException(status_code=404, detail="Domain not found.") - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return get_domain_by_id(domain_id) @api_router.post("/vulnerabilities/search") @@ -247,17 +200,13 @@ async def export_vulnerabilities(): response_model=VulnerabilitySchema, tags="Get vulnerability by id", ) -async def get_vulnerability_by_id(vuln_id): +async def call_get_vulnerability_by_id(vuln_id): """ Get vulnerability by id. Returns: object: a single Vulnerability object. """ - try: - vulnerability = Vulnerability.objects.get(id=vuln_id) - return vulnerability - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return get_vulnerability_by_id(vuln_id) @api_router.put( @@ -266,35 +215,30 @@ async def get_vulnerability_by_id(vuln_id): response_model=VulnerabilitySchema, tags="Update vulnerability", ) -async def update_vulnerability(vuln_id, data: VulnerabilitySchema): +async def call_update_vulnerability(vuln_id, data: VulnerabilitySchema): """ Update vulnerability by id. Returns: object: a single vulnerability object that has been modified. """ - try: - vulnerability = Vulnerability.objects.get(id=vuln_id) - vulnerability = data - vulnerability.save() - return vulnerability - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return update_vulnerability(vuln_id, data) @api_router.get( "/v2/users", response_model=List[UserSchema], # dependencies=[Depends(get_current_active_user)], + tags=["User"], ) -async def get_users( +async def call_get_users( state: Optional[List[str]] = Query(None), regionId: Optional[List[str]] = Query(None), invitePending: Optional[List[str]] = Query(None), # current_user: User = Depends(is_regional_admin) ): """ - Retrieve a list of users based on optional filter parameters. + Call get_users() Args: state (Optional[List[str]]): List of states to filter users by. @@ -308,23 +252,29 @@ async def get_users( Returns: List[User]: A list of users matching the filter criteria. """ - # if not current_user: - # raise HTTPException(status_code=401, detail="Unauthorized") - - # Prepare filter parameters - filter_params = {} - if state: - filter_params["state__in"] = state - if regionId: - filter_params["regionId__in"] = regionId - if invitePending: - filter_params["invitePending__in"] = invitePending - - # Query users with filter parameters and prefetch related roles - users = User.objects.filter(**filter_params).prefetch_related("roles") - - if not users.exists(): - raise HTTPException(status_code=404, detail="No users found") - - # Return the Pydantic models directly by calling from_orm - return [UserSchema.from_orm(user) for user in users] + return get_users(state, regionId, invitePending) + + +@api_router.get( + "/organizations", + # response_model=List[OrganizationSchema], + # dependencies=[Depends(get_current_active_user)], + tags=["Organizations"], +) +async def call_get_organizations( + state: Optional[List[str]] = Query(None), + regionId: Optional[List[str]] = Query(None), +): + """ + List all organizations with query parameters. + Args: + state (Optional[List[str]]): List of states to filter organizations by. + regionId (Optional[List[str]]): List of region IDs to filter organizations by. + + Raises: + HTTPException: If the user is not authorized or no organizations are found. + + Returns: + List[Organizations]: A list of organizations matching the filter criteria. + """ + return get_organizations(state, regionId) From 2fcec49a807913c70df14125df629e83c5c5a3ff Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 25 Sep 2024 11:16:18 -0400 Subject: [PATCH 024/314] Update manytomany columns in typescript and update models --- backend/src/models/organization-tag.ts | 12 ++- backend/src/models/scan-task.ts | 12 ++- backend/src/models/scan.ts | 24 ++++- backend/src/xfd_django/xfd_api/models.py | 96 +++++++---------- backend/src/xfd_django/xfd_api/scans.py | 125 ++++++++++++++--------- 5 files changed, 158 insertions(+), 111 deletions(-) diff --git a/backend/src/models/organization-tag.ts b/backend/src/models/organization-tag.ts index 19ce0721..a6831918 100644 --- a/backend/src/models/organization-tag.ts +++ b/backend/src/models/organization-tag.ts @@ -33,7 +33,17 @@ export class OrganizationTag extends BaseEntity { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) - @JoinTable() + @JoinTable({ + name: 'organization_tag_organizations_organization', + joinColumn: { + name: 'organizationtag_id', + referencedColumnName: 'id' + }, + inverseJoinColumn: { + name: 'organization_id', + referencedColumnName: 'id' + } + }) organizations: Organization[]; /** diff --git a/backend/src/models/scan-task.ts b/backend/src/models/scan-task.ts index 82b7896d..59c90619 100644 --- a/backend/src/models/scan-task.ts +++ b/backend/src/models/scan-task.ts @@ -44,7 +44,17 @@ export class ScanTask extends BaseEntity { onUpdate: 'CASCADE' } ) - @JoinTable() + @JoinTable({ + name: 'scan_task_organizations_organization', + joinColumn: { + name: 'scantask_id', + referencedColumnName: 'id' + }, + inverseJoinColumn: { + name: 'organization_id', + referencedColumnName: 'id' + } + }) organizations: Organization[]; @ManyToOne((type) => Scan, (scan) => scan.scanTasks, { diff --git a/backend/src/models/scan.ts b/backend/src/models/scan.ts index 4a098acf..0f71bd53 100644 --- a/backend/src/models/scan.ts +++ b/backend/src/models/scan.ts @@ -79,7 +79,17 @@ export class Scan extends BaseEntity { onUpdate: 'CASCADE' } ) - @JoinTable() + @JoinTable({ + name: 'scan_organizations_organization', + joinColumn: { + name: 'scan_id', + referencedColumnName: 'id' + }, + inverseJoinColumn: { + name: 'organization_id', + referencedColumnName: 'id' + } + }) organizations: Organization[]; /** @@ -90,7 +100,17 @@ export class Scan extends BaseEntity { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) - @JoinTable() + @JoinTable({ + name: 'scan_tags_organization_tag', + joinColumn: { + name: 'scan_id', + referencedColumnName: 'id' + }, + inverseJoinColumn: { + name: 'organizationtag_id', + referencedColumnName: 'id' + } + }) tags: OrganizationTag[]; @ManyToOne((type) => User, { diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 856f083c..473ddfb7 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -271,8 +271,9 @@ class Organization(models.Model): "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True ) # Relationships with other models (Scan, OrganizationTag) - granularScans = models.ManyToManyField('Scan', related_name="organizations", through='ScanOrganizationsOrganization') - tags = models.ManyToManyField('OrganizationTag', related_name="organizations", through='ScanTagsOrganizationTag') + granularScans = models.ManyToManyField('Scan', related_name='organizations', db_table="scan_organizations_organization") + tags = models.ManyToManyField('OrganizationTag', related_name='organizations', db_table="organization_tag_organizations_organization") + allScanTasks = models.ManyToManyField('ScanTask', related_name='organizations', db_table="organization_tag_organizations_organization") class Meta: """The meta class for Organization.""" @@ -288,6 +289,8 @@ class OrganizationTag(models.Model): createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) name = models.CharField(unique=True) + organizations = models.ManyToManyField('Organization', related_name='tags', db_table="organization_tag_organizations_organization") + scans = models.ManyToManyField('Scan', related_name='tags', db_table="scan_tags_organization_tag") class Meta: """The Meta class for OrganizationTag.""" @@ -296,29 +299,6 @@ class Meta: db_table = "organization_tag" -class OrganizationTagOrganizationsOrganization(models.Model): - """The OrganizationTagOrganizationsOrganization model.""" - - organizationTagId = models.ForeignKey( - OrganizationTag, - on_delete=models.CASCADE, - db_column="organizationTagId" - ) - organizationId = models.ForeignKey( - Organization, - on_delete=models.CASCADE, - db_column="organizationId" - ) - - class Meta: - """The Meta class for OrganizationTagOrganizationsOrganization.""" - - managed = False - db_table = "organization_tag_organizations_organization" - unique_together = (("organizationTagId", "organizationId"),) - auto_created = True - - class QueryResultCache(models.Model): """The QueryResultCache model.""" @@ -486,7 +466,7 @@ class Scan(models.Model): createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) name = models.CharField() - arguments = models.TextField() # JSON in the database but fails: the JSON object must be str, bytes or bytearray, not dict + arguments = models.TextField(default='{}') # JSON in the database but fails: the JSON object must be str, bytes or bytearray, not dict frequency = models.IntegerField() lastRun = models.DateTimeField(db_column="lastRun", blank=True, null=True) isGranular = models.BooleanField(db_column="isGranular", default=False) @@ -498,8 +478,8 @@ class Scan(models.Model): createdBy = models.ForeignKey( "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True ) - tags = models.ManyToManyField('OrganizationTag', through='ScanTagsOrganizationTag', related_name='scans') - organizations = models.ManyToManyField('Organization', through='ScanOrganizationsOrganization', related_name='scans') + tags = models.ManyToManyField('OrganizationTag', related_name='scans', db_table="scan_tags_organization_tag") + organizations = models.ManyToManyField('Organization', related_name='scans', db_table="scan_organizations_organization") class Meta: """The Meta class for Scan.""" @@ -508,29 +488,29 @@ class Meta: db_table = "scan" -class ScanOrganizationsOrganization(models.Model): - """The ScanOrganizationsOrganization model.""" +# class ScanOrganizationsOrganization(models.Model): +# """The ScanOrganizationsOrganization model.""" - scanId = models.ForeignKey('Scan', on_delete=models.CASCADE, db_column="scanId", primary_key=True) - organizationId = models.ForeignKey('Organization', on_delete=models.CASCADE, db_column="organizationId", primary_key=True) +# scanId = models.ForeignKey('Scan', on_delete=models.CASCADE, db_column="scanId", primary_key=True) +# organizationId = models.ForeignKey('Organization', on_delete=models.CASCADE, db_column="organizationId", primary_key=True) - class Meta: - db_table = "scan_organizations_organization" - unique_together = (("scanId", "organizationId"),) - # Do not create an id column automatically, treat both columns as composite primary keys - auto_created = True +# class Meta: +# db_table = "scan_organizations_organization" +# unique_together = (("scanId", "organizationId"),) +# # Do not create an id column automatically, treat both columns as composite primary keys +# auto_created = True -class ScanTagsOrganizationTag(models.Model): - """Intermediary model for the Many-to-Many relationship between Scan and OrganizationTag.""" +# class ScanTagsOrganizationTag(models.Model): +# """Intermediary model for the Many-to-Many relationship between Scan and OrganizationTag.""" - scanId = models.ForeignKey('Scan', on_delete=models.CASCADE, db_column="scanId", primary_key=True) - organizationTagId = models.ForeignKey('OrganizationTag', on_delete=models.CASCADE, db_column="organizationTagId", primary_key=True) +# scanId = models.ForeignKey('Scan', on_delete=models.CASCADE, db_column="scanId", primary_key=True) +# organizationTagId = models.ForeignKey('OrganizationTag', on_delete=models.CASCADE, db_column="organizationTagId", primary_key=True) - class Meta: - db_table = "scan_tags_organization_tag" - unique_together = (("scanId", "organizationTagId"),) - auto_created = True +# class Meta: +# db_table = "scan_tags_organization_tag" +# unique_together = (("scanId", "organizationTagId"),) +# auto_created = True class ScanTask(models.Model): @@ -566,22 +546,22 @@ class Meta: db_table = "scan_task" -class ScanTaskOrganizationsOrganization(models.Model): - """The ScanTaskOrganizationsOrganization model.""" +# class ScanTaskOrganizationsOrganization(models.Model): +# """The ScanTaskOrganizationsOrganization model.""" - scanTaskId = models.OneToOneField( - ScanTask, models.DO_NOTHING, db_column="scanTaskId", primary_key=True - ) # The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. - organizationId = models.ForeignKey( - Organization, models.DO_NOTHING, db_column="organizationId" - ) +# scanTaskId = models.OneToOneField( +# ScanTask, models.DO_NOTHING, db_column="scanTaskId", primary_key=True +# ) # The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. +# organizationId = models.ForeignKey( +# Organization, models.DO_NOTHING, db_column="organizationId" +# ) - class Meta: - """The Meta class for ScanTaskOrganizationsOrganization.""" +# class Meta: +# """The Meta class for ScanTaskOrganizationsOrganization.""" - managed = False - db_table = "scan_task_organizations_organization" - unique_together = (("scanTaskId", "organizationId"),) +# managed = False +# db_table = "scan_task_organizations_organization" +# unique_together = (("scanTaskId", "organizationId"),) class Service(models.Model): diff --git a/backend/src/xfd_django/xfd_api/scans.py b/backend/src/xfd_django/xfd_api/scans.py index 6bc17736..1a72a2cb 100644 --- a/backend/src/xfd_django/xfd_api/scans.py +++ b/backend/src/xfd_django/xfd_api/scans.py @@ -1,5 +1,5 @@ -from .models import Scan, Organization, OrganizationTag, ScanTagsOrganizationTag, ScanOrganizationsOrganization -from .schemas import ScanSchema +from .models import Scan, Organization, OrganizationTag +from .schemas import ScanSchema, CreateScan from django.db import transaction from fastapi import HTTPException from django.http import JsonResponse @@ -192,85 +192,112 @@ def list_scans(): """List scans.""" try: - # Fetch scans - scans = Scan.objects.all().values() - - # Convert scans and related tags to dict format - scan_list = [] - for scan in scans: - related_tags = OrganizationTag.objects.filter( - scantagsorganizationtag__scanId=scan['id'] - ).values() - - # Add tags to each scan - scan['tags'] = list(related_tags) - scan_list.append(scan) + # Fetch scans and prefetch related tags + scans = Scan.objects.prefetch_related('tags').all() # Fetch all organizations - organizations = list(Organization.objects.values('id', 'name')) + organizations = Organization.objects.values('id', 'name') - # Return everything as a JSON response + # Convert to list of dicts with related tags + scan_list = [] + for scan in scans: + scan_data = { + 'id': scan.id, + 'createdAt': scan.createdAt, + 'updatedAt': scan.updatedAt, + 'name': scan.name, + 'arguments': scan.arguments, + 'frequency': scan.frequency, + 'lastRun': scan.lastRun, + 'isGranular': scan.isGranular, + 'isUserModifiable': scan.isUserModifiable, + 'isSingleScan': scan.isSingleScan, + 'manualRunPending': scan.manualRunPending, + 'tags': [ + { + 'id': tag.id, + 'createdAt': tag.createdAt, + 'updatedAt': tag.updatedAt, + 'name': tag.name + } for tag in scan.tags.all() + ] + } + scan_list.append(scan_data) + + # Return response with scans, schema, and organizations response = { 'scans': scan_list, - 'schema': SCAN_SCHEMA, # Add your predefined SCAN_SCHEMA here - 'organizations': organizations + 'schema': SCAN_SCHEMA, + 'organizations': list(organizations) } - return response + return response except Exception as e: raise HTTPException(status_code=500, detail=str(e)) def list_granular_scans(): - """ - List all granular scans that can be modified by users. - """ + """List user-modifiable granular scans.""" try: + # Fetch scans that match the criteria (isGranular, isUserModifiable, isSingleScan) scans = Scan.objects.filter( - isGranular=True, isUserModifiable=True, isSingleScan=False - ) - return {"scans": list(scans), 'schema': SCAN_SCHEMA} + isGranular=True, + isUserModifiable=True, + isSingleScan=False + ).values('id', 'name', 'isUserModifiable') + + # Prepare the response + response = { + 'scans': list(scans), + 'schema': SCAN_SCHEMA # Predefined schema + } + + return response + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) -def create_scan(scan_data: Scan, current_user): +def create_scan(scan_data: CreateScan, current_user): """ Create a new scan. """ try: + + # TODO: Check if the user is a GlobalWriteAdmin. Create function in auth.py + # Create the scan instance using **scan_data.dict() scan_data_dict = scan_data.dict(exclude_unset=True, exclude={"organizations", "frequencyUnit", "tags"}) scan_data_dict['createdBy'] = current_user print(scan_data_dict) # Create scan using the dictionary unpacking - scan = Scan.objects.create(**scan_data_dict) + # scan = Scan.objects.create(**scan_data_dict) print("added Scan") - print(scan) + # print(scan) # Link organizations - if scan_data.organizations: - for org_id in scan_data.organizations: - organization, created = Organization.objects.get_or_create(id=org_id) - print(f"Linked Organization {organization}") - ScanOrganizationsOrganization.objects.create( - scanId=scan, - organizationId=organization - ) + # if scan_data.organizations: + # for org_id in scan_data.organizations: + # organization, created = Organization.objects.get_or_create(id=org_id) + # print(f"Linked Organization {organization}") + # ScanOrganizationsOrganization.objects.create( + # scanId=scan, + # organizationId=organization + # ) - # Link tags - if scan_data.tags: - for tag_data in scan_data.tags: - tag, created = OrganizationTag.objects.get_or_create(id=tag_data.id) - print(f"Linked Tag {tag}") - ScanTagsOrganizationTag.objects.create( - scanId=scan, - organizationTagId=tag - ) - - # Return the saved scan - return scan + # # Link tags + # if scan_data.tags: + # for tag_data in scan_data.tags: + # tag, created = OrganizationTag.objects.get_or_create(id=tag_data.id) + # print(f"Linked Tag {tag}") + # ScanTagsOrganizationTag.objects.create( + # scanId=scan, + # organizationTagId=tag + # ) + + # # Return the saved scan + # return scan except Organization.DoesNotExist: raise HTTPException(status_code=404, detail="Organization not found") From 85bc98de120be737094cac6c1a43cbeb15f2846a Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 26 Sep 2024 11:16:31 -0400 Subject: [PATCH 025/314] Add scans and scheduler --- backend/requirements.txt | 4 +- .../xfd_api/api_methods/__init__.py | 0 .../xfd_django/xfd_api/api_methods/scans.py | 302 +++++++++++++ backend/src/xfd_django/xfd_api/auth.py | 77 ++++ .../xfd_django/xfd_api/helpers/__init__.py | 0 .../xfd_api/helpers/getScanOrganizations.py | 21 + backend/src/xfd_django/xfd_api/models.py | 23 +- backend/src/xfd_django/xfd_api/scans.py | 408 ------------------ .../xfd_api/schema_models/__init__.py | 0 .../xfd_api/schema_models/organization.py | 34 ++ .../xfd_api/schema_models/organization_tag.py | 16 + .../xfd_django/xfd_api/schema_models/scan.py | 291 +++++++++++++ backend/src/xfd_django/xfd_api/schemas.py | 79 ---- .../src/xfd_django/xfd_api/tasks/__init__.py | 0 .../xfd_django/xfd_api/tasks/ecs_client.py | 166 +++++++ .../xfd_django/xfd_api/tasks/lambda_client.py | 28 ++ .../src/xfd_django/xfd_api/tasks/scheduler.py | 246 +++++++++++ backend/src/xfd_django/xfd_api/views.py | 135 +++--- 18 files changed, 1255 insertions(+), 575 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/api_methods/__init__.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/scans.py create mode 100644 backend/src/xfd_django/xfd_api/helpers/__init__.py create mode 100644 backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py delete mode 100644 backend/src/xfd_django/xfd_api/scans.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/__init__.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/organization.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/organization_tag.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/scan.py create mode 100644 backend/src/xfd_django/xfd_api/tasks/__init__.py create mode 100644 backend/src/xfd_django/xfd_api/tasks/ecs_client.py create mode 100644 backend/src/xfd_django/xfd_api/tasks/lambda_client.py create mode 100644 backend/src/xfd_django/xfd_api/tasks/scheduler.py diff --git a/backend/requirements.txt b/backend/requirements.txt index 1a7f0395..e67a7b63 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,4 +3,6 @@ mangum==0.17.0 uvicorn==0.30.1 django psycopg2-binary -PyJWT \ No newline at end of file +PyJWT +boto3 +docker \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/api_methods/__init__.py b/backend/src/xfd_django/xfd_api/api_methods/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/api_methods/scans.py b/backend/src/xfd_django/xfd_api/api_methods/scans.py new file mode 100644 index 00000000..ce517183 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/scans.py @@ -0,0 +1,302 @@ +from ..models import Scan, Organization, OrganizationTag +from ..schemas import SCAN_SCHEMA, NewScan +from ..auth import is_global_write_admin, is_global_view_admin +from django.db import transaction +from fastapi import HTTPException +from django.http import JsonResponse +from django.forms.models import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +import os +from ..lambda_client import LambdaClient + + +def list_scans(current_user): + """List scans.""" + try: + # Check if the user is a GlobalViewAdmin + if not is_global_view_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") + + # Fetch scans and prefetch related tags + scans = Scan.objects.prefetch_related('tags').all() + + # Fetch all organizations + organizations = Organization.objects.values('id', 'name') + + # Convert to list of dicts with related tags + scan_list = [] + for scan in scans: + scan_data = { + 'id': scan.id, + 'createdAt': scan.createdAt, + 'updatedAt': scan.updatedAt, + 'name': scan.name, + 'arguments': scan.arguments, + 'frequency': scan.frequency, + 'lastRun': scan.lastRun, + 'isGranular': scan.isGranular, + 'isUserModifiable': scan.isUserModifiable, + 'isSingleScan': scan.isSingleScan, + 'manualRunPending': scan.manualRunPending, + 'tags': [ + { + 'id': tag.id, + 'createdAt': tag.createdAt, + 'updatedAt': tag.updatedAt, + 'name': tag.name + } for tag in scan.tags.all() + ] + } + scan_list.append(scan_data) + + # Return response with scans, schema, and organizations + response = { + 'scans': scan_list, + 'schema': SCAN_SCHEMA, + 'organizations': list(organizations) + } + + return response + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def list_granular_scans(current_user): + """List granular scans.""" + try: + # Check if the user is a GlobalViewAdmin + if not is_global_view_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") + + # Fetch scans that match the criteria (isGranular, isUserModifiable, isSingleScan) + scans = Scan.objects.filter( + isGranular=True, + isUserModifiable=True, + isSingleScan=False + ).values('id', 'name', 'isUserModifiable') + + response = { + 'scans': list(scans), + 'schema': SCAN_SCHEMA + } + + return response + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def create_scan(scan_data: NewScan, current_user): + """Create a new scan.""" + try: + + # Check if the user is a GlobalWriteAdmin + if not is_global_write_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") + + # Check if scan name is valid + if scan_data.name not in SCAN_SCHEMA: + raise HTTPException(status_code=400, detail="Invalid scan name") + + # Create the scan instance + scan_data_dict = scan_data.dict(exclude_unset=True, exclude={"organizations", "tags"}) + scan_data_dict['createdBy'] = current_user + print(scan_data_dict) + + # Create the scan object + scan = Scan.objects.create(**scan_data_dict) + + # Link organizations + if scan_data.organizations: + scan.organizations.set(scan_data.organizations) + + # Link tags + if scan_data.tags: + tag_ids = [tag.id for tag in scan_data.tags] + scan.tags.set(tag_ids) + + return { + 'name': scan.name, + 'arguments': scan.arguments, + 'frequency': scan.frequency, + 'isGranular': scan.isGranular, + 'isUserModifiable': scan.isUserModifiable, + 'isSingleScan': scan.isSingleScan, + 'createdBy': { + 'id': current_user.id, + 'name': current_user.fullName + }, + 'tags': list(scan.tags.values('id')), + 'organizations': list(scan.organizations.values('id')), + } + + except Organization.DoesNotExist: + raise HTTPException(status_code=404, detail="Organization not found") + except OrganizationTag.DoesNotExist: + raise HTTPException(status_code=404, detail="Tag not found") + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + + + +def get_scan(scan_id: str, current_user): + """Get a scan by its ID. """ + + # Check if the user is a GlobalViewAdmin + if not is_global_view_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") + + try: + # Fetch the scan with its related organizations and tags + scan = Scan.objects.prefetch_related('organizations', 'tags').get(id=scan_id) + + # Fetch all organizations + all_organizations = Organization.objects.values('id', 'name') + except Scan.DoesNotExist: + raise HTTPException(status_code=404, detail="Scan not found") + + # Get related organizations with all fields and remove unwanted fields + related_organizations = list(scan.organizations.values()) + for org in related_organizations: + org.pop('parentId_id', None) + org.pop('createdById_id', None) + + # Serialize scan data + scan_data = { + 'id': str(scan.id), + 'createdAt': scan.createdAt, + 'updatedAt': scan.updatedAt, + 'name': scan.name, + 'arguments': scan.arguments, + 'lastRun': scan.lastRun, + 'frequency': scan.frequency, + 'isGranular': scan.isGranular, + 'isUserModifiable': scan.isUserModifiable, + 'isSingleScan': scan.isSingleScan, + 'manualRunPending': scan.manualRunPending, + 'organizations': related_organizations, + 'tags': list(scan.tags.values()) + } + + # Return the scan details along with its related data + return { + 'scan': scan_data, + 'schema': dict(SCAN_SCHEMA[scan.name]), + 'organizations': list(all_organizations) + } + + + +def update_scan(scan_id: str, scan_data: NewScan, current_user): + """Update a scan by its ID.""" + try: + # Check if the user is a GlobalWriteAdmin + if not is_global_write_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") + + # Validate scan ID + try: + scan = Scan.objects.get(id=scan_id) + except Scan.DoesNotExist: + raise HTTPException(status_code=404, detail="Scan not found") + + # Update the scan's fields with the new data + scan.name = scan_data.name + scan.arguments = scan_data.arguments + scan.frequency = scan_data.frequency + scan.isGranular = scan_data.isGranular + scan.isUserModifiable = scan_data.isUserModifiable + scan.isSingleScan = scan_data.isSingleScan + + # Update ManyToMany relationships + if scan_data.organizations: + scan.organizations.set(scan_data.organizations) + + if scan_data.tags: + tag_ids = [tag.id for tag in scan_data.tags] + scan.tags.set(tag_ids) + + # Save the updated scan + scan.save() + + return { + 'name': scan.name, + 'arguments': scan.arguments, + 'frequency': scan.frequency, + 'isGranular': scan.isGranular, + 'isUserModifiable': scan.isUserModifiable, + 'isSingleScan': scan.isSingleScan, + 'createdBy': { + 'id': current_user.id, + 'name': current_user.fullName + }, + 'tags': list(scan.tags.values('id')), + 'organizations': list(scan.organizations.values('id')), + } + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def delete_scan(scan_id: str, current_user): + """Delete a scan by its ID.""" + try: + # Check if the user is a GlobalWriteAdmin + if not is_global_write_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") + + # Validate scan ID + try: + scan = Scan.objects.get(id=scan_id) + except Scan.DoesNotExist: + raise HTTPException(status_code=404, detail="Scan not found") + + scan.delete() + + return {"status": "success", "message": f"Scan {scan_id} deleted successfully."} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def run_scan(scan_id: str, current_user): + """Mark a scan as manually triggered to run.""" + try: + # Check if the user is a GlobalWriteAdmin + if not is_global_write_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") + + # Validate the scan ID and check if it exists + try: + scan = Scan.objects.get(id=scan_id) + except Scan.DoesNotExist: + raise HTTPException(status_code=404, detail="Scan not found") + + scan.manualRunPending = True + scan.save() + return {"status": "success", "message": f"Scan {scan_id} deleted successfully."} + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +async def invoke_scheduler(current_user): + """Manually invoke the scan scheduler.""" + try: + #TODO: RUN THIS ON A SCHEDULE LOCALLY LIKE DEFINED IN APP.TS + # Check if the user is a GlobalWriteAdmin + if not is_global_write_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access.") + + # Initialize the Lambda client + lambda_client = LambdaClient() + + # Form the lambda function name using environment variable + lambda_function_name = f"{os.getenv('SLS_LAMBDA_PREFIX')}-scheduler" + print(lambda_function_name) + + # Run the Lambda command + response = await lambda_client.run_command(name=lambda_function_name) + + return response + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 81929c10..e4fc2487 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -106,3 +106,80 @@ def get_current_active_user( ) print(f"Authenticated user: {user.id}") return user + + +def is_global_write_admin(current_user) -> bool: + """Check if the user has global write admin permissions.""" + return current_user and current_user.userType == "globalAdmin" + +def is_global_view_admin(current_user) -> bool: + """Check if the user has global view permissions. """ + return current_user and current_user.userType in ["globalView", "globalAdmin"] + +def is_regional_admin(current_user) -> bool: + """Check if the user has regional admin permissions.""" + return current_user and current_user.userType in ["regionalAdmin", "globalAdmin"] + + +# TODO: Below is a template of what these could be nut isn't tested +# RECREATE ALL THE FUNCTIONS IN AUTH.TS + +# async def is_regional_admin_for_organization(request: Request, organization_id: str) -> bool: +# """ +# Check if the user is a regional admin and if the organization belongs to their region. + +# Args: +# request (Request): The FastAPI request object. +# organization_id (str): The ID of the organization. + +# Returns: +# bool: True if the user is a regional admin for the organization, False otherwise. +# """ +# current_user = request.state.user +# if not current_user or current_user.userType not in ["REGIONAL_ADMIN", "GLOBAL_ADMIN"]: +# return False + +# user_region = current_user.regionId +# organization_region = await get_region(organization_id) + +# return user_region == organization_region + +# def is_org_admin(request: Request, organization_id: str) -> bool: +# """ +# Check if the user is an admin of the given organization. + +# Args: +# request (Request): The FastAPI request object. +# organization_id (str): The ID of the organization. + +# Returns: +# bool: True if the user is an admin of the organization, False otherwise. +# """ +# current_user = request.state.user +# if not current_user: +# return False + +# # Check if the user is a global admin +# if current_user.userType == "GLOBAL_ADMIN": +# return True + +# # Check if the user is an admin for the given organization +# for role in current_user.roles: +# if role.organization.id == organization_id and role.role == 'admin': +# return True + +# return False + +# def get_user_id(request: Request) -> str: +# """ +# Returns the user's ID. + +# Args: +# request (Request): The FastAPI request object. + +# Returns: +# str: The ID of the current user. +# """ +# current_user = request.state.user +# return current_user.id if current_user else None + diff --git a/backend/src/xfd_django/xfd_api/helpers/__init__.py b/backend/src/xfd_django/xfd_api/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py b/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py new file mode 100644 index 00000000..218d9e5f --- /dev/null +++ b/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py @@ -0,0 +1,21 @@ +from typing import List +from ..models import Organization, Scan + +def get_scan_organizations(scan: Scan) -> List[Organization]: + """ + Returns the organizations that a scan should be run on. + A scan should be run on an organization if the scan is + enabled for that organization or for one of its tags. + """ + organizations_to_run = {} + + # Add organizations associated with tags + for tag in scan.tags.all(): + for organization in tag.organizations.all(): + organizations_to_run[organization.id] = organization + + # Add directly associated organizations + for organization in scan.organizations.all(): + organizations_to_run[organization.id] = organization + + return list(organizations_to_run.values()) diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 473ddfb7..606e6bd7 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -270,10 +270,11 @@ class Organization(models.Model): createdById = models.ForeignKey( "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True ) + # TODO: Get rid of this, don't need Many To Many in both tables # Relationships with other models (Scan, OrganizationTag) granularScans = models.ManyToManyField('Scan', related_name='organizations', db_table="scan_organizations_organization") tags = models.ManyToManyField('OrganizationTag', related_name='organizations', db_table="organization_tag_organizations_organization") - allScanTasks = models.ManyToManyField('ScanTask', related_name='organizations', db_table="organization_tag_organizations_organization") + allScanTasks = models.ManyToManyField('ScanTask', related_name='organizations', db_table="scan_task_organizations_organization") class Meta: """The meta class for Organization.""" @@ -516,9 +517,9 @@ class Meta: class ScanTask(models.Model): """The ScanTask model.""" - id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column="createdAt") - updatedAt = models.DateTimeField(db_column="updatedAt") + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) + updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) status = models.TextField() type = models.TextField() fargateTaskArn = models.TextField(db_column="fargateTaskArn", blank=True, null=True) @@ -528,16 +529,14 @@ class ScanTask(models.Model): startedAt = models.DateTimeField(db_column="startedAt", blank=True, null=True) finishedAt = models.DateTimeField(db_column="finishedAt", blank=True, null=True) queuedAt = models.DateTimeField(db_column="queuedAt", blank=True, null=True) - organizationId = models.ForeignKey( - Organization, - models.DO_NOTHING, - db_column="organizationId", - blank=True, - null=True, - ) scanId = models.ForeignKey( - Scan, models.DO_NOTHING, db_column="scanId", blank=True, null=True + Scan, + on_delete=models.DO_NOTHING, + db_column="scanId", + blank=True, + null=True ) + organizations = models.ManyToManyField('Organization', related_name='allScanTasks', db_table="scan_task_organizations_organization") class Meta: """The Meta class for ScanTask.""" diff --git a/backend/src/xfd_django/xfd_api/scans.py b/backend/src/xfd_django/xfd_api/scans.py deleted file mode 100644 index 1a72a2cb..00000000 --- a/backend/src/xfd_django/xfd_api/scans.py +++ /dev/null @@ -1,408 +0,0 @@ -from .models import Scan, Organization, OrganizationTag -from .schemas import ScanSchema, CreateScan -from django.db import transaction -from fastapi import HTTPException -from django.http import JsonResponse -from django.forms.models import model_to_dict -from django.core.serializers.json import DjangoJSONEncoder - - -SCAN_SCHEMA = { - "amass": ScanSchema( - type="fargate", - isPassive=False, - global_scan=False, - description="Open source tool that integrates passive APIs and active subdomain enumeration in order to discover target subdomains", - ), - "censys": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - description="Passive discovery of subdomains from public certificates", - ), - "censysCertificates": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - cpu="2048", - memory="6144", - numChunks=20, - description="Fetch TLS certificate data from censys certificates dataset", - ), - "censysIpv4": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - cpu="2048", - memory="6144", - numChunks=20, - description="Fetch passive port and banner data from censys ipv4 dataset", - ), - "cve": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - cpu="1024", - memory="8192", - description="Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog.", - ), - "vulnScanningSync": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - cpu="1024", - memory="8192", - description="Pull in vulnerability data from VSs Vulnerability database", - ), - "cveSync": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - cpu="1024", - memory="8192", - description="Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog.", - ), - "dnstwist": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - cpu="2048", - memory="16384", - description="Domain name permutation engine for detecting similar registered domains.", - ), - "dotgov": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - description='Create organizations based on root domains from the dotgov registrar dataset. All organizations are created with the "dotgov" tag and have a " (dotgov)" suffix added to their name.', - ), - "findomain": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - description="Open source tool that integrates passive APIs in order to discover target subdomains", - ), - "hibp": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - cpu="2048", - memory="16384", - description="Finds emails that have appeared in breaches related to a given domain", - ), - "intrigueIdent": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - cpu="1024", - memory="4096", - description="Open source tool that fingerprints web technologies based on HTTP responses", - ), - "lookingGlass": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - description="Finds vulnerabilities and malware from the LookingGlass API", - ), - "portscanner": ScanSchema( - type="fargate", - isPassive=False, - global_scan=False, - description="Active port scan of common ports", - ), - "rootDomainSync": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - description="Creates domains from root domains by doing a single DNS lookup for each root domain.", - ), - "rscSync": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - description="Retrieves and saves assessments from ReadySetCyber mission instance.", - ), - "savedSearch": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - description="Performs saved searches to update their search results", - ), - "searchSync": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - cpu="1024", - memory="4096", - description="Syncs records with Elasticsearch so that they appear in search results.", - ), - "shodan": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - cpu="1024", - memory="8192", - description="Fetch passive port, banner, and vulnerability data from shodan", - ), - "sslyze": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - description="SSL certificate inspection", - ), - "test": ScanSchema( - type="fargate", - isPassive=False, - global_scan=True, - description="Not a real scan, used to test", - ), - "trustymail": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - description="Evaluates SPF/DMARC records and checks MX records for STARTTLS support", - ), - "vulnSync": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - cpu="1024", - memory="8192", - description="Pull in vulnerability data from PEs Vulnerability database", - ), - "wappalyzer": ScanSchema( - type="fargate", - isPassive=True, - global_scan=False, - cpu="1024", - memory="4096", - description="Open source tool that fingerprints web technologies based on HTTP responses", - ), - "xpanseSync": ScanSchema( - type="fargate", - isPassive=True, - global_scan=True, - cpu="1024", - memory="8192", - description="Pull in xpanse vulnerability data from PEs Vulnerability database", - ), -} - - -def list_scans(): - """List scans.""" - try: - # Fetch scans and prefetch related tags - scans = Scan.objects.prefetch_related('tags').all() - - # Fetch all organizations - organizations = Organization.objects.values('id', 'name') - - # Convert to list of dicts with related tags - scan_list = [] - for scan in scans: - scan_data = { - 'id': scan.id, - 'createdAt': scan.createdAt, - 'updatedAt': scan.updatedAt, - 'name': scan.name, - 'arguments': scan.arguments, - 'frequency': scan.frequency, - 'lastRun': scan.lastRun, - 'isGranular': scan.isGranular, - 'isUserModifiable': scan.isUserModifiable, - 'isSingleScan': scan.isSingleScan, - 'manualRunPending': scan.manualRunPending, - 'tags': [ - { - 'id': tag.id, - 'createdAt': tag.createdAt, - 'updatedAt': tag.updatedAt, - 'name': tag.name - } for tag in scan.tags.all() - ] - } - scan_list.append(scan_data) - - # Return response with scans, schema, and organizations - response = { - 'scans': scan_list, - 'schema': SCAN_SCHEMA, - 'organizations': list(organizations) - } - - return response - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -def list_granular_scans(): - """List user-modifiable granular scans.""" - try: - # Fetch scans that match the criteria (isGranular, isUserModifiable, isSingleScan) - scans = Scan.objects.filter( - isGranular=True, - isUserModifiable=True, - isSingleScan=False - ).values('id', 'name', 'isUserModifiable') - - # Prepare the response - response = { - 'scans': list(scans), - 'schema': SCAN_SCHEMA # Predefined schema - } - - return response - - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -def create_scan(scan_data: CreateScan, current_user): - """ - Create a new scan. - """ - try: - - # TODO: Check if the user is a GlobalWriteAdmin. Create function in auth.py - - # Create the scan instance using **scan_data.dict() - scan_data_dict = scan_data.dict(exclude_unset=True, exclude={"organizations", "frequencyUnit", "tags"}) - scan_data_dict['createdBy'] = current_user - print(scan_data_dict) - - # Create scan using the dictionary unpacking - # scan = Scan.objects.create(**scan_data_dict) - - print("added Scan") - # print(scan) - # Link organizations - # if scan_data.organizations: - # for org_id in scan_data.organizations: - # organization, created = Organization.objects.get_or_create(id=org_id) - # print(f"Linked Organization {organization}") - # ScanOrganizationsOrganization.objects.create( - # scanId=scan, - # organizationId=organization - # ) - - # # Link tags - # if scan_data.tags: - # for tag_data in scan_data.tags: - # tag, created = OrganizationTag.objects.get_or_create(id=tag_data.id) - # print(f"Linked Tag {tag}") - # ScanTagsOrganizationTag.objects.create( - # scanId=scan, - # organizationTagId=tag - # ) - - # # Return the saved scan - # return scan - - except Organization.DoesNotExist: - raise HTTPException(status_code=404, detail="Organization not found") - except OrganizationTag.DoesNotExist: - raise HTTPException(status_code=404, detail="Tag not found") - except Exception as e: - print(e) - raise HTTPException(status_code=500, detail=str(e)) - - - -def get_scan(scan_id: str): - """ - Retrieve a scan by its ID. - - Parameters: - - scan_id: The ID of the scan to retrieve. - - Returns: - - The scan object with the specified ID. - """ - try: - scan = Scan.objects.get(id=scan_id) - if not scan: - raise HTTPException(status_code=404, detail="Scan not found") - return scan - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -def update_scan(scan_id: str, scan_data: Scan): - """ - Update a scan by its ID. - - Parameters: - - scan_id: The ID of the scan to update. - - scan_data: The new data to update the scan with. - - Returns: - - The updated scan object. - """ - try: - scan = Scan.objects.get(id=scan_id) - if not scan: - raise HTTPException(status_code=404, detail="Scan not found") - - for key, value in scan_data.dict().items(): - setattr(scan, key, value) - scan.save() - return scan - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -def delete_scan(scan_id: str): - """ - Delete a scan by its ID. - - Parameters: - - scan_id: The ID of the scan to delete. - - Returns: - - A message confirming the deletion. - """ - try: - scan = Scan.objects.get(id=scan_id) - if not scan: - raise HTTPException(status_code=404, detail="Scan not found") - scan.delete() - return {"message": "Scan deleted"} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -def run_scan(scan_id: str): - """ - Mark a scan as manually triggered to run. - - Parameters: - - scan_id: The ID of the scan to run. - - Returns: - - A message confirming that the scan has been triggered. - """ - try: - scan = Scan.objects.get(id=scan_id) - if not scan: - raise HTTPException(status_code=404, detail="Scan not found") - scan.manualRunPending = True - scan.save() - return {"message": "Scan manually run"} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -def invoke_scheduler(): - """ - Manually invoke the scan scheduler. - - Returns: - - A message confirming that the scheduler has been invoked. - """ - try: - # Implement logic to invoke the scheduler manually - # This could be an external service call or AWS Lambda invocation - return {"message": "Scheduler invoked successfully"} - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/__init__.py b/backend/src/xfd_django/xfd_api/schema_models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization.py b/backend/src/xfd_django/xfd_api/schema_models/organization.py new file mode 100644 index 00000000..3a659537 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/organization.py @@ -0,0 +1,34 @@ +"""Schemas to support Organization endpoints.""" + +# Standard Python Libraries +from datetime import datetime +from typing import Any, Dict, List, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, Json + + +class Organization(BaseModel): + """Organization schema reflecting model.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + acronym: Optional[str] + name: str + rootDomains: List[str] + ipBlocks: List[str] + isPassive: bool + pendingDomains: Optional[List[dict]] + country: Optional[str] + state: Optional[str] + regionId: Optional[str] + stateFips: Optional[int] + stateName: Optional[str] + county: Optional[str] + countyFips: Optional[int] + type: Optional[str] + + class Config: + orm_mode = True \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py b/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py new file mode 100644 index 00000000..a8463588 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py @@ -0,0 +1,16 @@ +"""Schemas to support Organization Tag endpoints.""" + +# Standard Python Libraries +from datetime import datetime +from typing import Any, Dict, List, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, Json + +class OrganizationalTags(BaseModel): + """Organization Tags.""" + id: UUID + createdAt: datetime + updatedAt: datetime + name: str \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan.py b/backend/src/xfd_django/xfd_api/schema_models/scan.py new file mode 100644 index 00000000..4fbb59eb --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/scan.py @@ -0,0 +1,291 @@ +"""Schemas to support Scan endpoints.""" + +# cisagov Libraries +from .organization_tag import OrganizationalTags +from.organization import Organization + +# Standard Python Libraries +from datetime import datetime +from typing import Any, Dict, List, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, Json + +class Scan(BaseModel): + """Scan schema reflecting model.""" + id: UUID + createdAt: datetime + updatedAt: datetime + name: str + arguments: Any + frequency: int + lastRun: Optional[datetime] + isGranular: bool + isUserModifiable: Optional[bool] + isSingleScan: bool + manualRunPending: bool + tags: Optional[List[OrganizationalTags]] + organizations: Optional[List[Organization]] + +class ScanSchema(BaseModel): + """Scan type schema.""" + + type: str = 'fargate' # Only 'fargate' is supported + description: str + + # Whether scan is passive (not allowed to hit the domain). + isPassive: bool + + # Whether scan is global. Global scans run once for all organizations, as opposed + # to non-global scans, which are run for each organization. + global_scan: bool + + # CPU and memory for the scan. See this page for more information: + # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html + cpu: Optional[str] = None + memory: Optional[str] = None + + # A scan is "chunked" if its work is divided and run in parallel by multiple workers. + # To make a scan chunked, make sure it is a global scan and specify the "numChunks" variable, + # which corresponds to the number of workers that will be created to run the task. + # Chunked scans can only be run on scans whose implementation takes into account the + # chunkNumber and numChunks parameters specified in commandOptions. + numChunks: Optional[int] = None + +class GranularScan(BaseModel): + """Granular scan model.""" + id: UUID + name: str + isUserModifiable: Optional[bool] + +class GetScansResponseModel(BaseModel): + """Get Scans response model.""" + scans: List[Scan] + schema: Dict[str, Any] + organizations: List[Dict[str, Any]] + +class GetGranularScansResponseModel(BaseModel): + """Get Scans response model.""" + scans: List[GranularScan] + schema: Dict[str, Any] + +class IdSchema(BaseModel): + """Schema for ID objects.""" + id: UUID + +class NewScan(BaseModel): + """Create Scan Schema.""" + name: str + arguments: Any + organizations: Optional[List[UUID]] + tags: Optional[List[IdSchema]] + frequency: int + isGranular: bool + isUserModifiable: Optional[bool] + isSingleScan: bool + +class CreateScanResponseModel(BaseModel): + """Create Scan Schema.""" + name: str + arguments: Any + frequency: int + isGranular: bool + isUserModifiable: Optional[bool] + isSingleScan: bool + createdBy: Optional[Any] + tags: Optional[List[IdSchema]] + organizations: Optional[List[IdSchema]] + +class GetScanResponseModel(BaseModel): + """Get Scans response model.""" + scan: Scan + schema: Dict[str, Any] + organizations: List[Dict[str, Any]] + +class GenericMessageResponseModel(BaseModel): + """Get Scans response model.""" + status: str + message: str + + +SCAN_SCHEMA = { + "amass": ScanSchema( + type="fargate", + isPassive=False, + global_scan=False, + description="Open source tool that integrates passive APIs and active subdomain enumeration in order to discover target subdomains", + ), + "censys": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Passive discovery of subdomains from public certificates", + ), + "censysCertificates": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="2048", + memory="6144", + numChunks=20, + description="Fetch TLS certificate data from censys certificates dataset", + ), + "censysIpv4": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="2048", + memory="6144", + numChunks=20, + description="Fetch passive port and banner data from censys ipv4 dataset", + ), + "cve": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog.", + ), + "vulnScanningSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Pull in vulnerability data from VSs Vulnerability database", + ), + "cveSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Matches detected software versions to CVEs from NIST NVD and CISA's Known Exploited Vulnerabilities Catalog.", + ), + "dnstwist": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="2048", + memory="16384", + description="Domain name permutation engine for detecting similar registered domains.", + ), + "dotgov": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + description='Create organizations based on root domains from the dotgov registrar dataset. All organizations are created with the "dotgov" tag and have a " (dotgov)" suffix added to their name.', + ), + "findomain": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Open source tool that integrates passive APIs in order to discover target subdomains", + ), + "hibp": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="2048", + memory="16384", + description="Finds emails that have appeared in breaches related to a given domain", + ), + "intrigueIdent": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="1024", + memory="4096", + description="Open source tool that fingerprints web technologies based on HTTP responses", + ), + "lookingGlass": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Finds vulnerabilities and malware from the LookingGlass API", + ), + "portscanner": ScanSchema( + type="fargate", + isPassive=False, + global_scan=False, + description="Active port scan of common ports", + ), + "rootDomainSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Creates domains from root domains by doing a single DNS lookup for each root domain.", + ), + "rscSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + description="Retrieves and saves assessments from ReadySetCyber mission instance.", + ), + "savedSearch": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + description="Performs saved searches to update their search results", + ), + "searchSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="4096", + description="Syncs records with Elasticsearch so that they appear in search results.", + ), + "shodan": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="1024", + memory="8192", + description="Fetch passive port, banner, and vulnerability data from shodan", + ), + "sslyze": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="SSL certificate inspection", + ), + "test": ScanSchema( + type="fargate", + isPassive=False, + global_scan=True, + description="Not a real scan, used to test", + ), + "trustymail": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + description="Evaluates SPF/DMARC records and checks MX records for STARTTLS support", + ), + "vulnSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Pull in vulnerability data from PEs Vulnerability database", + ), + "wappalyzer": ScanSchema( + type="fargate", + isPassive=True, + global_scan=False, + cpu="1024", + memory="4096", + description="Open source tool that fingerprints web technologies based on HTTP responses", + ), + "xpanseSync": ScanSchema( + type="fargate", + isPassive=True, + global_scan=True, + cpu="1024", + memory="8192", + description="Pull in xpanse vulnerability data from PEs Vulnerability database", + ), +} \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/schemas.py b/backend/src/xfd_django/xfd_api/schemas.py index a9f12535..c54513bc 100644 --- a/backend/src/xfd_django/xfd_api/schemas.py +++ b/backend/src/xfd_django/xfd_api/schemas.py @@ -206,84 +206,6 @@ class OrganizationalTags(BaseModel): updatedAt: datetime name: str -class Scan(BaseModel): - """Scan schema.""" - id: UUID - createdAt: datetime - updatedAt: datetime - name: str - arguments: Any - frequency: int - lastRun: Optional[datetime] - isGranular: bool - isUserModifiable: Optional[bool] - isSingleScan: bool - manualRunPending: bool - createdBy_id: Optional[Any] - tags: Optional[List[OrganizationalTags]] - -class ScanSchema(BaseModel): - """Scan type schema.""" - - type: str = 'fargate' # Only 'fargate' is supported - description: str - - # Whether scan is passive (not allowed to hit the domain). - isPassive: bool - - # Whether scan is global. Global scans run once for all organizations, as opposed - # to non-global scans, which are run for each organization. - global_scan: bool - - # CPU and memory for the scan. See this page for more information: - # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html - cpu: Optional[str] = None - memory: Optional[str] = None - - # A scan is "chunked" if its work is divided and run in parallel by multiple workers. - # To make a scan chunked, make sure it is a global scan and specify the "numChunks" variable, - # which corresponds to the number of workers that will be created to run the task. - # Chunked scans can only be run on scans whose implementation takes into account the - # chunkNumber and numChunks parameters specified in commandOptions. - numChunks: Optional[int] = None - -class GetScansResponseModel(BaseModel): - """Get Scans response model.""" - scans: List[Scan] - schema: Dict[str, Any] - organizations: List[Dict[str, Any]] - -class GetGranularScansResponseModel(BaseModel): - """Get Scans response model.""" - scans: List[Scan] - schema: Dict[str, Any] - -class IdSchema(BaseModel): - """Schema for ID objects.""" - id: UUID - -class CreateScan(BaseModel): - """Create Scan Schema.""" - name: str - arguments: Any - organizations: Optional[List[UUID]] - tags: Optional[List[IdSchema]] - frequency: int - frequencyUnit: str - isGranular: bool - isUserModifiable: Optional[bool] - isSingleScan: bool - -class CreateScanResponseModel(BaseModel): - """Create Scan Schema.""" - name: str - arguments: Any - frequency: int - isGranular: bool - isUserModifiable: Optional[bool] - isSingleScan: bool - createdBy: Optional[Any] - class ScanTask(BaseModel): """ScanTask schema.""" @@ -303,7 +225,6 @@ class ScanTask(BaseModel): organizationId: Optional[Any] scanId: Optional[Any] - class SearchBody(BaseModel): """SearchBody schema.""" diff --git a/backend/src/xfd_django/xfd_api/tasks/__init__.py b/backend/src/xfd_django/xfd_api/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/tasks/ecs_client.py b/backend/src/xfd_django/xfd_api/tasks/ecs_client.py new file mode 100644 index 00000000..a49def57 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tasks/ecs_client.py @@ -0,0 +1,166 @@ +import os +import json +import boto3 +import docker +from ..schemas import SCAN_SCHEMA + + +def to_snake_case(input_str): + """Converts a string to snake_case.""" + return input_str.replace(' ', '-') + + +class ECSClient: + def __init__(self, is_local=None): + # Determine if we're running locally or using ECS + self.is_local = is_local or os.getenv('IS_OFFLINE') or os.getenv('IS_LOCAL') + self.docker = docker.from_env() if self.is_local else None + self.ecs = boto3.client('ecs') if not self.is_local else None + self.cloudwatch_logs = boto3.client('logs') if not self.is_local else None + + async def run_command(self, command_options): + """Launches an ECS task or Docker container with the given command options.""" + scan_id = command_options['scanId'] + scan_name = command_options['scanName'] + num_chunks = command_options.get('numChunks') + chunk_number = command_options.get('chunkNumber') + scan_schema = SCAN_SCHEMA.get(scan_name, {}) + cpu = getattr(scan_schema, 'cpu', None) + memory = getattr(scan_schema, 'memory', None) + global_scan = getattr(scan_schema, 'global_scan', False) + # These properties are not specified when creating a ScanTask (as a single ScanTask + # can correspond to multiple organizations), but they are input into the the + # specific task function that runs per organization. + organization_id = command_options.get('organizationId') + organization_name = command_options.get('organizationName') + + + if self.is_local: + # Run the command in a local Docker container + try: + container_name = to_snake_case( + f"crossfeed_worker_{'global' if global_scan else organization_name}_{scan_name}_{int(os.urandom(4).hex(), 16)}" + ) + container = self.docker.containers.run( + 'crossfeed-worker', + name=container_name, + network_mode='xfd_backend', + mem_limit='4g', + environment={ + 'CROSSFEED_COMMAND_OPTIONS': json.dumps(command_options), + 'CF_API_KEY': os.getenv('CF_API_KEY'), + 'PE_API_KEY': os.getenv('PE_API_KEY'), + 'DB_DIALECT': os.getenv('DB_DIALECT'), + 'DB_HOST': os.getenv('DB_HOST'), + 'IS_LOCAL': 'true', + 'DB_PORT': os.getenv('DB_PORT'), + 'DB_NAME': os.getenv('DB_NAME'), + 'DB_USERNAME': os.getenv('DB_USERNAME'), + 'DB_PASSWORD': os.getenv('DB_PASSWORD'), + 'MDL_NAME': os.getenv('MDL_NAME'), + 'MDL_USERNAME': os.getenv('MDL_USERNAME'), + 'MDL_PASSWORD': os.getenv('MDL_PASSWORD'), + 'MI_ACCOUNT_NAME': os.getenv('MI_ACCOUNT_NAME'), + 'MI_PASSWORD': os.getenv('MI_PASSWORD'), + 'PE_DB_NAME': os.getenv('PE_DB_NAME'), + 'PE_DB_USERNAME': os.getenv('PE_DB_USERNAME'), + 'PE_DB_PASSWORD': os.getenv('PE_DB_PASSWORD'), + 'CENSYS_API_ID': os.getenv('CENSYS_API_ID'), + 'CENSYS_API_SECRET': os.getenv('CENSYS_API_SECRET'), + 'WORKER_USER_AGENT': os.getenv('WORKER_USER_AGENT'), + 'SHODAN_API_KEY': os.getenv('SHODAN_API_KEY'), + 'SIXGILL_CLIENT_ID': os.getenv('SIXGILL_CLIENT_ID'), + 'SIXGILL_CLIENT_SECRET': os.getenv('SIXGILL_CLIENT_SECRET'), + 'PE_SHODAN_API_KEYS': os.getenv('PE_SHODAN_API_KEYS'), + 'WORKER_SIGNATURE_PUBLIC_KEY': os.getenv('WORKER_SIGNATURE_PUBLIC_KEY'), + 'WORKER_SIGNATURE_PRIVATE_KEY': os.getenv('WORKER_SIGNATURE_PRIVATE_KEY'), + 'ELASTICSEARCH_ENDPOINT': os.getenv('ELASTICSEARCH_ENDPOINT'), + 'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'), + 'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'), + 'LG_API_KEY': os.getenv('LG_API_KEY'), + 'LG_WORKSPACE_NAME': os.getenv('LG_WORKSPACE_NAME') + }, + detach=True + ) + return { + 'tasks': [{'taskArn': container.name}], + 'failures': [] + } + except Exception as e: + print(e) + return { + 'tasks': [], + 'failures': [{}] + } + + # Run the command on ECS + tags = [ + {'key': 'scanId', 'value': scan_id}, + {'key': 'scanName', 'value': scan_name} + ] + if organization_name and organization_id: + tags.append({'key': 'organizationId', 'value': organization_id}) + tags.append({'key': 'organizationName', 'value': organization_name}) + if num_chunks is not None and chunk_number is not None: + tags.append({'key': 'numChunks', 'value': str(num_chunks)}) + tags.append({'key': 'chunkNumber', 'value': str(chunk_number)}) + + response = self.ecs.run_task( + cluster=os.getenv('FARGATE_CLUSTER_NAME'), + taskDefinition=os.getenv('FARGATE_TASK_DEFINITION_NAME'), + networkConfiguration={ + 'awsvpcConfiguration': { + 'assignPublicIp': 'ENABLED', + 'securityGroups': [os.getenv('FARGATE_SG_ID')], + 'subnets': [os.getenv('FARGATE_SUBNET_ID')] + } + }, + platformVersion='1.4.0', + launchType='FARGATE', + overrides={ + 'cpu': cpu, + 'memory': memory, + 'containerOverrides': [ + { + 'name': 'main', # Name from task definition + 'environment': [ + { + 'name': 'CROSSFEED_COMMAND_OPTIONS', + 'value': json.dumps(command_options) + }, + { + 'name': 'NODE_OPTIONS', + 'value': f"--max_old_space_size={memory}" if memory else '' + } + ] + } + ] + } + ) + return response + + async def get_logs(self, fargate_task_arn): + """Gets logs for a specific Fargate or Docker task.""" + if self.is_local: + log_stream = self.docker.containers.get(fargate_task_arn).logs(stdout=True, stderr=True, timestamps=True) + return ''.join(line[8:] for line in log_stream.split('\n')) + else: + log_stream_name = f"worker/main/{fargate_task_arn.split('/')[-1]}" + response = self.cloudwatch_logs.get_log_events( + logGroupName=os.getenv('FARGATE_LOG_GROUP_NAME'), + logStreamName=log_stream_name, + startFromHead=True + ) + events = response['events'] + return '\n'.join(f"{event['timestamp']} {event['message']}" for event in events) + + async def get_num_tasks(self): + """Retrieves the number of running tasks associated with the Fargate worker.""" + if self.is_local: + containers = self.docker.containers.list(filters={'ancestor': 'crossfeed-worker'}) + return len(containers) + tasks = self.ecs.list_tasks( + cluster=os.getenv('FARGATE_CLUSTER_NAME'), + launchType='FARGATE' + ) + return len(tasks.get('taskArns', [])) diff --git a/backend/src/xfd_django/xfd_api/tasks/lambda_client.py b/backend/src/xfd_django/xfd_api/tasks/lambda_client.py new file mode 100644 index 00000000..e88116d8 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tasks/lambda_client.py @@ -0,0 +1,28 @@ +import os +import boto3 +from .scheduler import handler as scheduler + +class LambdaClient: + def __init__(self): + # Determine if running locally or not + self.is_local = os.getenv('IS_OFFLINE') or os.getenv('IS_LOCAL') + if not self.is_local: + # Initialize Boto3 Lambda client only if not local + self.lambda_client = boto3.client('lambda', region_name=os.getenv('AWS_REGION', 'us-east-1')) + + async def run_command(self, name: str): + """Invokes a lambda function with the given name.""" + + print(f"Invoking lambda function: {name}") + if self.is_local: + # If running locally, directly call the scheduler function + await scheduler({}) + return {"status": 202, "message": ""} + else: + # Invoke the lambda function asynchronously + response = self.lambda_client.invoke( + FunctionName=name, + InvocationType='Event', + Payload='' + ) + return response diff --git a/backend/src/xfd_django/xfd_api/tasks/scheduler.py b/backend/src/xfd_django/xfd_api/tasks/scheduler.py new file mode 100644 index 00000000..09a501b8 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tasks/scheduler.py @@ -0,0 +1,246 @@ +import os +from django.utils import timezone +from ..models import Scan, Organization, ScanTask +from .ecs_client import ECSClient +from ..schemas import SCAN_SCHEMA +from ..helpers.getScanOrganizations import get_scan_organizations +from itertools import islice + +def chunk(iterable, size): + it = iter(iterable) + return iter(lambda: list(islice(it, size)), []) + +class Scheduler: + def __init__(self): + self.ecs = ECSClient() + self.num_existing_tasks = 0 + self.num_launched_tasks = 0 + self.max_concurrent_tasks = int(os.getenv('FARGATE_MAX_CONCURRENCY', '10')) + self.scans = [] + self.organizations = [] + self.queued_scan_tasks = [] + self.orgs_per_scan_task = 1 + + async def initialize(self, scans, organizations, queued_scan_tasks, orgs_per_scan_task): + self.scans = scans + self.organizations = organizations + self.queued_scan_tasks = queued_scan_tasks + self.orgs_per_scan_task = orgs_per_scan_task + self.num_existing_tasks = await self.ecs.get_num_tasks() + + print(f'Number of running Fargate tasks: {self.num_existing_tasks}') + print(f'Number of queued scan tasks: {len(self.queued_scan_tasks)}') + + async def launch_single_scan_task(self, organizations=None, scan=None, chunk_number=None, num_chunks=None, scan_task=None): + organizations = organizations or [] + scan_schema = SCAN_SCHEMA.get(scan.name, {}) + task_type = getattr(scan_schema, 'type', None) + global_scan = getattr(scan_schema, 'global_scan', None) + + scan_task = scan_task or ScanTask.objects.create( + scanId=scan, + type=task_type, + status='created' + ) + + # Set the many-to-many relationship with organizations + if not global_scan: + scan_task.organizations.set(organizations) + + command_options = scan_task.input or { + 'organizations': [{'name': org.name, 'id': str(org.id)} for org in organizations], + 'scanId': str(scan.id), + 'scanName': scan.name, + 'scanTaskId': str(scan_task.id), + 'numChunks': num_chunks, + 'chunkNumber': chunk_number, + 'isSingleScan': scan.isSingleScan + } + + scan_task.input = command_options + + if self.reached_scan_limit(): + scan_task.status = 'queued' + if not scan_task.queuedAt: + scan_task.queuedAt = timezone.now() + print(f'Reached maximum concurrency, queueing scantask {scan_task.id}') + scan_task.save() + return + + try: + if task_type == 'fargate': + result = await self.ecs.run_command(command_options) + if not result.get('tasks'): + print(f"Failed to start Fargate task for scan {scan.name}, failures: {result.get('failures')}") + raise Exception(f"Failed to start Fargate task for scan {scan.name}") + + task_arn = result['tasks'][0]['taskArn'] + scan_task.fargateTaskArn = task_arn + print(f'Successfully invoked scan {scan.name} with Fargate on {len(organizations)} organizations. Task ARN: {task_arn}') + else: + raise Exception(f'Invalid task type: {task_type}') + + scan_task.status = 'requested' + scan_task.requestedAt = timezone.now() + self.num_launched_tasks += 1 + except Exception as error: + print(f"Error invoking {scan.name} scan: {error}") + scan_task.output = str(error) + scan_task.status = 'failed' + scan_task.finishedAt = timezone.now() + + scan_task.save() + + async def launch_scan_task(self, organizations=None, scan=None): + organizations = organizations or [] + + scan_schema = SCAN_SCHEMA.get(scan.name, None) + num_chunks = getattr(scan_schema, 'numChunks', None) + + # If num_chunks is set, handle it; otherwise, default to launching a single task + if num_chunks: + # For running locally, set num_chunks to 1 + if os.getenv('IS_LOCAL'): + num_chunks = 1 + + # Sanitize num_chunks to ensure it doesn't exceed 100 + num_chunks = min(num_chunks, 100) + + for chunk_number in range(num_chunks): + await self.launch_single_scan_task( + organizations=organizations, + scan=scan, + chunk_number=chunk_number, + num_chunks=num_chunks + ) + else: + # Launch a single scan task when num_chunks is None or 0 + await self.launch_single_scan_task(organizations=organizations, scan=scan) + + def reached_scan_limit(self): + return (self.num_existing_tasks + self.num_launched_tasks) >= self.max_concurrent_tasks + + async def run(self): + for scan in self.scans: + prev_num_launched_tasks = self.num_launched_tasks + + if scan.name not in SCAN_SCHEMA: + print(f'Invalid scan name: {scan.name}') + continue + + scan_schema = SCAN_SCHEMA[scan.name] + if scan_schema.global_scan: + if not self.should_run_scan(scan): + continue + await self.launch_scan_task(scan=scan) + else: + organizations = get_scan_organizations(scan) if scan.isGranular else self.organizations + orgs_to_launch = [org for org in organizations if self.should_run_scan(scan=scan, organization=org)] + for org_chunk in chunk(orgs_to_launch, self.orgs_per_scan_task): + await self.launch_scan_task(organizations=org_chunk, scan=scan) + + if self.num_launched_tasks > prev_num_launched_tasks: + scan.lastRun = timezone.now() + scan.manualRunPending = False + scan.save() + + async def run_queued(self): + for scan_task in self.queued_scan_tasks: + await self.launch_single_scan_task(scan_task=scan_task, scan=scan_task.scan) + + def should_run_scan(self, scan, organization=None): + scan_schema = SCAN_SCHEMA.get(scan.name, {}) + is_passive = getattr(scan_schema, 'isPassive', False) + global_scan = getattr(scan_schema, 'global_scan', False) + + # Don't run non-passive scans on passive organizations. + if organization and organization.isPassive and not is_passive: + return False + + # Always run scans that have manualRunPending set to True. + if scan.manualRunPending: + print("Manual run pending") + return True + + # Function to filter the scan tasks based on whether it's global or organization-specific. + def filter_scan_tasks(tasks): + if global_scan: + return tasks.filter(scanId=scan) + else: + return tasks.filter(scanId=scan).filter( + organizations=organization + ) | tasks.filter(organizations__id=organization.id) + + # Check if there's a currently running or queued scan task for the given scan. + last_running_scan_task = filter_scan_tasks( + ScanTask.objects.filter( + status__in=['created', 'queued', 'requested', 'started'] + ).order_by('-createdAt') + ).first() + + # If there's a running or queued task, do not run another. + if last_running_scan_task: + print("Already running or queued") + return False + + # Check for the last finished scan task. + last_finished_scan_task = filter_scan_tasks( + ScanTask.objects.filter( + status__in=['finished', 'failed'], + finishedAt__isnull=False + ).order_by('-finishedAt') + ).first() + + # If a scan task was finished recently within the scan frequency, do not run. + if last_finished_scan_task and last_finished_scan_task.finishedAt: + print("Has been run since the last scan frequency") + frequency_seconds = scan.frequency * 1000 # Assuming frequency is in seconds. + if (timezone.now() - last_finished_scan_task.finishedAt).total_seconds() < frequency_seconds: + return False + + # If the scan is marked as a single scan and has already run once, do not run again. + if last_finished_scan_task and last_finished_scan_task.finishedAt and scan.isSingleScan: + print("Single scan") + return False + + return True + +async def handler(event): + """Handler for manually invoking the scheduler to run scans.""" + print('Running scheduler...') + + scan_ids = event.get('scanIds', []) + if 'scanId' in event: + scan_ids.append(event['scanId']) + + org_ids = event.get('organizationIds', []) + + # Fetch scans based on scan_ids if provided + if scan_ids: + scans = Scan.objects.filter(id__in=scan_ids).prefetch_related('organizations', 'tags') + else: + scans = Scan.objects.all().prefetch_related('organizations', 'tags') + + # Fetch organizations based on org_ids if provided + if org_ids: + organizations = Organization.objects.filter(id__in=org_ids) + else: + organizations = Organization.objects.all() + + queued_scan_tasks = ScanTask.objects.filter( + scanId__in=scan_ids, + status='queued' + ).order_by('queuedAt').select_related('scanId') + + scheduler = Scheduler() + await scheduler.initialize( + scans=scans, + organizations=organizations, + queued_scan_tasks=queued_scan_tasks, + orgs_per_scan_task=event.get('orgsPerScanTask') or int(os.getenv('SCHEDULER_ORGS_PER_SCANTASK', '1')) + ) + + await scheduler.run_queued() + await scheduler.run() + + print('Finished running scheduler.') \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 5b8b34af..1191385e 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -17,6 +17,15 @@ - .models """ +# cisagov Libraries +from . import schemas +from .auth import get_current_active_user +from .models import ApiKey, Cpe, Cve, Domain, Organization, Role, User, Vulnerability +from .schemas import Role as RoleSchema +from .schemas import User as UserSchema +from .api_methods import scans +from .schema_models import scan as scanSchema + # Standard Python Libraries from typing import Any, List, Optional, Union @@ -25,13 +34,6 @@ from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.security import APIKeyHeader, OAuth2PasswordBearer -# from .schemas import Cpe -from . import schemas -from .auth import get_current_active_user -from .models import ApiKey, Cpe, Cve, Domain, Organization, Role, User, Vulnerability -from .schemas import Role as RoleSchema -from .schemas import User as UserSchema -from . import scans api_router = APIRouter() @@ -302,118 +304,101 @@ async def get_users( return [UserSchema.from_orm(user) for user in users] -############################################################################## +# ======================================== +# Scan Endpoints +# ======================================== + @api_router.get( "/scans", dependencies=[Depends(get_current_active_user)], - response_model=schemas.GetScansResponseModel, + response_model=scanSchema.GetScansResponseModel, tags=["Scans"], ) async def list_scans(current_user: User = Depends(get_current_active_user)): """Retrieve a list of all scans.""" - return scans.list_scans() + return scans.list_scans(current_user) @api_router.get( "/granularScans", dependencies=[Depends(get_current_active_user)], - # response_model=schemas.GetGranularScansResponseModel, + response_model=scanSchema.GetGranularScansResponseModel, tags=["Scans"], ) async def list_granular_scans(current_user: User = Depends(get_current_active_user)): """Retrieve a list of granular scans. User must be authenticated.""" - return scans.list_granular_scans() + return scans.list_granular_scans(current_user) @api_router.post( "/scans", dependencies=[Depends(get_current_active_user)], - # response_model=schemas.CreateScanResponseModel, + response_model=scanSchema.CreateScanResponseModel, tags=["Scans"], ) async def create_scan( - scan_data: schemas.CreateScan, current_user: User = Depends(get_current_active_user) + scan_data: schemas.NewScan, current_user: User = Depends(get_current_active_user) ): """ Create a new scan.""" return scans.create_scan(scan_data, current_user) -@api_router.get("/scans/{scan_id}", response_model=schemas.Scan) +@api_router.get( + "/scans/{scan_id}", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.GetScanResponseModel, + tags=["Scans"] +) async def get_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): - """ - Endpoint to retrieve a scan by its ID. User must be authenticated. - - Args: - scan_id (str): The ID of the scan to retrieve. - current_user (User): The authenticated user, injected via Depends. - - Returns: - The scan object. - """ - return scans.get_scan(scan_id) + """Get a scan by its ID. User must be authenticated.""" + return scans.get_scan(scan_id, current_user) -@api_router.put("/scans/{scan_id}", response_model=schemas.Scan) +@api_router.put( + "/scans/{scan_id}", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.CreateScanResponseModel, + tags=["Scans"] +) async def update_scan( scan_id: str, - scan_data: schemas.Scan, + scan_data: scanSchema.NewScan, current_user: User = Depends(get_current_active_user), ): - """ - Endpoint to update a scan by its ID. User must be authenticated. - - Args: - scan_id (str): The ID of the scan to update. - scan_data (ScanUpdate): The updated scan data. - current_user (User): The authenticated user, injected via Depends. - - Returns: - The updated scan object. - """ - return scans.update_scan(scan_id, scan_data) + """Update a scan by its ID.""" + return scans.update_scan(scan_id, scan_data, current_user) -@api_router.delete("/scans/{scan_id}") +@api_router.delete( + "/scans/{scan_id}", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.GenericMessageResponseModel, + tags=["Scans"] +) async def delete_scan( scan_id: str, current_user: User = Depends(get_current_active_user) ): - """ - Endpoint to delete a scan by its ID. User must be authenticated. - - Args: - scan_id (str): The ID of the scan to delete. - current_user (User): The authenticated user, injected via Depends. + """Delete a scan by its ID.""" + return scans.delete_scan(scan_id, current_user) - Returns: - A confirmation message. - """ - return scans.delete_scan(scan_id) - -@api_router.post("/scans/{scan_id}/run") +@api_router.post( + "/scans/{scan_id}/run", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.GenericMessageResponseModel, + tags=["Scans"] +) async def run_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): - """ - Endpoint to manually run a scan by its ID. User must be authenticated. + """Manually run a scan by its ID""" + return scans.run_scan(scan_id, current_user) - Args: - scan_id (str): The ID of the scan to run. - current_user (User): The authenticated user, injected via Depends. - - Returns: - A confirmation message that the scan has been triggered. - """ - return scans.run_scan(scan_id) - -@api_router.post("/scheduler/invoke") +@api_router.post( + "/scheduler/invoke", + dependencies=[Depends(get_current_active_user)], + tags=["Scans"] +) async def invoke_scheduler(current_user: User = Depends(get_current_active_user)): - """ - Endpoint to manually invoke the scan scheduler. User must be authenticated. - - Args: - current_user (User): The authenticated user, injected via Depends. - - Returns: - A confirmation message that the scheduler was invoked. - """ - return scans.invoke_scheduler() + """Manually invoke the scan scheduler.""" + response = await scans.invoke_scheduler(current_user) + return response From 41e593b9afa5e1bfa9e75e5c9d4d24adcf5ae164 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 26 Sep 2024 11:29:09 -0400 Subject: [PATCH 026/314] Cleanup for PR --- backend/src/api/scans.ts | 2 - .../xfd_api/api_methods/{scans.py => scan.py} | 17 +- backend/src/xfd_django/xfd_api/auth.py | 65 +----- .../xfd_django/xfd_api/helpers/__init__.py | 1 + .../xfd_api/helpers/getScanOrganizations.py | 8 +- backend/src/xfd_django/xfd_api/models.py | 56 +---- .../xfd_api/schema_models/__init__.py | 1 + .../xfd_api/schema_models/organization.py | 2 +- .../xfd_api/schema_models/organization_tag.py | 3 +- .../xfd_django/xfd_api/schema_models/scan.py | 2 +- .../src/xfd_django/xfd_api/tasks/__init__.py | 1 + docker-compose.yml | 197 +++++++++--------- frontend/src/pages/Scans/ScansView.tsx | 9 - 13 files changed, 135 insertions(+), 229 deletions(-) rename backend/src/xfd_django/xfd_api/api_methods/{scans.py => scan.py} (97%) diff --git a/backend/src/api/scans.ts b/backend/src/api/scans.ts index f1d95070..79d20483 100644 --- a/backend/src/api/scans.ts +++ b/backend/src/api/scans.ts @@ -348,8 +348,6 @@ export const update = wrapHandler(async (event) => { * - Scans */ export const create = wrapHandler(async (event) => { - console.log(event); - console.log(event.body); if (!isGlobalWriteAdmin(event)) return Unauthorized; await connectToDatabase(); const body = await validateBody(NewScan, event.body); diff --git a/backend/src/xfd_django/xfd_api/api_methods/scans.py b/backend/src/xfd_django/xfd_api/api_methods/scan.py similarity index 97% rename from backend/src/xfd_django/xfd_api/api_methods/scans.py rename to backend/src/xfd_django/xfd_api/api_methods/scan.py index ce517183..56bb9735 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/scans.py +++ b/backend/src/xfd_django/xfd_api/api_methods/scan.py @@ -1,13 +1,16 @@ +"""API methods to support Scan enpoints.""" + +# cisagov Libraries +from ..auth import is_global_write_admin, is_global_view_admin from ..models import Scan, Organization, OrganizationTag from ..schemas import SCAN_SCHEMA, NewScan -from ..auth import is_global_write_admin, is_global_view_admin -from django.db import transaction -from fastapi import HTTPException -from django.http import JsonResponse -from django.forms.models import model_to_dict -from django.core.serializers.json import DjangoJSONEncoder +from ..tasks.lambda_client import LambdaClient + +# Standard Python Libraries import os -from ..lambda_client import LambdaClient + +# Third-Party Libraries +from fastapi import HTTPException def list_scans(current_user): diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index e4fc2487..b6722278 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -1,23 +1,9 @@ -""" -This module provides authentication utilities for the FastAPI application. - -It includes functions to: -- Decode JWT tokens and retrieve the current user. -- Retrieve a user by their API key. -- Ensure the current user is authenticated and active. - -Functions: - - get_current_user: Decodes a JWT token to retrieve the current user. - - get_user_by_api_key: Retrieves a user by their API key. - - get_current_active_user: Ensures the current user is authenticated and active. - -Dependencies: - - fastapi - - django - - hashlib - - .jwt_utils - - .models -""" +"""Authentication utilities for the FastAPI application.""" + +# cisagov Libraries +from .jwt_utils import decode_jwt_token +from .models import ApiKey + # Standard Python Libraries from hashlib import sha256 @@ -26,26 +12,12 @@ from fastapi import Depends, HTTPException, Security, status from fastapi.security import APIKeyHeader, OAuth2PasswordBearer -from .jwt_utils import decode_jwt_token -from .models import ApiKey - oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) def get_current_user(token: str = Depends(oauth2_scheme)): - """ - Decode a JWT token to retrieve the current user. - - Args: - token (str): The JWT token. - - Raises: - HTTPException: If the token is invalid or expired. - - Returns: - User: The user object decoded from the token. - """ + """Decode a JWT token to retrieve the current user.""" user = decode_jwt_token(token) if user is None: raise HTTPException( @@ -56,15 +28,7 @@ def get_current_user(token: str = Depends(oauth2_scheme)): def get_user_by_api_key(api_key: str): - """ - Retrieve a user by their API key. - - Args: - api_key (str): The API key. - - Returns: - User: The user object associated with the API key, or None if not found. - """ + """Get a user by their API key.""" hashed_key = sha256(api_key.encode()).hexdigest() try: api_key_instance = ApiKey.objects.get(hashedKey=hashed_key) @@ -81,18 +45,7 @@ def get_current_active_user( api_key: str = Security(api_key_header), # token: str = Depends(oauth2_scheme), ): - """ - Ensure the current user is authenticated and active. - - Args: - api_key (str): The API key. - - Raises: - HTTPException: If the user is not authenticated. - - Returns: - User: The authenticated user object. - """ + """Ensure the current user is authenticated and active.""" user = None if api_key: user = get_user_by_api_key(api_key) diff --git a/backend/src/xfd_django/xfd_api/helpers/__init__.py b/backend/src/xfd_django/xfd_api/helpers/__init__.py index e69de29b..0f3d7da2 100644 --- a/backend/src/xfd_django/xfd_api/helpers/__init__.py +++ b/backend/src/xfd_django/xfd_api/helpers/__init__.py @@ -0,0 +1 @@ +"""Initialize helpers directory.""" \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py b/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py index 218d9e5f..d5241f85 100644 --- a/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py +++ b/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py @@ -1,6 +1,12 @@ -from typing import List +"""Get scan organizations methods.""" + +# cisagov Libraries from ..models import Organization, Scan +# Standard Python Libraries +from typing import List + + def get_scan_organizations(scan: Scan) -> List[Organization]: """ Returns the organizations that a scan should be run on. diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 606e6bd7..424fe4ce 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -1,11 +1,6 @@ -""" Django ORM models """ -# This is an auto-generated Django model module. -# You'll have to do the following manually to clean this up: -# * Rearrange models' order -# * Make sure each model has one field with primary_key=True -# * Make sure each ForeignKey and OneToOneField has `on_delete` set to the desired behavior -# * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table -# Feel free to rename the models, but don't rename db_table values or field names. +""" Django ORM models.""" + + # Third-Party Libraries from django.db import models from django.contrib.postgres.fields import ArrayField, JSONField @@ -270,7 +265,7 @@ class Organization(models.Model): createdById = models.ForeignKey( "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True ) - # TODO: Get rid of this, don't need Many To Many in both tables + # TODO: Consider geting rid of this, don't need Many To Many in both tables # Relationships with other models (Scan, OrganizationTag) granularScans = models.ManyToManyField('Scan', related_name='organizations', db_table="scan_organizations_organization") tags = models.ManyToManyField('OrganizationTag', related_name='organizations', db_table="organization_tag_organizations_organization") @@ -489,31 +484,6 @@ class Meta: db_table = "scan" -# class ScanOrganizationsOrganization(models.Model): -# """The ScanOrganizationsOrganization model.""" - -# scanId = models.ForeignKey('Scan', on_delete=models.CASCADE, db_column="scanId", primary_key=True) -# organizationId = models.ForeignKey('Organization', on_delete=models.CASCADE, db_column="organizationId", primary_key=True) - -# class Meta: -# db_table = "scan_organizations_organization" -# unique_together = (("scanId", "organizationId"),) -# # Do not create an id column automatically, treat both columns as composite primary keys -# auto_created = True - - -# class ScanTagsOrganizationTag(models.Model): -# """Intermediary model for the Many-to-Many relationship between Scan and OrganizationTag.""" - -# scanId = models.ForeignKey('Scan', on_delete=models.CASCADE, db_column="scanId", primary_key=True) -# organizationTagId = models.ForeignKey('OrganizationTag', on_delete=models.CASCADE, db_column="organizationTagId", primary_key=True) - -# class Meta: -# db_table = "scan_tags_organization_tag" -# unique_together = (("scanId", "organizationTagId"),) -# auto_created = True - - class ScanTask(models.Model): """The ScanTask model.""" @@ -545,24 +515,6 @@ class Meta: db_table = "scan_task" -# class ScanTaskOrganizationsOrganization(models.Model): -# """The ScanTaskOrganizationsOrganization model.""" - -# scanTaskId = models.OneToOneField( -# ScanTask, models.DO_NOTHING, db_column="scanTaskId", primary_key=True -# ) # The composite primary key (scanTaskId, organizationId) found, that is not supported. The first column is selected. -# organizationId = models.ForeignKey( -# Organization, models.DO_NOTHING, db_column="organizationId" -# ) - -# class Meta: -# """The Meta class for ScanTaskOrganizationsOrganization.""" - -# managed = False -# db_table = "scan_task_organizations_organization" -# unique_together = (("scanTaskId", "organizationId"),) - - class Service(models.Model): """The Service model.""" diff --git a/backend/src/xfd_django/xfd_api/schema_models/__init__.py b/backend/src/xfd_django/xfd_api/schema_models/__init__.py index e69de29b..6d78daca 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/__init__.py +++ b/backend/src/xfd_django/xfd_api/schema_models/__init__.py @@ -0,0 +1 @@ +"""Initialize schema directory.""" \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization.py b/backend/src/xfd_django/xfd_api/schema_models/organization.py index 3a659537..23a28e5d 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/organization.py +++ b/backend/src/xfd_django/xfd_api/schema_models/organization.py @@ -2,7 +2,7 @@ # Standard Python Libraries from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import List, Optional from uuid import UUID # Third-Party Libraries diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py b/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py index a8463588..3518f041 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py +++ b/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py @@ -2,11 +2,10 @@ # Standard Python Libraries from datetime import datetime -from typing import Any, Dict, List, Optional from uuid import UUID # Third-Party Libraries -from pydantic import BaseModel, Json +from pydantic import BaseModel class OrganizationalTags(BaseModel): """Organization Tags.""" diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan.py b/backend/src/xfd_django/xfd_api/schema_models/scan.py index 4fbb59eb..97490cbc 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/scan.py +++ b/backend/src/xfd_django/xfd_api/schema_models/scan.py @@ -10,7 +10,7 @@ from uuid import UUID # Third-Party Libraries -from pydantic import BaseModel, Json +from pydantic import BaseModel class Scan(BaseModel): """Scan schema reflecting model.""" diff --git a/backend/src/xfd_django/xfd_api/tasks/__init__.py b/backend/src/xfd_django/xfd_api/tasks/__init__.py index e69de29b..d82af8ec 100644 --- a/backend/src/xfd_django/xfd_api/tasks/__init__.py +++ b/backend/src/xfd_django/xfd_api/tasks/__init__.py @@ -0,0 +1 @@ +"""Initialize tasks directory.""" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4bcc22b8..1f936475 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,7 @@ services: dockerfile: ./Dockerfile.python volumes: - ./backend:/app/src + - /var/run/docker.sock:/var/run/docker.sock networks: - backend ports: @@ -57,109 +58,109 @@ services: depends_on: - db - # minio: - # image: bitnami/minio:2020.9.26 - # user: root - # command: 'minio server /data' - # networks: - # - backend - # volumes: - # - ./minio-data:/data - # ports: - # - 9000:9000 - # environment: - # - MINIO_ACCESS_KEY=aws_access_key - # - MINIO_SECRET_KEY=aws_secret_key - # logging: - # driver: json-file + minio: + image: bitnami/minio:2020.9.26 + user: root + command: 'minio server /data' + networks: + - backend + volumes: + - ./minio-data:/data + ports: + - 9000:9000 + environment: + - MINIO_ACCESS_KEY=aws_access_key + - MINIO_SECRET_KEY=aws_secret_key + logging: + driver: json-file - # docs: - # build: - # context: ./ - # dockerfile: ./Dockerfile.docs - # volumes: - # - ./docs/src:/app/docs/src - # - ./docs/gatsby-browser.js:/app/docs/gatsby-browser.js - # - ./docs/gatsby-config.js:/app/docs/gatsby-config.js - # - ./docs/gatsby-node.js:/app/docs/gatsby-node.js - # ports: - # - '4000:4000' - # - '44475:44475' + docs: + build: + context: ./ + dockerfile: ./Dockerfile.docs + volumes: + - ./docs/src:/app/docs/src + - ./docs/gatsby-browser.js:/app/docs/gatsby-browser.js + - ./docs/gatsby-config.js:/app/docs/gatsby-config.js + - ./docs/gatsby-node.js:/app/docs/gatsby-node.js + ports: + - '4000:4000' + - '44475:44475' - # es: - # image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0 - # environment: - # - discovery.type=single-node - # - bootstrap.memory_lock=true - # - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' - # command: ['elasticsearch', '-Elogger.level=WARN'] - # networks: - # - backend - # ulimits: - # memlock: - # soft: -1 - # hard: -1 - # volumes: - # - es-data:/usr/share/elasticsearch/data - # ports: - # - 9200:9200 - # - 9300:9300 - # logging: - # driver: none + es: + image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0 + environment: + - discovery.type=single-node + - bootstrap.memory_lock=true + - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' + command: ['elasticsearch', '-Elogger.level=WARN'] + networks: + - backend + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - es-data:/usr/share/elasticsearch/data + ports: + - 9200:9200 + - 9300:9300 + logging: + driver: none - # kib: - # image: docker.elastic.co/kibana/kibana:7.9.0 - # networks: - # - backend - # ports: - # - 5601:5601 - # environment: - # ELASTICSEARCH_URL: http://es:9200 - # ELASTICSEARCH_HOSTS: http://es:9200 - # LOGGING_QUIET: 'true' + kib: + image: docker.elastic.co/kibana/kibana:7.9.0 + networks: + - backend + ports: + - 5601:5601 + environment: + ELASTICSEARCH_URL: http://es:9200 + ELASTICSEARCH_HOSTS: http://es:9200 + LOGGING_QUIET: 'true' - # matomodb: - # image: mariadb:10.6 - # command: --max-allowed-packet=64MB - # networks: - # - backend - # volumes: - # - ./matomo-db-data:/var/lib/mysql - # environment: - # - MYSQL_ROOT_PASSWORD=password - # logging: - # driver: none + matomodb: + image: mariadb:10.6 + command: --max-allowed-packet=64MB + networks: + - backend + volumes: + - ./matomo-db-data:/var/lib/mysql + environment: + - MYSQL_ROOT_PASSWORD=password + logging: + driver: none - # matomo: - # image: matomo:3.14.1 - # user: root - # networks: - # - backend - # volumes: - # - ./matomo-data:/var/www/html - # environment: - # - MATOMO_DATABASE_HOST=matomodb - # - MATOMO_DATABASE_ADAPTER=mysql - # - MATOMO_DATABASE_TABLES_PREFIX=matomo_ - # - MATOMO_DATABASE_USERNAME=root - # - MATOMO_DATABASE_PASSWORD=password - # - MATOMO_DATABASE_DBNAME=matomo - # - MATOMO_GENERAL_PROXY_URI_HEADER=1 - # - MATOMO_GENERAL_ASSUME_SECURE_PROTOCOL=1 - # logging: - # driver: none - # rabbitmq: - # image: 'rabbitmq:3.8-management' - # ports: - # - '5672:5672' # RabbitMQ default port - # - '15672:15672' # RabbitMQ management plugin - # networks: - # - backend - # environment: - # RABBITMQ_DEFAULT_USER: guest - # RABBITMQ_DEFAULT_PASS: guest - # volumes: - # - rabbitmq-data:/var/lib/rabbitmq + matomo: + image: matomo:3.14.1 + user: root + networks: + - backend + volumes: + - ./matomo-data:/var/www/html + environment: + - MATOMO_DATABASE_HOST=matomodb + - MATOMO_DATABASE_ADAPTER=mysql + - MATOMO_DATABASE_TABLES_PREFIX=matomo_ + - MATOMO_DATABASE_USERNAME=root + - MATOMO_DATABASE_PASSWORD=password + - MATOMO_DATABASE_DBNAME=matomo + - MATOMO_GENERAL_PROXY_URI_HEADER=1 + - MATOMO_GENERAL_ASSUME_SECURE_PROTOCOL=1 + logging: + driver: none + rabbitmq: + image: 'rabbitmq:3.8-management' + ports: + - '5672:5672' # RabbitMQ default port + - '15672:15672' # RabbitMQ management plugin + networks: + - backend + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + volumes: + - rabbitmq-data:/var/lib/rabbitmq volumes: postgres-data: diff --git a/frontend/src/pages/Scans/ScansView.tsx b/frontend/src/pages/Scans/ScansView.tsx index df55b8ee..f255224e 100644 --- a/frontend/src/pages/Scans/ScansView.tsx +++ b/frontend/src/pages/Scans/ScansView.tsx @@ -111,15 +111,6 @@ const ScansView: React.FC = () => { body.arguments = JSON.parse(body.arguments); setFrequency(body); - // Log the body to the console for testing - console.log('Full Request Body:', { - ...body, - organizations: body.organizations - ? body.organizations.map((e) => e.value) - : [], - tags: body.tags ? body.tags.map((e) => ({ id: e.value })) : [] - }); - const scan = await apiPost('/scans', { body: { ...body, From ddd1bd1c790a0ece72f627972f154bbb575f6403 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 26 Sep 2024 11:32:07 -0400 Subject: [PATCH 027/314] Reccomment kib in docker-compose --- docker-compose.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1f936475..fbe339fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,16 +108,16 @@ services: logging: driver: none - kib: - image: docker.elastic.co/kibana/kibana:7.9.0 - networks: - - backend - ports: - - 5601:5601 - environment: - ELASTICSEARCH_URL: http://es:9200 - ELASTICSEARCH_HOSTS: http://es:9200 - LOGGING_QUIET: 'true' + # kib: + # image: docker.elastic.co/kibana/kibana:7.9.0 + # networks: + # - backend + # ports: + # - 5601:5601 + # environment: + # ELASTICSEARCH_URL: http://es:9200 + # ELASTICSEARCH_HOSTS: http://es:9200 + # LOGGING_QUIET: 'true' matomodb: image: mariadb:10.6 From 941a633861efc23d0aec2a6d2899deafcda979d7 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 26 Sep 2024 11:53:16 -0400 Subject: [PATCH 028/314] Clean up imports --- .../xfd_django/xfd_api/api_methods/scan.py | 2 +- .../xfd_django/xfd_api/schema_models/scan.py | 4 +-- .../xfd_django/xfd_api/tasks/ecs_client.py | 12 +++++++-- .../xfd_django/xfd_api/tasks/lambda_client.py | 11 +++++++- .../src/xfd_django/xfd_api/tasks/scheduler.py | 25 ++++++++++++++++--- backend/src/xfd_django/xfd_api/views.py | 20 +++++++-------- 6 files changed, 54 insertions(+), 20 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/scan.py b/backend/src/xfd_django/xfd_api/api_methods/scan.py index 56bb9735..29bb5a06 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/scan.py +++ b/backend/src/xfd_django/xfd_api/api_methods/scan.py @@ -3,7 +3,7 @@ # cisagov Libraries from ..auth import is_global_write_admin, is_global_view_admin from ..models import Scan, Organization, OrganizationTag -from ..schemas import SCAN_SCHEMA, NewScan +from ..schema_models.scan import SCAN_SCHEMA, NewScan from ..tasks.lambda_client import LambdaClient # Standard Python Libraries diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan.py b/backend/src/xfd_django/xfd_api/schema_models/scan.py index 97490cbc..0b3c44aa 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/scan.py +++ b/backend/src/xfd_django/xfd_api/schema_models/scan.py @@ -25,8 +25,8 @@ class Scan(BaseModel): isUserModifiable: Optional[bool] isSingleScan: bool manualRunPending: bool - tags: Optional[List[OrganizationalTags]] - organizations: Optional[List[Organization]] + tags: Optional[List[OrganizationalTags]] = [] + organizations: Optional[List[Organization]] = [] class ScanSchema(BaseModel): """Scan type schema.""" diff --git a/backend/src/xfd_django/xfd_api/tasks/ecs_client.py b/backend/src/xfd_django/xfd_api/tasks/ecs_client.py index a49def57..1eac7052 100644 --- a/backend/src/xfd_django/xfd_api/tasks/ecs_client.py +++ b/backend/src/xfd_django/xfd_api/tasks/ecs_client.py @@ -1,8 +1,15 @@ -import os +"""AWS Elastic Container Service Client.""" + +# cisagov Libraries +from ..schema_models.scan import SCAN_SCHEMA + +# Standard Python Libraries import json +import os + +# Third-Party Libraries import boto3 import docker -from ..schemas import SCAN_SCHEMA def to_snake_case(input_str): @@ -12,6 +19,7 @@ def to_snake_case(input_str): class ECSClient: def __init__(self, is_local=None): + """Initialize.""" # Determine if we're running locally or using ECS self.is_local = is_local or os.getenv('IS_OFFLINE') or os.getenv('IS_LOCAL') self.docker = docker.from_env() if self.is_local else None diff --git a/backend/src/xfd_django/xfd_api/tasks/lambda_client.py b/backend/src/xfd_django/xfd_api/tasks/lambda_client.py index e88116d8..10a20c39 100644 --- a/backend/src/xfd_django/xfd_api/tasks/lambda_client.py +++ b/backend/src/xfd_django/xfd_api/tasks/lambda_client.py @@ -1,9 +1,18 @@ +"""AWS Lambda Client.""" + +# cisagov Libraries +from .scheduler import handler as scheduler + +# Standard Python Libraries import os + +# Third-Party Libraries import boto3 -from .scheduler import handler as scheduler + class LambdaClient: def __init__(self): + """Initialize.""" # Determine if running locally or not self.is_local = os.getenv('IS_OFFLINE') or os.getenv('IS_LOCAL') if not self.is_local: diff --git a/backend/src/xfd_django/xfd_api/tasks/scheduler.py b/backend/src/xfd_django/xfd_api/tasks/scheduler.py index 09a501b8..d8513a78 100644 --- a/backend/src/xfd_django/xfd_api/tasks/scheduler.py +++ b/backend/src/xfd_django/xfd_api/tasks/scheduler.py @@ -1,17 +1,27 @@ -import os -from django.utils import timezone -from ..models import Scan, Organization, ScanTask +"""Scheduler method containing AWS Lambda handler.""" + + +# cisagov Libraries from .ecs_client import ECSClient -from ..schemas import SCAN_SCHEMA from ..helpers.getScanOrganizations import get_scan_organizations +from ..models import Scan, Organization, ScanTask +from ..schema_models.scan import SCAN_SCHEMA + +# Standard Python Libraries from itertools import islice +import os + +# Third-Party Libraries +from django.utils import timezone def chunk(iterable, size): + """Chunk a list into a nested list.""" it = iter(iterable) return iter(lambda: list(islice(it, size)), []) class Scheduler: def __init__(self): + """Initialize.""" self.ecs = ECSClient() self.num_existing_tasks = 0 self.num_launched_tasks = 0 @@ -22,6 +32,7 @@ def __init__(self): self.orgs_per_scan_task = 1 async def initialize(self, scans, organizations, queued_scan_tasks, orgs_per_scan_task): + """Initialize.""" self.scans = scans self.organizations = organizations self.queued_scan_tasks = queued_scan_tasks @@ -32,6 +43,7 @@ async def initialize(self, scans, organizations, queued_scan_tasks, orgs_per_sca print(f'Number of queued scan tasks: {len(self.queued_scan_tasks)}') async def launch_single_scan_task(self, organizations=None, scan=None, chunk_number=None, num_chunks=None, scan_task=None): + """Launch single scan.""" organizations = organizations or [] scan_schema = SCAN_SCHEMA.get(scan.name, {}) task_type = getattr(scan_schema, 'type', None) @@ -92,6 +104,7 @@ async def launch_single_scan_task(self, organizations=None, scan=None, chunk_num scan_task.save() async def launch_scan_task(self, organizations=None, scan=None): + """Launch scan task.""" organizations = organizations or [] scan_schema = SCAN_SCHEMA.get(scan.name, None) @@ -118,9 +131,11 @@ async def launch_scan_task(self, organizations=None, scan=None): await self.launch_single_scan_task(organizations=organizations, scan=scan) def reached_scan_limit(self): + """Check scan limit.""" return (self.num_existing_tasks + self.num_launched_tasks) >= self.max_concurrent_tasks async def run(self): + """Run scheduler.""" for scan in self.scans: prev_num_launched_tasks = self.num_launched_tasks @@ -145,10 +160,12 @@ async def run(self): scan.save() async def run_queued(self): + """Run queued scans.""" for scan_task in self.queued_scan_tasks: await self.launch_single_scan_task(scan_task=scan_task, scan=scan_task.scan) def should_run_scan(self, scan, organization=None): + """Check if the scan should run.""" scan_schema = SCAN_SCHEMA.get(scan.name, {}) is_passive = getattr(scan_schema, 'isPassive', False) global_scan = getattr(scan_schema, 'global_scan', False) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 1191385e..86de47d3 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -23,7 +23,7 @@ from .models import ApiKey, Cpe, Cve, Domain, Organization, Role, User, Vulnerability from .schemas import Role as RoleSchema from .schemas import User as UserSchema -from .api_methods import scans +from .api_methods import scan from .schema_models import scan as scanSchema # Standard Python Libraries @@ -316,7 +316,7 @@ async def get_users( ) async def list_scans(current_user: User = Depends(get_current_active_user)): """Retrieve a list of all scans.""" - return scans.list_scans(current_user) + return scan.list_scans(current_user) @api_router.get( @@ -327,7 +327,7 @@ async def list_scans(current_user: User = Depends(get_current_active_user)): ) async def list_granular_scans(current_user: User = Depends(get_current_active_user)): """Retrieve a list of granular scans. User must be authenticated.""" - return scans.list_granular_scans(current_user) + return scan.list_granular_scans(current_user) @api_router.post( @@ -337,10 +337,10 @@ async def list_granular_scans(current_user: User = Depends(get_current_active_us tags=["Scans"], ) async def create_scan( - scan_data: schemas.NewScan, current_user: User = Depends(get_current_active_user) + scan_data: scanSchema.NewScan, current_user: User = Depends(get_current_active_user) ): """ Create a new scan.""" - return scans.create_scan(scan_data, current_user) + return scan.create_scan(scan_data, current_user) @api_router.get( @@ -351,7 +351,7 @@ async def create_scan( ) async def get_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): """Get a scan by its ID. User must be authenticated.""" - return scans.get_scan(scan_id, current_user) + return scan.get_scan(scan_id, current_user) @api_router.put( @@ -366,7 +366,7 @@ async def update_scan( current_user: User = Depends(get_current_active_user), ): """Update a scan by its ID.""" - return scans.update_scan(scan_id, scan_data, current_user) + return scan.update_scan(scan_id, scan_data, current_user) @api_router.delete( @@ -379,7 +379,7 @@ async def delete_scan( scan_id: str, current_user: User = Depends(get_current_active_user) ): """Delete a scan by its ID.""" - return scans.delete_scan(scan_id, current_user) + return scan.delete_scan(scan_id, current_user) @api_router.post( @@ -390,7 +390,7 @@ async def delete_scan( ) async def run_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): """Manually run a scan by its ID""" - return scans.run_scan(scan_id, current_user) + return scan.run_scan(scan_id, current_user) @api_router.post( @@ -400,5 +400,5 @@ async def run_scan(scan_id: str, current_user: User = Depends(get_current_active ) async def invoke_scheduler(current_user: User = Depends(get_current_active_user)): """Manually invoke the scan scheduler.""" - response = await scans.invoke_scheduler(current_user) + response = await scan.invoke_scheduler(current_user) return response From 6234c30b9e9da138b077e6d734f0c88eb0151068 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 26 Sep 2024 11:57:18 -0400 Subject: [PATCH 029/314] Update schemas --- .../xfd_api/schema_models/organization.py | 5 +- backend/src/xfd_django/xfd_api/schemas.py | 345 ------------------ 2 files changed, 1 insertion(+), 349 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization.py b/backend/src/xfd_django/xfd_api/schema_models/organization.py index 23a28e5d..b690cb22 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/organization.py +++ b/backend/src/xfd_django/xfd_api/schema_models/organization.py @@ -28,7 +28,4 @@ class Organization(BaseModel): stateName: Optional[str] county: Optional[str] countyFips: Optional[int] - type: Optional[str] - - class Config: - orm_mode = True \ No newline at end of file + type: Optional[str] \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/schemas.py b/backend/src/xfd_django/xfd_api/schemas.py index c54513bc..b7a72acb 100644 --- a/backend/src/xfd_django/xfd_api/schemas.py +++ b/backend/src/xfd_django/xfd_api/schemas.py @@ -1,346 +1 @@ """Schemas.py.""" -# Third-Party Libraries -# from pydantic.types import UUID1, UUID -# Standard Python Libraries -from datetime import datetime -from typing import Any, Dict, List, Optional -from uuid import UUID - -# Third-Party Libraries -from pydantic import BaseModel, Json - - -class Assessment(BaseModel): - """Assessment schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - rscId: str - type: str - userId: Optional[Any] - - -class Category(BaseModel): - """Category schema.""" - - id: UUID - name: str - number: str - shortName: Optional[str] - - -class Cpe(BaseModel): - """Cpe schema.""" - - id: UUID - name: Optional[str] - version: Optional[str] - vendor: Optional[str] - lastSeenAt: datetime - - -class Cve(BaseModel): - """Cve schema.""" - - id: UUID - name: Optional[str] - publishedAt: datetime - modifiedAt: datetime - status: str - description: Optional[str] - cvssV2Source: Optional[str] - cvssV2Type: Optional[str] - cvssV2VectorString: Optional[str] - cvssV2BaseSeverity: Optional[str] - cvssV2ExploitabilityScore: Optional[str] - cvssV2ImpactScore: Optional[str] - cvssV3Source: Optional[str] - cvssV3Type: Optional[str] - cvssV3VectorString: Optional[str] - cvssV3BaseSeverity: Optional[str] - cvssV3ExploitabilityScore: Optional[str] - cvssV3ImpactScore: Optional[str] - cvssV4Source: Optional[str] - cvssV4Type: Optional[str] - cvssV4VectorString: Optional[str] - cvssV4BaseSeverity: Optional[str] - cvssV4ExploitabilityScore: Optional[str] - cvssV4ImpactScore: Optional[str] - weaknesses: Optional[str] - references: Optional[str] - - -class Domain(BaseModel): - """Domain schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - syncedAt: datetime - ip: str - fromRootDomain: Optional[str] - subdomainSource: Optional[str] - ipOnly: bool - reverseName: Optional[str] - name: Optional[str] - screenshot: Optional[str] - country: Optional[str] - asn: Optional[str] - cloudHosted: bool - ssl: Optional[Any] - censysCertificatesResults: Optional[dict] - trustymailResults: Optional[dict] - discoveredById: Optional[Any] - organizationId: Any - - -class DomainFilters(BaseModel): - """DomainFilters schema.""" - - ports: Optional[str] = None - service: Optional[str] = None - reverseName: Optional[str] = None - ip: Optional[str] = None - organization: Optional[str] = None - organizationName: Optional[str] = None - vulnerabilities: Optional[str] = None - tag: Optional[str] = None - - -class DomainSearch(BaseModel): - """DomainSearch schema.""" - - page: int = 1 - sort: str - order: str - filters: Optional[DomainFilters] - pageSize: Optional[int] = None - - -class Notification(BaseModel): - """Notification schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - startDatetime: Optional[datetime] - endDateTime: Optional[datetime] - maintenanceType: Optional[str] - status: Optional[str] - updatedBy: datetime - message: Optional[str] - -class Organization(BaseModel): - """Organization schema reflecting model.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - acronym: Optional[str] - name: str - rootDomains: List[str] - ipBlocks: List[str] - isPassive: bool - pendingDomains: Optional[List[dict]] - country: Optional[str] - state: Optional[str] - regionId: Optional[str] - stateFips: Optional[int] - stateName: Optional[str] - county: Optional[str] - countyFips: Optional[int] - type: Optional[str] - - class Config: - orm_mode = True - - -class Question(BaseModel): - """Question schema.""" - - id: UUID - name: str - description: str - longForm: str - number: str - categoryId: Optional[Any] - - -class Resource(BaseModel): - """Resource schema.""" - - id: UUID - description: str - name: str - type: str - url: str - - -class Response(BaseModel): - """Response schema.""" - - id: UUID - selection: str - assessmentId: Optional[Any] - questionId: Optional[Any] - - -class Role(BaseModel): - """Role schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - role: str - approved: bool - createdById: Optional[Any] - approvedById: Optional[Any] - userId: Optional[Any] - organizationId: Optional[Any] - -class OrganizationalTags(BaseModel): - """Organization Tags.""" - id: UUID - createdAt: datetime - updatedAt: datetime - name: str - - -class ScanTask(BaseModel): - """ScanTask schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - status: str - type: str - fargateTaskArn: Optional[str] - input: Optional[str] - output: Optional[str] - requestedAt: Optional[datetime] - startedAt: Optional[datetime] - finishedAt: Optional[datetime] - queuedAt: Optional[datetime] - organizationId: Optional[Any] - scanId: Optional[Any] - -class SearchBody(BaseModel): - """SearchBody schema.""" - - current: int - resultsPerPage: int - searchTerm: str - sortDirection: str - sortField: str - filters: Json[Any] - organizationId: Optional[UUID] - tagId: Optional[UUID] - - -class Service(BaseModel): - """Service schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - serviceSource: Optional[str] - port: int - service: Optional[str] - lastSeen: Optional[datetime] - banner: Optional[str] - products: Json[Any] - censysMetadata: Json[Any] - censysIpv4Results: Json[Any] - shodanResults: Json[Any] - wappalyzerResults: Json[Any] - domainId: Optional[Any] - discoveredById: Optional[Any] - - -class User(BaseModel): - """User schema.""" - - id: UUID - cognitoId: Optional[str] - loginGovId: Optional[str] - createdAt: datetime - updatedAt: datetime - firstName: str - lastName: str - fullName: str - email: str - invitePending: bool - loginBlockedByMaintenance: bool - dateAcceptedTerms: Optional[datetime] - acceptedTermsVersion: Optional[str] - lastLoggedIn: Optional[datetime] - userType: str - regionId: Optional[str] - state: Optional[str] - oktaId: Optional[str] - roles: Optional[List[Role]] = [] - - @classmethod - def from_orm(cls, obj): - # Convert roles to a list of RoleSchema before passing to Pydantic - user_dict = obj.__dict__.copy() - user_dict["roles"] = [Role.from_orm(role) for role in obj.roles.all()] - return cls(**user_dict) - - class Config: - orm_mode = True - from_attributes = True - - -class Vulnerability(BaseModel): - """Vulnerability schema.""" - - id: UUID - createdAt: datetime - updatedAt: datetime - lastSeen: datetime - title: Optional[str] - cve: Optional[str] - cwe: Optional[str] - cpe: Optional[str] - description: Optional[str] - references: Json[Any] - cvss: float - severity: Optional[str] - needsPopulation: bool - state: Optional[str] - substate: Optional[str] - source: Optional[str] - notes: Optional[str] - actions: Json[Any] - structuredData: Json[Any] - isKev: bool - domainId: UUID - serviceId: UUID - - -class VulnerabilityFilters(BaseModel): - """VulnerabilityFilters schema.""" - - id: Optional[UUID] - title: Optional[str] - domain: Optional[str] - severity: Optional[str] - cpe: Optional[str] - state: Optional[str] - substate: Optional[str] - organization: Optional[UUID] - tag: Optional[UUID] - isKev: Optional[bool] - - -class VulnerabilitySearch(BaseModel): - """VulnerabilitySearch schema.""" - - page: int - sort: Optional[str] - order: str - filters: Optional[VulnerabilityFilters] - pageSize: Optional[int] - groupBy: Optional[str] From 30162b3d80fedc6bc295bd72b5c0e94ef60c5c34 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 26 Sep 2024 12:26:21 -0400 Subject: [PATCH 030/314] run pre-commit --- backend/requirements.txt | 8 +- backend/src/xfd_django/manage.py | 6 +- backend/src/xfd_django/xfd_api/admin.py | 1 + .../xfd_django/xfd_api/api_methods/scan.py | 223 +++++++++--------- backend/src/xfd_django/xfd_api/auth.py | 12 +- .../xfd_django/xfd_api/helpers/__init__.py | 2 +- .../xfd_api/helpers/getScanOrganizations.py | 5 +- backend/src/xfd_django/xfd_api/models.py | 56 +++-- .../xfd_api/schema_models/__init__.py | 2 +- .../xfd_api/schema_models/organization_tag.py | 4 +- .../xfd_django/xfd_api/schema_models/scan.py | 30 ++- .../src/xfd_django/xfd_api/tasks/__init__.py | 2 +- .../xfd_django/xfd_api/tasks/ecs_client.py | 199 ++++++++-------- .../xfd_django/xfd_api/tasks/lambda_client.py | 15 +- .../src/xfd_django/xfd_api/tasks/scheduler.py | 184 +++++++++------ backend/src/xfd_django/xfd_api/tests.py | 1 + backend/src/xfd_django/xfd_api/views.py | 42 ++-- backend/src/xfd_django/xfd_django/settings.py | 7 +- backend/src/xfd_django/xfd_django/urls.py | 3 +- backend/src/xfd_django/xfd_django/wsgi.py | 4 +- 20 files changed, 448 insertions(+), 358 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index e67a7b63..de640b24 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,8 +1,8 @@ +boto3 +django +docker fastapi==0.111.0 mangum==0.17.0 -uvicorn==0.30.1 -django psycopg2-binary PyJWT -boto3 -docker \ No newline at end of file +uvicorn==0.30.1 diff --git a/backend/src/xfd_django/manage.py b/backend/src/xfd_django/manage.py index 1d98fa5e..1f417b9e 100755 --- a/backend/src/xfd_django/manage.py +++ b/backend/src/xfd_django/manage.py @@ -1,13 +1,15 @@ #!/usr/bin/env python """Django's command-line utility for administrative tasks.""" +# Standard Python Libraries import os import sys def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'xfd_django.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xfd_django.settings") try: + # Third-Party Libraries from django.core.management import execute_from_command_line except ImportError as exc: raise ImportError( @@ -18,5 +20,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/backend/src/xfd_django/xfd_api/admin.py b/backend/src/xfd_django/xfd_api/admin.py index 8c38f3f3..cd6162ad 100644 --- a/backend/src/xfd_django/xfd_api/admin.py +++ b/backend/src/xfd_django/xfd_api/admin.py @@ -1,3 +1,4 @@ +# Third-Party Libraries from django.contrib import admin # Register your models here. diff --git a/backend/src/xfd_django/xfd_api/api_methods/scan.py b/backend/src/xfd_django/xfd_api/api_methods/scan.py index 29bb5a06..2cb934d2 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/scan.py +++ b/backend/src/xfd_django/xfd_api/api_methods/scan.py @@ -1,62 +1,64 @@ """API methods to support Scan enpoints.""" -# cisagov Libraries -from ..auth import is_global_write_admin, is_global_view_admin -from ..models import Scan, Organization, OrganizationTag -from ..schema_models.scan import SCAN_SCHEMA, NewScan -from ..tasks.lambda_client import LambdaClient - # Standard Python Libraries import os # Third-Party Libraries from fastapi import HTTPException +from ..auth import is_global_view_admin, is_global_write_admin +from ..models import Organization, OrganizationTag, Scan +from ..schema_models.scan import SCAN_SCHEMA, NewScan +from ..tasks.lambda_client import LambdaClient + def list_scans(current_user): """List scans.""" try: # Check if the user is a GlobalViewAdmin if not is_global_view_admin(current_user): - raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) # Fetch scans and prefetch related tags - scans = Scan.objects.prefetch_related('tags').all() + scans = Scan.objects.prefetch_related("tags").all() # Fetch all organizations - organizations = Organization.objects.values('id', 'name') + organizations = Organization.objects.values("id", "name") # Convert to list of dicts with related tags scan_list = [] for scan in scans: scan_data = { - 'id': scan.id, - 'createdAt': scan.createdAt, - 'updatedAt': scan.updatedAt, - 'name': scan.name, - 'arguments': scan.arguments, - 'frequency': scan.frequency, - 'lastRun': scan.lastRun, - 'isGranular': scan.isGranular, - 'isUserModifiable': scan.isUserModifiable, - 'isSingleScan': scan.isSingleScan, - 'manualRunPending': scan.manualRunPending, - 'tags': [ + "id": scan.id, + "createdAt": scan.createdAt, + "updatedAt": scan.updatedAt, + "name": scan.name, + "arguments": scan.arguments, + "frequency": scan.frequency, + "lastRun": scan.lastRun, + "isGranular": scan.isGranular, + "isUserModifiable": scan.isUserModifiable, + "isSingleScan": scan.isSingleScan, + "manualRunPending": scan.manualRunPending, + "tags": [ { - 'id': tag.id, - 'createdAt': tag.createdAt, - 'updatedAt': tag.updatedAt, - 'name': tag.name - } for tag in scan.tags.all() - ] + "id": tag.id, + "createdAt": tag.createdAt, + "updatedAt": tag.updatedAt, + "name": tag.name, + } + for tag in scan.tags.all() + ], } scan_list.append(scan_data) # Return response with scans, schema, and organizations response = { - 'scans': scan_list, - 'schema': SCAN_SCHEMA, - 'organizations': list(organizations) + "scans": scan_list, + "schema": SCAN_SCHEMA, + "organizations": list(organizations), } return response @@ -69,19 +71,16 @@ def list_granular_scans(current_user): try: # Check if the user is a GlobalViewAdmin if not is_global_view_admin(current_user): - raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") - + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + # Fetch scans that match the criteria (isGranular, isUserModifiable, isSingleScan) scans = Scan.objects.filter( - isGranular=True, - isUserModifiable=True, - isSingleScan=False - ).values('id', 'name', 'isUserModifiable') + isGranular=True, isUserModifiable=True, isSingleScan=False + ).values("id", "name", "isUserModifiable") - response = { - 'scans': list(scans), - 'schema': SCAN_SCHEMA - } + response = {"scans": list(scans), "schema": SCAN_SCHEMA} return response @@ -92,19 +91,21 @@ def list_granular_scans(current_user): def create_scan(scan_data: NewScan, current_user): """Create a new scan.""" try: - # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): - raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") - + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + # Check if scan name is valid if scan_data.name not in SCAN_SCHEMA: raise HTTPException(status_code=400, detail="Invalid scan name") - + # Create the scan instance - scan_data_dict = scan_data.dict(exclude_unset=True, exclude={"organizations", "tags"}) - scan_data_dict['createdBy'] = current_user - print(scan_data_dict) + scan_data_dict = scan_data.dict( + exclude_unset=True, exclude={"organizations", "tags"} + ) + scan_data_dict["createdBy"] = current_user # Create the scan object scan = Scan.objects.create(**scan_data_dict) @@ -117,22 +118,19 @@ def create_scan(scan_data: NewScan, current_user): if scan_data.tags: tag_ids = [tag.id for tag in scan_data.tags] scan.tags.set(tag_ids) - + return { - 'name': scan.name, - 'arguments': scan.arguments, - 'frequency': scan.frequency, - 'isGranular': scan.isGranular, - 'isUserModifiable': scan.isUserModifiable, - 'isSingleScan': scan.isSingleScan, - 'createdBy': { - 'id': current_user.id, - 'name': current_user.fullName - }, - 'tags': list(scan.tags.values('id')), - 'organizations': list(scan.organizations.values('id')), + "name": scan.name, + "arguments": scan.arguments, + "frequency": scan.frequency, + "isGranular": scan.isGranular, + "isUserModifiable": scan.isUserModifiable, + "isSingleScan": scan.isSingleScan, + "createdBy": {"id": current_user.id, "name": current_user.fullName}, + "tags": list(scan.tags.values("id")), + "organizations": list(scan.organizations.values("id")), } - + except Organization.DoesNotExist: raise HTTPException(status_code=404, detail="Organization not found") except OrganizationTag.DoesNotExist: @@ -140,70 +138,72 @@ def create_scan(scan_data: NewScan, current_user): except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) - def get_scan(scan_id: str, current_user): - """Get a scan by its ID. """ + """Get a scan by its ID.""" # Check if the user is a GlobalViewAdmin if not is_global_view_admin(current_user): - raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") - + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + try: # Fetch the scan with its related organizations and tags - scan = Scan.objects.prefetch_related('organizations', 'tags').get(id=scan_id) + scan = Scan.objects.prefetch_related("organizations", "tags").get(id=scan_id) # Fetch all organizations - all_organizations = Organization.objects.values('id', 'name') + all_organizations = Organization.objects.values("id", "name") except Scan.DoesNotExist: raise HTTPException(status_code=404, detail="Scan not found") - + # Get related organizations with all fields and remove unwanted fields related_organizations = list(scan.organizations.values()) for org in related_organizations: - org.pop('parentId_id', None) - org.pop('createdById_id', None) + org.pop("parentId_id", None) + org.pop("createdById_id", None) # Serialize scan data scan_data = { - 'id': str(scan.id), - 'createdAt': scan.createdAt, - 'updatedAt': scan.updatedAt, - 'name': scan.name, - 'arguments': scan.arguments, - 'lastRun': scan.lastRun, - 'frequency': scan.frequency, - 'isGranular': scan.isGranular, - 'isUserModifiable': scan.isUserModifiable, - 'isSingleScan': scan.isSingleScan, - 'manualRunPending': scan.manualRunPending, - 'organizations': related_organizations, - 'tags': list(scan.tags.values()) + "id": str(scan.id), + "createdAt": scan.createdAt, + "updatedAt": scan.updatedAt, + "name": scan.name, + "arguments": scan.arguments, + "lastRun": scan.lastRun, + "frequency": scan.frequency, + "isGranular": scan.isGranular, + "isUserModifiable": scan.isUserModifiable, + "isSingleScan": scan.isSingleScan, + "manualRunPending": scan.manualRunPending, + "organizations": related_organizations, + "tags": list(scan.tags.values()), } # Return the scan details along with its related data return { - 'scan': scan_data, - 'schema': dict(SCAN_SCHEMA[scan.name]), - 'organizations': list(all_organizations) + "scan": scan_data, + "schema": dict(SCAN_SCHEMA[scan.name]), + "organizations": list(all_organizations), } - def update_scan(scan_id: str, scan_data: NewScan, current_user): """Update a scan by its ID.""" try: # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): - raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") - + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + # Validate scan ID try: scan = Scan.objects.get(id=scan_id) except Scan.DoesNotExist: raise HTTPException(status_code=404, detail="Scan not found") - + # Update the scan's fields with the new data scan.name = scan_data.name scan.arguments = scan_data.arguments @@ -219,23 +219,20 @@ def update_scan(scan_id: str, scan_data: NewScan, current_user): if scan_data.tags: tag_ids = [tag.id for tag in scan_data.tags] scan.tags.set(tag_ids) - + # Save the updated scan scan.save() return { - 'name': scan.name, - 'arguments': scan.arguments, - 'frequency': scan.frequency, - 'isGranular': scan.isGranular, - 'isUserModifiable': scan.isUserModifiable, - 'isSingleScan': scan.isSingleScan, - 'createdBy': { - 'id': current_user.id, - 'name': current_user.fullName - }, - 'tags': list(scan.tags.values('id')), - 'organizations': list(scan.organizations.values('id')), + "name": scan.name, + "arguments": scan.arguments, + "frequency": scan.frequency, + "isGranular": scan.isGranular, + "isUserModifiable": scan.isUserModifiable, + "isSingleScan": scan.isSingleScan, + "createdBy": {"id": current_user.id, "name": current_user.fullName}, + "tags": list(scan.tags.values("id")), + "organizations": list(scan.organizations.values("id")), } except Exception as e: @@ -247,14 +244,16 @@ def delete_scan(scan_id: str, current_user): try: # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): - raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") - + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + # Validate scan ID try: scan = Scan.objects.get(id=scan_id) except Scan.DoesNotExist: raise HTTPException(status_code=404, detail="Scan not found") - + scan.delete() return {"status": "success", "message": f"Scan {scan_id} deleted successfully."} @@ -267,14 +266,16 @@ def run_scan(scan_id: str, current_user): try: # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): - raise HTTPException(status_code=403, detail="Unauthorized access. View logs for details.") - + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + # Validate the scan ID and check if it exists try: scan = Scan.objects.get(id=scan_id) except Scan.DoesNotExist: raise HTTPException(status_code=404, detail="Scan not found") - + scan.manualRunPending = True scan.save() return {"status": "success", "message": f"Scan {scan_id} deleted successfully."} @@ -285,11 +286,11 @@ def run_scan(scan_id: str, current_user): async def invoke_scheduler(current_user): """Manually invoke the scan scheduler.""" try: - #TODO: RUN THIS ON A SCHEDULE LOCALLY LIKE DEFINED IN APP.TS + # TODO: RUN THIS ON A SCHEDULE LOCALLY LIKE DEFINED IN APP.TS # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): raise HTTPException(status_code=403, detail="Unauthorized access.") - + # Initialize the Lambda client lambda_client = LambdaClient() diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index b6722278..80966dbe 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -1,9 +1,5 @@ """Authentication utilities for the FastAPI application.""" -# cisagov Libraries -from .jwt_utils import decode_jwt_token -from .models import ApiKey - # Standard Python Libraries from hashlib import sha256 @@ -12,6 +8,9 @@ from fastapi import Depends, HTTPException, Security, status from fastapi.security import APIKeyHeader, OAuth2PasswordBearer +from .jwt_utils import decode_jwt_token +from .models import ApiKey + oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) @@ -65,10 +64,12 @@ def is_global_write_admin(current_user) -> bool: """Check if the user has global write admin permissions.""" return current_user and current_user.userType == "globalAdmin" + def is_global_view_admin(current_user) -> bool: - """Check if the user has global view permissions. """ + """Check if the user has global view permissions.""" return current_user and current_user.userType in ["globalView", "globalAdmin"] + def is_regional_admin(current_user) -> bool: """Check if the user has regional admin permissions.""" return current_user and current_user.userType in ["regionalAdmin", "globalAdmin"] @@ -135,4 +136,3 @@ def is_regional_admin(current_user) -> bool: # """ # current_user = request.state.user # return current_user.id if current_user else None - diff --git a/backend/src/xfd_django/xfd_api/helpers/__init__.py b/backend/src/xfd_django/xfd_api/helpers/__init__.py index 0f3d7da2..eb8f2f47 100644 --- a/backend/src/xfd_django/xfd_api/helpers/__init__.py +++ b/backend/src/xfd_django/xfd_api/helpers/__init__.py @@ -1 +1 @@ -"""Initialize helpers directory.""" \ No newline at end of file +"""Initialize helpers directory.""" diff --git a/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py b/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py index d5241f85..c8575214 100644 --- a/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py +++ b/backend/src/xfd_django/xfd_api/helpers/getScanOrganizations.py @@ -1,11 +1,10 @@ """Get scan organizations methods.""" -# cisagov Libraries -from ..models import Organization, Scan - # Standard Python Libraries from typing import List +from ..models import Organization, Scan + def get_scan_organizations(scan: Scan) -> List[Organization]: """ diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 1193e6dc..9642d31f 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -1,10 +1,12 @@ """ Django ORM models.""" +# Standard Python Libraries +import uuid + # Third-Party Libraries -from django.db import models from django.contrib.postgres.fields import ArrayField, JSONField -import uuid +from django.db import models class ApiKey(models.Model): @@ -267,9 +269,19 @@ class Organization(models.Model): ) # TODO: Consider geting rid of this, don't need Many To Many in both tables # Relationships with other models (Scan, OrganizationTag) - granularScans = models.ManyToManyField('Scan', related_name='organizations', db_table="scan_organizations_organization") - tags = models.ManyToManyField('OrganizationTag', related_name='organizations', db_table="organization_tag_organizations_organization") - allScanTasks = models.ManyToManyField('ScanTask', related_name='organizations', db_table="scan_task_organizations_organization") + granularScans = models.ManyToManyField( + "Scan", related_name="organizations", db_table="scan_organizations_organization" + ) + tags = models.ManyToManyField( + "OrganizationTag", + related_name="organizations", + db_table="organization_tag_organizations_organization", + ) + allScanTasks = models.ManyToManyField( + "ScanTask", + related_name="organizations", + db_table="scan_task_organizations_organization", + ) class Meta: """The meta class for Organization.""" @@ -285,8 +297,14 @@ class OrganizationTag(models.Model): createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) name = models.CharField(unique=True) - organizations = models.ManyToManyField('Organization', related_name='tags', db_table="organization_tag_organizations_organization") - scans = models.ManyToManyField('Scan', related_name='tags', db_table="scan_tags_organization_tag") + organizations = models.ManyToManyField( + "Organization", + related_name="tags", + db_table="organization_tag_organizations_organization", + ) + scans = models.ManyToManyField( + "Scan", related_name="tags", db_table="scan_tags_organization_tag" + ) class Meta: """The Meta class for OrganizationTag.""" @@ -462,7 +480,9 @@ class Scan(models.Model): createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) name = models.CharField() - arguments = models.TextField(default='{}') # JSON in the database but fails: the JSON object must be str, bytes or bytearray, not dict + arguments = models.TextField( + default="{}" + ) # JSON in the database but fails: the JSON object must be str, bytes or bytearray, not dict frequency = models.IntegerField() lastRun = models.DateTimeField(db_column="lastRun", blank=True, null=True) isGranular = models.BooleanField(db_column="isGranular", default=False) @@ -474,8 +494,12 @@ class Scan(models.Model): createdBy = models.ForeignKey( "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True ) - tags = models.ManyToManyField('OrganizationTag', related_name='scans', db_table="scan_tags_organization_tag") - organizations = models.ManyToManyField('Organization', related_name='scans', db_table="scan_organizations_organization") + tags = models.ManyToManyField( + "OrganizationTag", related_name="scans", db_table="scan_tags_organization_tag" + ) + organizations = models.ManyToManyField( + "Organization", related_name="scans", db_table="scan_organizations_organization" + ) class Meta: """The Meta class for Scan.""" @@ -500,13 +524,13 @@ class ScanTask(models.Model): finishedAt = models.DateTimeField(db_column="finishedAt", blank=True, null=True) queuedAt = models.DateTimeField(db_column="queuedAt", blank=True, null=True) scanId = models.ForeignKey( - Scan, - on_delete=models.DO_NOTHING, - db_column="scanId", - blank=True, - null=True + Scan, on_delete=models.DO_NOTHING, db_column="scanId", blank=True, null=True + ) + organizations = models.ManyToManyField( + "Organization", + related_name="allScanTasks", + db_table="scan_task_organizations_organization", ) - organizations = models.ManyToManyField('Organization', related_name='allScanTasks', db_table="scan_task_organizations_organization") class Meta: """The Meta class for ScanTask.""" diff --git a/backend/src/xfd_django/xfd_api/schema_models/__init__.py b/backend/src/xfd_django/xfd_api/schema_models/__init__.py index 6d78daca..d65b3e4e 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/__init__.py +++ b/backend/src/xfd_django/xfd_api/schema_models/__init__.py @@ -1 +1 @@ -"""Initialize schema directory.""" \ No newline at end of file +"""Initialize schema directory.""" diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py b/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py index 3518f041..9935586f 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py +++ b/backend/src/xfd_django/xfd_api/schema_models/organization_tag.py @@ -7,9 +7,11 @@ # Third-Party Libraries from pydantic import BaseModel + class OrganizationalTags(BaseModel): """Organization Tags.""" + id: UUID createdAt: datetime updatedAt: datetime - name: str \ No newline at end of file + name: str diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan.py b/backend/src/xfd_django/xfd_api/schema_models/scan.py index c6a5d91f..6c6d2817 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/scan.py +++ b/backend/src/xfd_django/xfd_api/schema_models/scan.py @@ -1,9 +1,5 @@ """Schemas to support Scan endpoints.""" -# cisagov Libraries -from .organization_tag import OrganizationalTags -from.organization import Organization - # Standard Python Libraries from datetime import datetime from typing import Any, Dict, List, Optional @@ -12,8 +8,13 @@ # Third-Party Libraries from pydantic import BaseModel +from .organization import Organization +from .organization_tag import OrganizationalTags + + class Scan(BaseModel): """Scan schema reflecting model.""" + id: UUID createdAt: datetime updatedAt: datetime @@ -28,10 +29,11 @@ class Scan(BaseModel): tags: Optional[List[OrganizationalTags]] = [] organizations: Optional[List[Organization]] = [] + class ScanSchema(BaseModel): """Scan type schema.""" - - type: str = 'fargate' # Only 'fargate' is supported + + type: str = "fargate" # Only 'fargate' is supported description: str # Whether scan is passive (not allowed to hit the domain). @@ -53,29 +55,39 @@ class ScanSchema(BaseModel): # chunkNumber and numChunks parameters specified in commandOptions. numChunks: Optional[int] = None + class GranularScan(BaseModel): """Granular scan model.""" + id: UUID name: str isUserModifiable: Optional[bool] + class GetScansResponseModel(BaseModel): """Get Scans response model.""" + scans: List[Scan] schema: Dict[str, Any] organizations: List[Dict[str, Any]] + class GetGranularScansResponseModel(BaseModel): """Get Scans response model.""" + scans: List[GranularScan] schema: Dict[str, Any] + class IdSchema(BaseModel): """Schema for ID objects.""" + id: UUID + class NewScan(BaseModel): """Create Scan Schema.""" + name: str arguments: Any organizations: Optional[List[UUID]] @@ -85,8 +97,10 @@ class NewScan(BaseModel): isUserModifiable: Optional[bool] isSingleScan: bool + class CreateScanResponseModel(BaseModel): """Create Scan Schema.""" + name: str arguments: Any frequency: int @@ -97,14 +111,18 @@ class CreateScanResponseModel(BaseModel): tags: Optional[List[IdSchema]] organizations: Optional[List[IdSchema]] + class GetScanResponseModel(BaseModel): """Get Scans response model.""" + scan: Scan schema: Dict[str, Any] organizations: List[Dict[str, Any]] + class GenericMessageResponseModel(BaseModel): """Get Scans response model.""" + status: str message: str diff --git a/backend/src/xfd_django/xfd_api/tasks/__init__.py b/backend/src/xfd_django/xfd_api/tasks/__init__.py index d82af8ec..9cc66e91 100644 --- a/backend/src/xfd_django/xfd_api/tasks/__init__.py +++ b/backend/src/xfd_django/xfd_api/tasks/__init__.py @@ -1 +1 @@ -"""Initialize tasks directory.""" \ No newline at end of file +"""Initialize tasks directory.""" diff --git a/backend/src/xfd_django/xfd_api/tasks/ecs_client.py b/backend/src/xfd_django/xfd_api/tasks/ecs_client.py index 1eac7052..a2a47f7a 100644 --- a/backend/src/xfd_django/xfd_api/tasks/ecs_client.py +++ b/backend/src/xfd_django/xfd_api/tasks/ecs_client.py @@ -1,8 +1,5 @@ """AWS Elastic Container Service Client.""" -# cisagov Libraries -from ..schema_models.scan import SCAN_SCHEMA - # Standard Python Libraries import json import os @@ -11,37 +8,38 @@ import boto3 import docker +from ..schema_models.scan import SCAN_SCHEMA + def to_snake_case(input_str): """Converts a string to snake_case.""" - return input_str.replace(' ', '-') + return input_str.replace(" ", "-") class ECSClient: def __init__(self, is_local=None): """Initialize.""" # Determine if we're running locally or using ECS - self.is_local = is_local or os.getenv('IS_OFFLINE') or os.getenv('IS_LOCAL') + self.is_local = is_local or os.getenv("IS_OFFLINE") or os.getenv("IS_LOCAL") self.docker = docker.from_env() if self.is_local else None - self.ecs = boto3.client('ecs') if not self.is_local else None - self.cloudwatch_logs = boto3.client('logs') if not self.is_local else None + self.ecs = boto3.client("ecs") if not self.is_local else None + self.cloudwatch_logs = boto3.client("logs") if not self.is_local else None async def run_command(self, command_options): """Launches an ECS task or Docker container with the given command options.""" - scan_id = command_options['scanId'] - scan_name = command_options['scanName'] - num_chunks = command_options.get('numChunks') - chunk_number = command_options.get('chunkNumber') + scan_id = command_options["scanId"] + scan_name = command_options["scanName"] + num_chunks = command_options.get("numChunks") + chunk_number = command_options.get("chunkNumber") scan_schema = SCAN_SCHEMA.get(scan_name, {}) - cpu = getattr(scan_schema, 'cpu', None) - memory = getattr(scan_schema, 'memory', None) - global_scan = getattr(scan_schema, 'global_scan', False) + cpu = getattr(scan_schema, "cpu", None) + memory = getattr(scan_schema, "memory", None) + global_scan = getattr(scan_schema, "global_scan", False) # These properties are not specified when creating a ScanTask (as a single ScanTask # can correspond to multiple organizations), but they are input into the the # specific task function that runs per organization. - organization_id = command_options.get('organizationId') - organization_name = command_options.get('organizationName') - + organization_id = command_options.get("organizationId") + organization_name = command_options.get("organizationName") if self.is_local: # Run the command in a local Docker container @@ -50,125 +48,130 @@ async def run_command(self, command_options): f"crossfeed_worker_{'global' if global_scan else organization_name}_{scan_name}_{int(os.urandom(4).hex(), 16)}" ) container = self.docker.containers.run( - 'crossfeed-worker', + "crossfeed-worker", name=container_name, - network_mode='xfd_backend', - mem_limit='4g', + network_mode="xfd_backend", + mem_limit="4g", environment={ - 'CROSSFEED_COMMAND_OPTIONS': json.dumps(command_options), - 'CF_API_KEY': os.getenv('CF_API_KEY'), - 'PE_API_KEY': os.getenv('PE_API_KEY'), - 'DB_DIALECT': os.getenv('DB_DIALECT'), - 'DB_HOST': os.getenv('DB_HOST'), - 'IS_LOCAL': 'true', - 'DB_PORT': os.getenv('DB_PORT'), - 'DB_NAME': os.getenv('DB_NAME'), - 'DB_USERNAME': os.getenv('DB_USERNAME'), - 'DB_PASSWORD': os.getenv('DB_PASSWORD'), - 'MDL_NAME': os.getenv('MDL_NAME'), - 'MDL_USERNAME': os.getenv('MDL_USERNAME'), - 'MDL_PASSWORD': os.getenv('MDL_PASSWORD'), - 'MI_ACCOUNT_NAME': os.getenv('MI_ACCOUNT_NAME'), - 'MI_PASSWORD': os.getenv('MI_PASSWORD'), - 'PE_DB_NAME': os.getenv('PE_DB_NAME'), - 'PE_DB_USERNAME': os.getenv('PE_DB_USERNAME'), - 'PE_DB_PASSWORD': os.getenv('PE_DB_PASSWORD'), - 'CENSYS_API_ID': os.getenv('CENSYS_API_ID'), - 'CENSYS_API_SECRET': os.getenv('CENSYS_API_SECRET'), - 'WORKER_USER_AGENT': os.getenv('WORKER_USER_AGENT'), - 'SHODAN_API_KEY': os.getenv('SHODAN_API_KEY'), - 'SIXGILL_CLIENT_ID': os.getenv('SIXGILL_CLIENT_ID'), - 'SIXGILL_CLIENT_SECRET': os.getenv('SIXGILL_CLIENT_SECRET'), - 'PE_SHODAN_API_KEYS': os.getenv('PE_SHODAN_API_KEYS'), - 'WORKER_SIGNATURE_PUBLIC_KEY': os.getenv('WORKER_SIGNATURE_PUBLIC_KEY'), - 'WORKER_SIGNATURE_PRIVATE_KEY': os.getenv('WORKER_SIGNATURE_PRIVATE_KEY'), - 'ELASTICSEARCH_ENDPOINT': os.getenv('ELASTICSEARCH_ENDPOINT'), - 'AWS_ACCESS_KEY_ID': os.getenv('AWS_ACCESS_KEY_ID'), - 'AWS_SECRET_ACCESS_KEY': os.getenv('AWS_SECRET_ACCESS_KEY'), - 'LG_API_KEY': os.getenv('LG_API_KEY'), - 'LG_WORKSPACE_NAME': os.getenv('LG_WORKSPACE_NAME') + "CROSSFEED_COMMAND_OPTIONS": json.dumps(command_options), + "CF_API_KEY": os.getenv("CF_API_KEY"), + "PE_API_KEY": os.getenv("PE_API_KEY"), + "DB_DIALECT": os.getenv("DB_DIALECT"), + "DB_HOST": os.getenv("DB_HOST"), + "IS_LOCAL": "true", + "DB_PORT": os.getenv("DB_PORT"), + "DB_NAME": os.getenv("DB_NAME"), + "DB_USERNAME": os.getenv("DB_USERNAME"), + "DB_PASSWORD": os.getenv("DB_PASSWORD"), + "MDL_NAME": os.getenv("MDL_NAME"), + "MDL_USERNAME": os.getenv("MDL_USERNAME"), + "MDL_PASSWORD": os.getenv("MDL_PASSWORD"), + "MI_ACCOUNT_NAME": os.getenv("MI_ACCOUNT_NAME"), + "MI_PASSWORD": os.getenv("MI_PASSWORD"), + "PE_DB_NAME": os.getenv("PE_DB_NAME"), + "PE_DB_USERNAME": os.getenv("PE_DB_USERNAME"), + "PE_DB_PASSWORD": os.getenv("PE_DB_PASSWORD"), + "CENSYS_API_ID": os.getenv("CENSYS_API_ID"), + "CENSYS_API_SECRET": os.getenv("CENSYS_API_SECRET"), + "WORKER_USER_AGENT": os.getenv("WORKER_USER_AGENT"), + "SHODAN_API_KEY": os.getenv("SHODAN_API_KEY"), + "SIXGILL_CLIENT_ID": os.getenv("SIXGILL_CLIENT_ID"), + "SIXGILL_CLIENT_SECRET": os.getenv("SIXGILL_CLIENT_SECRET"), + "PE_SHODAN_API_KEYS": os.getenv("PE_SHODAN_API_KEYS"), + "WORKER_SIGNATURE_PUBLIC_KEY": os.getenv( + "WORKER_SIGNATURE_PUBLIC_KEY" + ), + "WORKER_SIGNATURE_PRIVATE_KEY": os.getenv( + "WORKER_SIGNATURE_PRIVATE_KEY" + ), + "ELASTICSEARCH_ENDPOINT": os.getenv("ELASTICSEARCH_ENDPOINT"), + "AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID"), + "AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY"), + "LG_API_KEY": os.getenv("LG_API_KEY"), + "LG_WORKSPACE_NAME": os.getenv("LG_WORKSPACE_NAME"), }, - detach=True + detach=True, ) - return { - 'tasks': [{'taskArn': container.name}], - 'failures': [] - } + return {"tasks": [{"taskArn": container.name}], "failures": []} except Exception as e: print(e) - return { - 'tasks': [], - 'failures': [{}] - } + return {"tasks": [], "failures": [{}]} # Run the command on ECS tags = [ - {'key': 'scanId', 'value': scan_id}, - {'key': 'scanName', 'value': scan_name} + {"key": "scanId", "value": scan_id}, + {"key": "scanName", "value": scan_name}, ] if organization_name and organization_id: - tags.append({'key': 'organizationId', 'value': organization_id}) - tags.append({'key': 'organizationName', 'value': organization_name}) + tags.append({"key": "organizationId", "value": organization_id}) + tags.append({"key": "organizationName", "value": organization_name}) if num_chunks is not None and chunk_number is not None: - tags.append({'key': 'numChunks', 'value': str(num_chunks)}) - tags.append({'key': 'chunkNumber', 'value': str(chunk_number)}) + tags.append({"key": "numChunks", "value": str(num_chunks)}) + tags.append({"key": "chunkNumber", "value": str(chunk_number)}) response = self.ecs.run_task( - cluster=os.getenv('FARGATE_CLUSTER_NAME'), - taskDefinition=os.getenv('FARGATE_TASK_DEFINITION_NAME'), + cluster=os.getenv("FARGATE_CLUSTER_NAME"), + taskDefinition=os.getenv("FARGATE_TASK_DEFINITION_NAME"), networkConfiguration={ - 'awsvpcConfiguration': { - 'assignPublicIp': 'ENABLED', - 'securityGroups': [os.getenv('FARGATE_SG_ID')], - 'subnets': [os.getenv('FARGATE_SUBNET_ID')] + "awsvpcConfiguration": { + "assignPublicIp": "ENABLED", + "securityGroups": [os.getenv("FARGATE_SG_ID")], + "subnets": [os.getenv("FARGATE_SUBNET_ID")], } }, - platformVersion='1.4.0', - launchType='FARGATE', + platformVersion="1.4.0", + launchType="FARGATE", overrides={ - 'cpu': cpu, - 'memory': memory, - 'containerOverrides': [ + "cpu": cpu, + "memory": memory, + "containerOverrides": [ { - 'name': 'main', # Name from task definition - 'environment': [ + "name": "main", # Name from task definition + "environment": [ { - 'name': 'CROSSFEED_COMMAND_OPTIONS', - 'value': json.dumps(command_options) + "name": "CROSSFEED_COMMAND_OPTIONS", + "value": json.dumps(command_options), }, { - 'name': 'NODE_OPTIONS', - 'value': f"--max_old_space_size={memory}" if memory else '' - } - ] + "name": "NODE_OPTIONS", + "value": f"--max_old_space_size={memory}" + if memory + else "", + }, + ], } - ] - } + ], + }, ) return response async def get_logs(self, fargate_task_arn): """Gets logs for a specific Fargate or Docker task.""" if self.is_local: - log_stream = self.docker.containers.get(fargate_task_arn).logs(stdout=True, stderr=True, timestamps=True) - return ''.join(line[8:] for line in log_stream.split('\n')) + log_stream = self.docker.containers.get(fargate_task_arn).logs( + stdout=True, stderr=True, timestamps=True + ) + return "".join(line[8:] for line in log_stream.split("\n")) else: log_stream_name = f"worker/main/{fargate_task_arn.split('/')[-1]}" response = self.cloudwatch_logs.get_log_events( - logGroupName=os.getenv('FARGATE_LOG_GROUP_NAME'), + logGroupName=os.getenv("FARGATE_LOG_GROUP_NAME"), logStreamName=log_stream_name, - startFromHead=True + startFromHead=True, + ) + events = response["events"] + return "\n".join( + f"{event['timestamp']} {event['message']}" for event in events ) - events = response['events'] - return '\n'.join(f"{event['timestamp']} {event['message']}" for event in events) async def get_num_tasks(self): """Retrieves the number of running tasks associated with the Fargate worker.""" if self.is_local: - containers = self.docker.containers.list(filters={'ancestor': 'crossfeed-worker'}) + containers = self.docker.containers.list( + filters={"ancestor": "crossfeed-worker"} + ) return len(containers) tasks = self.ecs.list_tasks( - cluster=os.getenv('FARGATE_CLUSTER_NAME'), - launchType='FARGATE' + cluster=os.getenv("FARGATE_CLUSTER_NAME"), launchType="FARGATE" ) - return len(tasks.get('taskArns', [])) + return len(tasks.get("taskArns", [])) diff --git a/backend/src/xfd_django/xfd_api/tasks/lambda_client.py b/backend/src/xfd_django/xfd_api/tasks/lambda_client.py index 10a20c39..d85f1088 100644 --- a/backend/src/xfd_django/xfd_api/tasks/lambda_client.py +++ b/backend/src/xfd_django/xfd_api/tasks/lambda_client.py @@ -1,23 +1,24 @@ """AWS Lambda Client.""" -# cisagov Libraries -from .scheduler import handler as scheduler - # Standard Python Libraries import os # Third-Party Libraries import boto3 +from .scheduler import handler as scheduler + class LambdaClient: def __init__(self): """Initialize.""" # Determine if running locally or not - self.is_local = os.getenv('IS_OFFLINE') or os.getenv('IS_LOCAL') + self.is_local = os.getenv("IS_OFFLINE") or os.getenv("IS_LOCAL") if not self.is_local: # Initialize Boto3 Lambda client only if not local - self.lambda_client = boto3.client('lambda', region_name=os.getenv('AWS_REGION', 'us-east-1')) + self.lambda_client = boto3.client( + "lambda", region_name=os.getenv("AWS_REGION", "us-east-1") + ) async def run_command(self, name: str): """Invokes a lambda function with the given name.""" @@ -30,8 +31,6 @@ async def run_command(self, name: str): else: # Invoke the lambda function asynchronously response = self.lambda_client.invoke( - FunctionName=name, - InvocationType='Event', - Payload='' + FunctionName=name, InvocationType="Event", Payload="" ) return response diff --git a/backend/src/xfd_django/xfd_api/tasks/scheduler.py b/backend/src/xfd_django/xfd_api/tasks/scheduler.py index d8513a78..8a820010 100644 --- a/backend/src/xfd_django/xfd_api/tasks/scheduler.py +++ b/backend/src/xfd_django/xfd_api/tasks/scheduler.py @@ -1,12 +1,6 @@ """Scheduler method containing AWS Lambda handler.""" -# cisagov Libraries -from .ecs_client import ECSClient -from ..helpers.getScanOrganizations import get_scan_organizations -from ..models import Scan, Organization, ScanTask -from ..schema_models.scan import SCAN_SCHEMA - # Standard Python Libraries from itertools import islice import os @@ -14,24 +8,33 @@ # Third-Party Libraries from django.utils import timezone +from ..helpers.getScanOrganizations import get_scan_organizations +from ..models import Organization, Scan, ScanTask +from ..schema_models.scan import SCAN_SCHEMA +from .ecs_client import ECSClient + + def chunk(iterable, size): """Chunk a list into a nested list.""" it = iter(iterable) return iter(lambda: list(islice(it, size)), []) + class Scheduler: def __init__(self): """Initialize.""" self.ecs = ECSClient() self.num_existing_tasks = 0 self.num_launched_tasks = 0 - self.max_concurrent_tasks = int(os.getenv('FARGATE_MAX_CONCURRENCY', '10')) + self.max_concurrent_tasks = int(os.getenv("FARGATE_MAX_CONCURRENCY", "10")) self.scans = [] self.organizations = [] self.queued_scan_tasks = [] self.orgs_per_scan_task = 1 - async def initialize(self, scans, organizations, queued_scan_tasks, orgs_per_scan_task): + async def initialize( + self, scans, organizations, queued_scan_tasks, orgs_per_scan_task + ): """Initialize.""" self.scans = scans self.organizations = organizations @@ -39,20 +42,25 @@ async def initialize(self, scans, organizations, queued_scan_tasks, orgs_per_sca self.orgs_per_scan_task = orgs_per_scan_task self.num_existing_tasks = await self.ecs.get_num_tasks() - print(f'Number of running Fargate tasks: {self.num_existing_tasks}') - print(f'Number of queued scan tasks: {len(self.queued_scan_tasks)}') - - async def launch_single_scan_task(self, organizations=None, scan=None, chunk_number=None, num_chunks=None, scan_task=None): + print(f"Number of running Fargate tasks: {self.num_existing_tasks}") + print(f"Number of queued scan tasks: {len(self.queued_scan_tasks)}") + + async def launch_single_scan_task( + self, + organizations=None, + scan=None, + chunk_number=None, + num_chunks=None, + scan_task=None, + ): """Launch single scan.""" organizations = organizations or [] scan_schema = SCAN_SCHEMA.get(scan.name, {}) - task_type = getattr(scan_schema, 'type', None) - global_scan = getattr(scan_schema, 'global_scan', None) + task_type = getattr(scan_schema, "type", None) + global_scan = getattr(scan_schema, "global_scan", None) scan_task = scan_task or ScanTask.objects.create( - scanId=scan, - type=task_type, - status='created' + scanId=scan, type=task_type, status="created" ) # Set the many-to-many relationship with organizations @@ -60,60 +68,68 @@ async def launch_single_scan_task(self, organizations=None, scan=None, chunk_num scan_task.organizations.set(organizations) command_options = scan_task.input or { - 'organizations': [{'name': org.name, 'id': str(org.id)} for org in organizations], - 'scanId': str(scan.id), - 'scanName': scan.name, - 'scanTaskId': str(scan_task.id), - 'numChunks': num_chunks, - 'chunkNumber': chunk_number, - 'isSingleScan': scan.isSingleScan + "organizations": [ + {"name": org.name, "id": str(org.id)} for org in organizations + ], + "scanId": str(scan.id), + "scanName": scan.name, + "scanTaskId": str(scan_task.id), + "numChunks": num_chunks, + "chunkNumber": chunk_number, + "isSingleScan": scan.isSingleScan, } scan_task.input = command_options if self.reached_scan_limit(): - scan_task.status = 'queued' + scan_task.status = "queued" if not scan_task.queuedAt: scan_task.queuedAt = timezone.now() - print(f'Reached maximum concurrency, queueing scantask {scan_task.id}') + print(f"Reached maximum concurrency, queueing scantask {scan_task.id}") scan_task.save() return try: - if task_type == 'fargate': + if task_type == "fargate": result = await self.ecs.run_command(command_options) - if not result.get('tasks'): - print(f"Failed to start Fargate task for scan {scan.name}, failures: {result.get('failures')}") - raise Exception(f"Failed to start Fargate task for scan {scan.name}") - - task_arn = result['tasks'][0]['taskArn'] + if not result.get("tasks"): + print( + f"Failed to start Fargate task for scan {scan.name}, failures: {result.get('failures')}" + ) + raise Exception( + f"Failed to start Fargate task for scan {scan.name}" + ) + + task_arn = result["tasks"][0]["taskArn"] scan_task.fargateTaskArn = task_arn - print(f'Successfully invoked scan {scan.name} with Fargate on {len(organizations)} organizations. Task ARN: {task_arn}') + print( + f"Successfully invoked scan {scan.name} with Fargate on {len(organizations)} organizations. Task ARN: {task_arn}" + ) else: - raise Exception(f'Invalid task type: {task_type}') + raise Exception(f"Invalid task type: {task_type}") - scan_task.status = 'requested' + scan_task.status = "requested" scan_task.requestedAt = timezone.now() self.num_launched_tasks += 1 except Exception as error: print(f"Error invoking {scan.name} scan: {error}") scan_task.output = str(error) - scan_task.status = 'failed' + scan_task.status = "failed" scan_task.finishedAt = timezone.now() - + scan_task.save() async def launch_scan_task(self, organizations=None, scan=None): """Launch scan task.""" organizations = organizations or [] - + scan_schema = SCAN_SCHEMA.get(scan.name, None) - num_chunks = getattr(scan_schema, 'numChunks', None) - + num_chunks = getattr(scan_schema, "numChunks", None) + # If num_chunks is set, handle it; otherwise, default to launching a single task if num_chunks: # For running locally, set num_chunks to 1 - if os.getenv('IS_LOCAL'): + if os.getenv("IS_LOCAL"): num_chunks = 1 # Sanitize num_chunks to ensure it doesn't exceed 100 @@ -121,10 +137,10 @@ async def launch_scan_task(self, organizations=None, scan=None): for chunk_number in range(num_chunks): await self.launch_single_scan_task( - organizations=organizations, - scan=scan, - chunk_number=chunk_number, - num_chunks=num_chunks + organizations=organizations, + scan=scan, + chunk_number=chunk_number, + num_chunks=num_chunks, ) else: # Launch a single scan task when num_chunks is None or 0 @@ -132,7 +148,9 @@ async def launch_scan_task(self, organizations=None, scan=None): def reached_scan_limit(self): """Check scan limit.""" - return (self.num_existing_tasks + self.num_launched_tasks) >= self.max_concurrent_tasks + return ( + self.num_existing_tasks + self.num_launched_tasks + ) >= self.max_concurrent_tasks async def run(self): """Run scheduler.""" @@ -140,7 +158,7 @@ async def run(self): prev_num_launched_tasks = self.num_launched_tasks if scan.name not in SCAN_SCHEMA: - print(f'Invalid scan name: {scan.name}') + print(f"Invalid scan name: {scan.name}") continue scan_schema = SCAN_SCHEMA[scan.name] @@ -149,8 +167,16 @@ async def run(self): continue await self.launch_scan_task(scan=scan) else: - organizations = get_scan_organizations(scan) if scan.isGranular else self.organizations - orgs_to_launch = [org for org in organizations if self.should_run_scan(scan=scan, organization=org)] + organizations = ( + get_scan_organizations(scan) + if scan.isGranular + else self.organizations + ) + orgs_to_launch = [ + org + for org in organizations + if self.should_run_scan(scan=scan, organization=org) + ] for org_chunk in chunk(orgs_to_launch, self.orgs_per_scan_task): await self.launch_scan_task(organizations=org_chunk, scan=scan) @@ -167,8 +193,8 @@ async def run_queued(self): def should_run_scan(self, scan, organization=None): """Check if the scan should run.""" scan_schema = SCAN_SCHEMA.get(scan.name, {}) - is_passive = getattr(scan_schema, 'isPassive', False) - global_scan = getattr(scan_schema, 'global_scan', False) + is_passive = getattr(scan_schema, "isPassive", False) + global_scan = getattr(scan_schema, "global_scan", False) # Don't run non-passive scans on passive organizations. if organization and organization.isPassive and not is_passive: @@ -191,8 +217,8 @@ def filter_scan_tasks(tasks): # Check if there's a currently running or queued scan task for the given scan. last_running_scan_task = filter_scan_tasks( ScanTask.objects.filter( - status__in=['created', 'queued', 'requested', 'started'] - ).order_by('-createdAt') + status__in=["created", "queued", "requested", "started"] + ).order_by("-createdAt") ).first() # If there's a running or queued task, do not run another. @@ -203,61 +229,73 @@ def filter_scan_tasks(tasks): # Check for the last finished scan task. last_finished_scan_task = filter_scan_tasks( ScanTask.objects.filter( - status__in=['finished', 'failed'], - finishedAt__isnull=False - ).order_by('-finishedAt') + status__in=["finished", "failed"], finishedAt__isnull=False + ).order_by("-finishedAt") ).first() # If a scan task was finished recently within the scan frequency, do not run. if last_finished_scan_task and last_finished_scan_task.finishedAt: print("Has been run since the last scan frequency") - frequency_seconds = scan.frequency * 1000 # Assuming frequency is in seconds. - if (timezone.now() - last_finished_scan_task.finishedAt).total_seconds() < frequency_seconds: + frequency_seconds = ( + scan.frequency * 1000 + ) # Assuming frequency is in seconds. + if ( + timezone.now() - last_finished_scan_task.finishedAt + ).total_seconds() < frequency_seconds: return False # If the scan is marked as a single scan and has already run once, do not run again. - if last_finished_scan_task and last_finished_scan_task.finishedAt and scan.isSingleScan: + if ( + last_finished_scan_task + and last_finished_scan_task.finishedAt + and scan.isSingleScan + ): print("Single scan") return False return True + async def handler(event): """Handler for manually invoking the scheduler to run scans.""" - print('Running scheduler...') + print("Running scheduler...") - scan_ids = event.get('scanIds', []) - if 'scanId' in event: - scan_ids.append(event['scanId']) + scan_ids = event.get("scanIds", []) + if "scanId" in event: + scan_ids.append(event["scanId"]) - org_ids = event.get('organizationIds', []) + org_ids = event.get("organizationIds", []) # Fetch scans based on scan_ids if provided if scan_ids: - scans = Scan.objects.filter(id__in=scan_ids).prefetch_related('organizations', 'tags') + scans = Scan.objects.filter(id__in=scan_ids).prefetch_related( + "organizations", "tags" + ) else: - scans = Scan.objects.all().prefetch_related('organizations', 'tags') + scans = Scan.objects.all().prefetch_related("organizations", "tags") # Fetch organizations based on org_ids if provided if org_ids: organizations = Organization.objects.filter(id__in=org_ids) else: organizations = Organization.objects.all() - - queued_scan_tasks = ScanTask.objects.filter( - scanId__in=scan_ids, - status='queued' - ).order_by('queuedAt').select_related('scanId') + + queued_scan_tasks = ( + ScanTask.objects.filter(scanId__in=scan_ids, status="queued") + .order_by("queuedAt") + .select_related("scanId") + ) scheduler = Scheduler() await scheduler.initialize( scans=scans, organizations=organizations, queued_scan_tasks=queued_scan_tasks, - orgs_per_scan_task=event.get('orgsPerScanTask') or int(os.getenv('SCHEDULER_ORGS_PER_SCANTASK', '1')) + orgs_per_scan_task=event.get("orgsPerScanTask") + or int(os.getenv("SCHEDULER_ORGS_PER_SCANTASK", "1")), ) await scheduler.run_queued() await scheduler.run() - print('Finished running scheduler.') \ No newline at end of file + print("Finished running scheduler.") diff --git a/backend/src/xfd_django/xfd_api/tests.py b/backend/src/xfd_django/xfd_api/tests.py index 7ce503c2..2884d116 100644 --- a/backend/src/xfd_django/xfd_api/tests.py +++ b/backend/src/xfd_django/xfd_api/tests.py @@ -1,3 +1,4 @@ +# Third-Party Libraries from django.test import TestCase # Create your tests here. diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 3c7c73db..8b3e7f11 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -17,13 +17,13 @@ - .models """ -# cisagov Libraries -from .auth import get_current_active_user -from .models import Assessment, User -# TODO: I think all our import should be like this blow -from .api_methods import scan -from .schema_models import scan as scanSchema +# Standard Python Libraries +from typing import List, Optional + +# Third-Party Libraries +from fastapi import APIRouter, Depends, HTTPException, Query +from .api_methods import scan from .api_methods.api_keys import get_api_keys from .api_methods.cpe import get_cpes_by_id from .api_methods.cve import get_cves_by_id, get_cves_by_name @@ -32,27 +32,21 @@ from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user - +from .models import Assessment, User +from .schema_models import scan as scanSchema from .schema_models.assessment import Assessment from .schema_models.cpe import Cpe as CpeSchema from .schema_models.cve import Cve as CveSchema -from .schema_models.domain import DomainSearch from .schema_models.domain import Domain as DomainSchema +from .schema_models.domain import DomainSearch from .schema_models.organization import Organization as OrganizationSchema - from .schema_models.user import User as UserSchema from .schema_models.vulnerability import Vulnerability as VulnerabilitySchema -# Standard Python Libraries -from typing import List, Optional - -# Third-Party Libraries -from fastapi import APIRouter, Depends, HTTPException, Query - - # Define API router api_router = APIRouter() + # Healthcheck endpoint @api_router.get("/healthcheck", tags=["Testing"]) async def healthcheck(): @@ -321,10 +315,12 @@ async def call_get_organizations( """ return get_organizations(state, regionId) + # ======================================== # Scan Endpoints # ======================================== + @api_router.get( "/scans", dependencies=[Depends(get_current_active_user)], @@ -356,7 +352,7 @@ async def list_granular_scans(current_user: User = Depends(get_current_active_us async def create_scan( scan_data: scanSchema.NewScan, current_user: User = Depends(get_current_active_user) ): - """ Create a new scan.""" + """Create a new scan.""" return scan.create_scan(scan_data, current_user) @@ -364,7 +360,7 @@ async def create_scan( "/scans/{scan_id}", dependencies=[Depends(get_current_active_user)], response_model=scanSchema.GetScanResponseModel, - tags=["Scans"] + tags=["Scans"], ) async def get_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): """Get a scan by its ID. User must be authenticated.""" @@ -375,7 +371,7 @@ async def get_scan(scan_id: str, current_user: User = Depends(get_current_active "/scans/{scan_id}", dependencies=[Depends(get_current_active_user)], response_model=scanSchema.CreateScanResponseModel, - tags=["Scans"] + tags=["Scans"], ) async def update_scan( scan_id: str, @@ -390,7 +386,7 @@ async def update_scan( "/scans/{scan_id}", dependencies=[Depends(get_current_active_user)], response_model=scanSchema.GenericMessageResponseModel, - tags=["Scans"] + tags=["Scans"], ) async def delete_scan( scan_id: str, current_user: User = Depends(get_current_active_user) @@ -403,7 +399,7 @@ async def delete_scan( "/scans/{scan_id}/run", dependencies=[Depends(get_current_active_user)], response_model=scanSchema.GenericMessageResponseModel, - tags=["Scans"] + tags=["Scans"], ) async def run_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): """Manually run a scan by its ID""" @@ -411,9 +407,7 @@ async def run_scan(scan_id: str, current_user: User = Depends(get_current_active @api_router.post( - "/scheduler/invoke", - dependencies=[Depends(get_current_active_user)], - tags=["Scans"] + "/scheduler/invoke", dependencies=[Depends(get_current_active_user)], tags=["Scans"] ) async def invoke_scheduler(current_user: User = Depends(get_current_active_user)): """Manually invoke the scan scheduler.""" diff --git a/backend/src/xfd_django/xfd_django/settings.py b/backend/src/xfd_django/xfd_django/settings.py index 07c61e13..67145819 100644 --- a/backend/src/xfd_django/xfd_django/settings.py +++ b/backend/src/xfd_django/xfd_django/settings.py @@ -36,7 +36,12 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [".execute-api.us-east-1.amazonaws.com", 'https://api.staging-cd.crossfeed.cyber.dhs.gov', 'http://localhost:3000', 'http://localhost'] +ALLOWED_HOSTS = [ + ".execute-api.us-east-1.amazonaws.com", + "https://api.staging-cd.crossfeed.cyber.dhs.gov", + "http://localhost:3000", + "http://localhost", +] MESSAGE_TAGS = { messages.DEBUG: "alert-secondary", diff --git a/backend/src/xfd_django/xfd_django/urls.py b/backend/src/xfd_django/xfd_django/urls.py index 026b20cb..523c0b8e 100644 --- a/backend/src/xfd_django/xfd_django/urls.py +++ b/backend/src/xfd_django/xfd_django/urls.py @@ -14,9 +14,10 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ +# Third-Party Libraries from django.contrib import admin from django.urls import path urlpatterns = [ - path('admin/', admin.site.urls), + path("admin/", admin.site.urls), ] diff --git a/backend/src/xfd_django/xfd_django/wsgi.py b/backend/src/xfd_django/xfd_django/wsgi.py index 14c13f58..bf5958c3 100644 --- a/backend/src/xfd_django/xfd_django/wsgi.py +++ b/backend/src/xfd_django/xfd_django/wsgi.py @@ -7,10 +7,12 @@ https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ """ +# Standard Python Libraries import os +# Third-Party Libraries from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'xfd_django.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xfd_django.settings") application = get_wsgi_application() From 242040eb0df64bf9b5e43aff11be89460abe7a57 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 26 Sep 2024 12:27:22 -0400 Subject: [PATCH 031/314] Run pre-commit --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index fbe339fd..a6585a7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,7 +57,7 @@ services: - ./.env depends_on: - db - + minio: image: bitnami/minio:2020.9.26 user: root From 91341479737d25b5d8fd46847fd12a08d457a545 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Thu, 26 Sep 2024 15:07:04 -0500 Subject: [PATCH 032/314] Fix organizations, WIP. --- .../xfd_django/xfd_api/api_methods/domain.py | 24 ++++++++++++ .../xfd_api/api_methods/organization.py | 31 ++++----------- .../xfd_django/xfd_api/api_methods/user.py | 39 ++++--------------- .../xfd_api/schema_models/organization.py | 8 ++-- backend/src/xfd_django/xfd_api/views.py | 32 +++++---------- 5 files changed, 53 insertions(+), 81 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/domain.py b/backend/src/xfd_django/xfd_api/api_methods/domain.py index 92a0c1dd..deb4953f 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/domain.py +++ b/backend/src/xfd_django/xfd_api/api_methods/domain.py @@ -7,6 +7,7 @@ from fastapi import HTTPException from ..models import Domain +from ..schemas import DomainFilters, DomainSearch def get_domain_by_id(domain_id: str): @@ -22,3 +23,26 @@ def get_domain_by_id(domain_id: str): raise HTTPException(status_code=404, detail="Domain not found.") except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +def search_domains(domain_search: DomainSearch): + """ + List domains by search filter + Arguments: + domain_search: A DomainSearch object to filter by. + Returns: + object: a list of Domain objects + """ + try: + domains = Domain.objects.filter(domain_search) + return domains + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def export_domains(domain_search: DomainSearch): + try: + domains = Domain.objects.filter(domain_search) + # TODO Continue developing export logic + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/organization.py b/backend/src/xfd_django/xfd_api/api_methods/organization.py index 19c522d5..447aa5e9 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/organization.py +++ b/backend/src/xfd_django/xfd_api/api_methods/organization.py @@ -49,16 +49,11 @@ def read_orgs(): raise HTTPException(status_code=500, detail=str(e)) -def get_organizations( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), -): +def get_organizations(regionId): """ List all organizations with query parameters. Args: - state (Optional[List[str]]): List of states to filter organizations by. - regionId (Optional[List[str]]): List of region IDs to filter organizations by. - + regionId : region IDs to filter organizations by. Raises: HTTPException: If the user is not authorized or no organizations are found. @@ -66,20 +61,8 @@ def get_organizations( List[Organizations]: A list of organizations matching the filter criteria. """ - # if not current_user: - # raise HTTPException(status_code=401, detail="Unauthorized") - - # Prepare filter parameters - filter_params = {} - if state: - filter_params["state__in"] = state - if regionId: - filter_params["regionId__in"] = regionId - - organizations = Organization.objects.filter(**filter_params) - - if not organizations.exists(): - raise HTTPException(status_code=404, detail="No organizations found") - - # Return the Pydantic models directly by calling from_orm - return organizations + try: + organizations = Organization.objects.filter(regionId=regionId) + return organizations + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/user.py b/backend/src/xfd_django/xfd_api/api_methods/user.py index 260b8849..d56f36f0 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/user.py +++ b/backend/src/xfd_django/xfd_api/api_methods/user.py @@ -12,44 +12,21 @@ from ..schemas import User as UserSchema -def get_users( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), - invitePending: Optional[List[str]] = Query(None), - # current_user: User = Depends(is_regional_admin) -): +def get_users(regionId): """ Retrieve a list of users based on optional filter parameters. Args: - state (Optional[List[str]]): List of states to filter users by. - regionId (Optional[List[str]]): List of region IDs to filter users by. - invitePending (Optional[List[str]]): List of invite pending statuses to filter users by. - current_user (User): The current authenticated user, must be a regional admin. - + regionId : Region ID to filter users by. Raises: HTTPException: If the user is not authorized or no users are found. Returns: List[User]: A list of users matching the filter criteria. """ - # if not current_user: - # raise HTTPException(status_code=401, detail="Unauthorized") - - # Prepare filter parameters - filter_params = {} - if state: - filter_params["state__in"] = state - if regionId: - filter_params["regionId__in"] = regionId - if invitePending: - filter_params["invitePending__in"] = invitePending - - # Query users with filter parameters and prefetch related roles - users = User.objects.filter(**filter_params).prefetch_related("roles") - - if not users.exists(): - raise HTTPException(status_code=404, detail="No users found") - - # Return the Pydantic models directly by calling from_orm - return [UserSchema.from_orm(user) for user in users] + + try: + users = User.objects.filter(regionId=regionId).prefetch_related("roles") + return [UserSchema.from_orm(user) for user in users] + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization.py b/backend/src/xfd_django/xfd_api/schema_models/organization.py index f52f2406..12e7bd1a 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/organization.py +++ b/backend/src/xfd_django/xfd_api/schema_models/organization.py @@ -3,7 +3,7 @@ # from pydantic.types import UUID1, UUID # Standard Python Libraries from datetime import datetime -from typing import Optional +from typing import List, Optional from uuid import UUID # Third-Party Libraries @@ -18,10 +18,10 @@ class Organization(BaseModel): updatedAt: datetime acronym: Optional[str] name: str - rootDomains: str - ipBlocks: str + rootDomains: List[str] + ipBlocks: List[str] isPassive: bool - pendingDomains: str + pendingDomains: List[str] country: Optional[str] state: Optional[str] regionId: Optional[str] diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 570bbbfc..89b3df4a 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -272,25 +272,17 @@ async def call_update_vulnerability(vuln_id, data: VulnerabilitySchema): @api_router.get( - "/v2/users", + "/users/{regionId}", response_model=List[UserSchema], # dependencies=[Depends(get_current_active_user)], tags=["User"], ) -async def call_get_users( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), - invitePending: Optional[List[str]] = Query(None), - # current_user: User = Depends(is_regional_admin) -): +async def call_get_users(regionId): """ Call get_users() Args: - state (Optional[List[str]]): List of states to filter users by. - regionId (Optional[List[str]]): List of region IDs to filter users by. - invitePending (Optional[List[str]]): List of invite pending statuses to filter users by. - current_user (User): The current authenticated user, must be a regional admin. + regionId: Region IDs to filter users by. Raises: HTTPException: If the user is not authorized or no users are found. @@ -298,24 +290,20 @@ async def call_get_users( Returns: List[User]: A list of users matching the filter criteria. """ - return get_users(state, regionId, invitePending) + return get_users(regionId) @api_router.get( - "/organizations", - # response_model=List[OrganizationSchema], + "/organizations/{regionId}", + response_model=List[OrganizationSchema], # dependencies=[Depends(get_current_active_user)], - tags=["Organizations"], + tags=["Organization"], ) -async def call_get_organizations( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), -): +async def call_get_organizations(regionId): """ List all organizations with query parameters. Args: - state (Optional[List[str]]): List of states to filter organizations by. - regionId (Optional[List[str]]): List of region IDs to filter organizations by. + regionId : Region IDs to filter organizations by. Raises: HTTPException: If the user is not authorized or no organizations are found. @@ -323,4 +311,4 @@ async def call_get_organizations( Returns: List[Organizations]: A list of organizations matching the filter criteria. """ - return get_organizations(state, regionId) + return get_organizations(regionId) From 4ff129447734597cd3bc037630771156686c37b9 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 30 Sep 2024 16:00:43 -0400 Subject: [PATCH 033/314] Add get_tag_organizations --- backend/src/xfd_django/xfd_api/auth.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 80966dbe..6b256b89 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -9,7 +9,7 @@ from fastapi.security import APIKeyHeader, OAuth2PasswordBearer from .jwt_utils import decode_jwt_token -from .models import ApiKey +from .models import ApiKey, OrganizationTag oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) @@ -74,6 +74,23 @@ def is_regional_admin(current_user) -> bool: """Check if the user has regional admin permissions.""" return current_user and current_user.userType in ["regionalAdmin", "globalAdmin"] +def get_tag_organizations(current_user, tag_id: str) -> list[str]: + """Returns the organizations belonging to a tag, if the user can access the tag.""" + # Check if the user is a global view admin + if not is_global_view_admin(current_user): + return [] + + # Fetch the OrganizationTag and its related organizations + tag = OrganizationTag.objects.prefetch_related("organizations").filter(id=tag_id).first() + if tag: + # Return a list of organization IDs + return [org.id for org in tag.organizations.all()] + + # Return an empty list if tag is not found + return [] + + + # TODO: Below is a template of what these could be nut isn't tested # RECREATE ALL THE FUNCTIONS IN AUTH.TS From bfbee9b9a25b1ef95f32ea0d5698ef2b4f406598 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Mon, 30 Sep 2024 16:01:14 -0400 Subject: [PATCH 034/314] Add get tag organizations --- backend/src/xfd_django/xfd_api/auth.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 80966dbe..6b256b89 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -9,7 +9,7 @@ from fastapi.security import APIKeyHeader, OAuth2PasswordBearer from .jwt_utils import decode_jwt_token -from .models import ApiKey +from .models import ApiKey, OrganizationTag oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) @@ -74,6 +74,23 @@ def is_regional_admin(current_user) -> bool: """Check if the user has regional admin permissions.""" return current_user and current_user.userType in ["regionalAdmin", "globalAdmin"] +def get_tag_organizations(current_user, tag_id: str) -> list[str]: + """Returns the organizations belonging to a tag, if the user can access the tag.""" + # Check if the user is a global view admin + if not is_global_view_admin(current_user): + return [] + + # Fetch the OrganizationTag and its related organizations + tag = OrganizationTag.objects.prefetch_related("organizations").filter(id=tag_id).first() + if tag: + # Return a list of organization IDs + return [org.id for org in tag.organizations.all()] + + # Return an empty list if tag is not found + return [] + + + # TODO: Below is a template of what these could be nut isn't tested # RECREATE ALL THE FUNCTIONS IN AUTH.TS From b652c72282b02205c05f0c9d5e1696fdf1b29063 Mon Sep 17 00:00:00 2001 From: nickviola Date: Tue, 1 Oct 2024 09:37:03 -0500 Subject: [PATCH 035/314] Update views to break out api logic and add schema changes to support pydantic --- .gitignore | 3 + backend/requirements.txt | 2 + .../xfd_api/api_methods/__init__.py | 0 .../xfd_django/xfd_api/api_methods/api_key.py | 97 ++++ .../xfd_django/xfd_api/api_methods/auth.py | 52 ++ .../xfd_api/api_methods/notification.py | 82 +++ .../xfd_django/xfd_api/api_methods/user.py | 0 backend/src/xfd_django/xfd_api/auth.py | 4 +- backend/src/xfd_django/xfd_api/login_gov.py | 5 +- backend/src/xfd_django/xfd_api/models.py | 2 +- .../xfd_api/schema_models/api_key.py | 38 ++ .../xfd_api/schema_models/assessment.py | 3 + .../xfd_api/schema_models/category.py | 3 + .../xfd_django/xfd_api/schema_models/cpe.py | 3 + .../xfd_django/xfd_api/schema_models/cve.py | 3 + .../xfd_api/schema_models/domain.py | 8 +- .../xfd_api/schema_models/notification.py | 3 + .../xfd_api/schema_models/organization.py | 3 + .../xfd_api/schema_models/question.py | 3 + .../xfd_api/schema_models/resource.py | 3 + .../xfd_api/schema_models/response.py | 3 + .../xfd_django/xfd_api/schema_models/role.py | 7 +- .../xfd_django/xfd_api/schema_models/scan.py | 6 + .../xfd_api/schema_models/searchbody.py | 3 + .../xfd_api/schema_models/service.py | 3 + .../xfd_django/xfd_api/schema_models/user.py | 25 +- .../xfd_api/schema_models/vulnerability.py | 9 + backend/src/xfd_django/xfd_api/schemas.py | 31 -- .../src/xfd_django/xfd_api/tests/conftest.py | 9 + .../src/xfd_django/xfd_api/tests/pytest.ini | 5 + .../xfd_django/xfd_api/tests/test_api_key.py | 203 ++++++++ .../src/xfd_django/xfd_api/tests/test_auth.py | 119 +++++ .../xfd_api/tests/test_notification.py | 146 ++++++ .../src/xfd_django/xfd_api/tests/test_user.py | 0 backend/src/xfd_django/xfd_api/views.py | 467 ++++++------------ .../xfd_django/xfd_django/test_settings.py | 136 +++++ 36 files changed, 1118 insertions(+), 371 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/api_methods/__init__.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/api_key.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/auth.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/notification.py create mode 100644 backend/src/xfd_django/xfd_api/api_methods/user.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/api_key.py delete mode 100644 backend/src/xfd_django/xfd_api/schemas.py create mode 100644 backend/src/xfd_django/xfd_api/tests/conftest.py create mode 100644 backend/src/xfd_django/xfd_api/tests/pytest.ini create mode 100644 backend/src/xfd_django/xfd_api/tests/test_api_key.py create mode 100644 backend/src/xfd_django/xfd_api/tests/test_auth.py create mode 100644 backend/src/xfd_django/xfd_api/tests/test_notification.py create mode 100644 backend/src/xfd_django/xfd_api/tests/test_user.py create mode 100644 backend/src/xfd_django/xfd_django/test_settings.py diff --git a/.gitignore b/.gitignore index 1248786c..af10ab87 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,6 @@ minio-data infrastructure/lambdas/security_headers.zip *.hcl .iac-data + +# VS Code +*.vscode/* diff --git a/backend/requirements.txt b/backend/requirements.txt index 7ba5750d..b5fc5b50 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -4,5 +4,7 @@ fastapi==0.111.0 mangum==0.17.0 psycopg2-binary PyJWT +pytest +pytest-django requests==2.32.3 uvicorn==0.30.1 diff --git a/backend/src/xfd_django/xfd_api/api_methods/__init__.py b/backend/src/xfd_django/xfd_api/api_methods/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/api_methods/api_key.py b/backend/src/xfd_django/xfd_api/api_methods/api_key.py new file mode 100644 index 00000000..73d666e7 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/api_key.py @@ -0,0 +1,97 @@ +"""/api-keys API logic""" + +# Standard Python Libraries +import hashlib +import secrets +import uuid + +# Third-Party Libraries +from fastapi import HTTPException, status +from xfd_api.models import ApiKey +from xfd_api.schema_models.api_key import ApiKey as ApiKeySchema + + +def post(current_user): + """POST API LOGIC""" + # Generate a random 16-byte API key + key = secrets.token_hex(16) + + # Hash the API key + hashed_key = hashlib.sha256(key.encode()).hexdigest() + + # Create with schema validation + api_key = ApiKeySchema( + id=uuid.uuid4(), + hashedKey=hashed_key, + lastFour=key[-4:], + userId=current_user, + ) + + # Return Serialized data from Schema + return ApiKeySchema.model_validate(api_key).model_dump( + exclude={"hashedKey", "userId"}, api_key=key + ) + + +def delete(id, current_user): + """DELETE API LOGIC""" + try: + # Confirm id is a valid UUID + uuid.UUID(id) + + # Delete by id + api_key = ApiKey.objects.get(id=id) + api_key.delete() + + # Delete Response TODO: confirm output + return {"status": "success", "message": "API Key deleted successfully"} + + except ApiKey.DoesNotExist: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="API Key not found" + ) + except ValueError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Invalid API Key ID" + ) + + +def get_all(current_user): + """GET All API LOGIC""" + try: + # Get all ApiKey objects from the database + api_keys = ApiKey.objects.all() + + # Return schema validated response + validated_response = [ + ApiKeySchema.model_validate(item).model_dump( + exclude={"hashedKey", "userId"} + ) + for item in api_keys + ] + + return validated_response + + except Exception as error: + raise HTTPException(status_code=500, detail=str(error)) + + +def get_by_id(id, current_user): + """GET API KEY by id""" + try: + # Confirm id is a valid UUID + uuid.UUID(id) + + # Find the ApiKey by its ID + api_key = ApiKey.objects.get(id=id) + + # Return validated output + return ApiKeySchema.model_validate(obj=api_key).model_dump( + exclude={"hashedKey", "userId"} + ) + + except ApiKey.DoesNotExist: + raise HTTPException(status_code=404, detail="API Key not found") + + except Exception as error: + raise HTTPException(status_code=500, detail=str(error)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/auth.py b/backend/src/xfd_django/xfd_api/api_methods/auth.py new file mode 100644 index 00000000..6ea81240 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/auth.py @@ -0,0 +1,52 @@ +"""Auth API logic""" +# Third-Party Libraries +from fastapi import HTTPException, status +from fastapi.responses import JSONResponse +from xfd_api.auth import get_jwt_from_code, process_user + + +async def handle_okta_callback(request): + """POST API LOGIC""" + print(f"Request from /auth/okta-callback: {str(request)}") + body = await request.json() + print(f"Request json from callback: {str(request)}") + print(f"Request json from callback: {body}") + print(f"Body type: {type(body)}") + code = body.get("code") + print(f"Code: {code}") + if not code: + return HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Code not found in request body", + ) + jwt_data = await get_jwt_from_code(code) + print(f"JWT Data: {jwt_data}") + if jwt_data is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid authorization code or failed to retrieve tokens", + ) + access_token = jwt_data.get("access_token") + refresh_token = jwt_data.get("refresh_token") + decoded_token = jwt_data.get("decoded_token") + + resp = await process_user(decoded_token, access_token, refresh_token) + token = resp.get("token") + + # Create a JSONResponse object to return the response and set the cookie + response = JSONResponse( + content={"message": "User authenticated", "data": resp, "token": token} + ) + response.body = resp + # response.body = resp + response.set_cookie(key="token", value=token) + + # Set the 'crossfeed-token' cookie + response.set_cookie( + key="crossfeed-token", + value=token, + # httponly=True, # This makes the cookie inaccessible to JavaScript + # secure=True, # Ensures the cookie is only sent over HTTPS + # samesite="Lax" # Restricts when cookies are sent, adjust as necessary (e.g., "Strict" or "None") + ) + return response diff --git a/backend/src/xfd_django/xfd_api/api_methods/notification.py b/backend/src/xfd_django/xfd_api/api_methods/notification.py new file mode 100644 index 00000000..c531f8d0 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/notification.py @@ -0,0 +1,82 @@ +"""/api-keys API logic""" + +# Standard Python Libraries +import uuid + +# Third-Party Libraries +from fastapi import HTTPException, status +from xfd_api.models import Notification +from xfd_api.schema_models.notification import Notification as NotificationSchema + + +def post(data, current_user): + """POST LOGIC""" + data.extend(id=uuid.uuid4()) + # Create the record in the database + result = Notification.objects.create(data) + + # Return Serialized data from Schema + return NotificationSchema.from_orm(result).dict() + + +def delete(id, current_user): + """DELETE LOGIC""" + try: + # Validate that key_id is a valid UUID + uuid.UUID(id) + result = Notification.objects.get(id=id, userId=current_user) + + # Delete the Item + result.delete() + return {"status": "success", "message": "Item deleted successfully"} + except ValueError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Invalid id value" + ) + except Notification.DoesNotExist: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Item not found" + ) + + +def get_all(current_user): + """GET All LOGIC""" + try: + # Get all objects from the database + result = Notification.objects.all() + + # Convert each object to Schema using from_orm + return [NotificationSchema.from_orm(item) for item in result] + + except Exception as error: + raise HTTPException(status_code=500, detail=str(error)) + + +def get_by_id(id, current_user): + """GET by id""" + try: + # Find the item by its id + result = Notification.objects.get(id=id) + + # Convert the result to Schema using from_orm + return NotificationSchema.from_orm(result) + + except Notification.DoesNotExist: + raise HTTPException(status_code=404, detail="Item not found") + except Exception as error: + raise HTTPException(status_code=500, detail=str(error)) + + +def get_508_banner(current_user): + """GET 508 banner.""" + # TODO: Adding placeholder until we determine if we still need this. + # Remove logic if no longer needed or update to actual return object. + try: + # Get the 508 banner from the DB + result = "" + + # Format/Return Banner + return result + + except Exception as error: + raise HTTPException(status_code=500, detail=str(error)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/user.py b/backend/src/xfd_django/xfd_api/api_methods/user.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index f83a5a3e..06f08b96 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -59,7 +59,7 @@ def create_jwt_token(user): payload = { "id": str(user.id), "email": user.email, - "exp": datetime.utcnow() + timedelta(hours=JWT_TIMEOUT_HOURS), + "exp": datetime.now(datetime.timezone.utc) + timedelta(hours=JWT_TIMEOUT_HOURS), } return jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) @@ -194,7 +194,7 @@ def get_current_active_user(token: str = Depends(oauth2_scheme)): ) # Fetch the user by ID from the database user = User.objects.get(id=user_id) - print(f"User found: {user}") + print(f"User found: {user_to_dict(user)}") if user is None: print("User not found") raise HTTPException( diff --git a/backend/src/xfd_django/xfd_api/login_gov.py b/backend/src/xfd_django/xfd_api/login_gov.py index cc961331..52dea792 100644 --- a/backend/src/xfd_django/xfd_api/login_gov.py +++ b/backend/src/xfd_django/xfd_api/login_gov.py @@ -33,7 +33,7 @@ def random_string(length): return secrets.token_hex(length // 2) -# 1. Login function that returns authorization URL, state, and nonce +# Login function that returns authorization URL, state, and nonce def login(): """Equivalent function to initiate OpenID Connect login.""" # Fetch OpenID Connect configuration @@ -55,7 +55,7 @@ def login(): return {"url": authorization_url, "state": state, "nonce": nonce} -# 2. Callback function to exchange authorization code for tokens and user info +# Callback function to exchange authorization code for tokens and user info def callback(body): """Equivalent function to handle OpenID Connect callback.""" config_response = requests.get(discovery_url) @@ -86,6 +86,7 @@ def callback(body): # Decode the ID token without verifying the signature (optional depending on your security model) decoded_token = jwt.decode(id_token, options={"verify_signature": False}) + print(f"Decoded Token from login_gov: {decoded_token}") return decoded_token diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 4e0f4c19..240f2e21 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -245,7 +245,7 @@ class Meta: class Organization(models.Model): """The Organization model.""" - id = models.UUIDField(primary_key=True) + id = models.UUIDField(primary_key=True, serialize=True) createdAt = models.DateTimeField(db_column="createdAt") updatedAt = models.DateTimeField(db_column="updatedAt") acronym = models.CharField(unique=True, blank=True, null=True) diff --git a/backend/src/xfd_django/xfd_api/schema_models/api_key.py b/backend/src/xfd_django/xfd_api/schema_models/api_key.py new file mode 100644 index 00000000..84df02af --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/api_key.py @@ -0,0 +1,38 @@ +"""Api schema.""" +# Standard Python Libraries +from datetime import datetime +from typing import Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, ConfigDict + + +class ApiKey(BaseModel): + """Pydantic model for the ApiKey model.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + lastUsed: Optional[datetime] + hashedKey: Optional[str] + lastFour: Optional[str] + userId: Optional[UUID] + + @classmethod + def model_validate(cls, obj): + # Ensure that we convert the UUIDs to strings when validating + api_key_data = obj.__dict__.copy() + + # Remove the '_state' field or any other unwanted internal Django fields + api_key_data.pop("_state", None) + api_key_data["userId"] = api_key_data.pop("userId_id", None) + + for key, val in api_key_data.items(): + # Convert any UUIDs to strings + if isinstance(val, UUID): + api_key_data[key] = str(val) + return cls(**api_key_data) + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/assessment.py b/backend/src/xfd_django/xfd_api/schema_models/assessment.py index d8281984..09b96c12 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/assessment.py +++ b/backend/src/xfd_django/xfd_api/schema_models/assessment.py @@ -19,3 +19,6 @@ class Assessment(BaseModel): rscId: str type: str userId: Optional[Any] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/category.py b/backend/src/xfd_django/xfd_api/schema_models/category.py index 14456fca..5aea830b 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/category.py +++ b/backend/src/xfd_django/xfd_api/schema_models/category.py @@ -16,3 +16,6 @@ class Category(BaseModel): name: str number: str shortName: Optional[str] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/cpe.py b/backend/src/xfd_django/xfd_api/schema_models/cpe.py index 578df647..557e4355 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/cpe.py +++ b/backend/src/xfd_django/xfd_api/schema_models/cpe.py @@ -18,3 +18,6 @@ class Cpe(BaseModel): version: Optional[str] vendor: Optional[str] lastSeenAt: datetime + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/cve.py b/backend/src/xfd_django/xfd_api/schema_models/cve.py index 50aa9f23..50efab6b 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/cve.py +++ b/backend/src/xfd_django/xfd_api/schema_models/cve.py @@ -39,3 +39,6 @@ class Cve(BaseModel): cvssV4ImpactScore: Optional[str] weaknesses: Optional[str] references: Optional[str] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/domain.py b/backend/src/xfd_django/xfd_api/schema_models/domain.py index edfbdee6..96a30464 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/domain.py +++ b/backend/src/xfd_django/xfd_api/schema_models/domain.py @@ -36,7 +36,7 @@ class Domain(BaseModel): class Config: """Domain base schema config.""" - orm_mode = True + from_attributes = True validate_assignment = True @@ -52,6 +52,9 @@ class DomainFilters(BaseModel): vulnerabilities: Optional[str] = None tag: Optional[str] = None + class Config: + from_attributes = True + class DomainSearch(BaseModel): """DomainSearch schema.""" @@ -61,3 +64,6 @@ class DomainSearch(BaseModel): order: str filters: Optional[DomainFilters] pageSize: Optional[int] = None + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/notification.py b/backend/src/xfd_django/xfd_api/schema_models/notification.py index 91b765ab..157c4932 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/notification.py +++ b/backend/src/xfd_django/xfd_api/schema_models/notification.py @@ -22,3 +22,6 @@ class Notification(BaseModel): status: Optional[str] updatedBy: datetime message: Optional[str] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization.py b/backend/src/xfd_django/xfd_api/schema_models/organization.py index f52f2406..730d0765 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/organization.py +++ b/backend/src/xfd_django/xfd_api/schema_models/organization.py @@ -30,3 +30,6 @@ class Organization(BaseModel): county: Optional[str] countyFips: Optional[int] type: Optional[str] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/question.py b/backend/src/xfd_django/xfd_api/schema_models/question.py index 310a5ded..a2fba7a6 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/question.py +++ b/backend/src/xfd_django/xfd_api/schema_models/question.py @@ -18,3 +18,6 @@ class Question(BaseModel): longForm: str number: str categoryId: Optional[Any] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/resource.py b/backend/src/xfd_django/xfd_api/schema_models/resource.py index 59dfb40b..50b4aeba 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/resource.py +++ b/backend/src/xfd_django/xfd_api/schema_models/resource.py @@ -16,3 +16,6 @@ class Resource(BaseModel): name: str type: str url: str + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/response.py b/backend/src/xfd_django/xfd_api/schema_models/response.py index 20580b70..48cb2b68 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/response.py +++ b/backend/src/xfd_django/xfd_api/schema_models/response.py @@ -16,3 +16,6 @@ class Response(BaseModel): selection: str assessmentId: Optional[Any] questionId: Optional[Any] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/role.py b/backend/src/xfd_django/xfd_api/schema_models/role.py index 4035d24e..4dabd9ef 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/role.py +++ b/backend/src/xfd_django/xfd_api/schema_models/role.py @@ -3,11 +3,11 @@ # from pydantic.types import UUID1, UUID # Standard Python Libraries from datetime import datetime -from typing import Any, List, Optional +from typing import Any, Optional from uuid import UUID # Third-Party Libraries -from pydantic import BaseModel, Json +from pydantic import BaseModel class Role(BaseModel): @@ -22,3 +22,6 @@ class Role(BaseModel): approvedById: Optional[Any] userId: Optional[Any] organizationId: Optional[Any] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan.py b/backend/src/xfd_django/xfd_api/schema_models/scan.py index fb7803c5..4c29e5ee 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/scan.py +++ b/backend/src/xfd_django/xfd_api/schema_models/scan.py @@ -26,6 +26,9 @@ class Scan(BaseModel): manualRunPending: bool createdBy: Optional[Any] + class Config: + from_attributes = True + class ScanTask(BaseModel): """ScanTask schema.""" @@ -44,3 +47,6 @@ class ScanTask(BaseModel): queuedAt: Optional[datetime] organizationId: Optional[Any] scanId: Optional[Any] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/searchbody.py b/backend/src/xfd_django/xfd_api/schema_models/searchbody.py index cdfd884b..b0094038 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/searchbody.py +++ b/backend/src/xfd_django/xfd_api/schema_models/searchbody.py @@ -20,3 +20,6 @@ class SearchBody(BaseModel): filters: Json[Any] organizationId: Optional[UUID] tagId: Optional[UUID] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/service.py b/backend/src/xfd_django/xfd_api/schema_models/service.py index 6990def1..f9bd73b8 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/service.py +++ b/backend/src/xfd_django/xfd_api/schema_models/service.py @@ -28,3 +28,6 @@ class Service(BaseModel): wappalyzerResults: Json[Any] domainId: Optional[Any] discoveredById: Optional[Any] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/user.py b/backend/src/xfd_django/xfd_api/schema_models/user.py index d27239ee..40465fe1 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/user.py +++ b/backend/src/xfd_django/xfd_api/schema_models/user.py @@ -1,6 +1,5 @@ """User schemas.""" -# Third-Party Libraries -# from pydantic.types import UUID1, UUID + # Standard Python Libraries from datetime import datetime from typing import List, Optional @@ -9,6 +8,7 @@ # Third-Party Libraries from pydantic import BaseModel +from .api_key import ApiKey from .role import Role @@ -34,14 +34,27 @@ class User(BaseModel): state: Optional[str] oktaId: Optional[str] roles: Optional[List[Role]] = [] + apiKeys: Optional[List[ApiKey]] = [] @classmethod - def from_orm(cls, obj): - # Convert roles to a list of RoleSchema before passing to Pydantic + def model_validate(cls, obj): + # Convert fields before passing to Pydantic Schema user_dict = obj.__dict__.copy() - user_dict["roles"] = [Role.from_orm(role) for role in obj.roles.all()] + user_dict["roles"] = [ + Role.model_validate(role).model_dump() for role in obj.roles.all() + ] + user_dict["apiKeys"] = [ + ApiKey.model_validate(api_key).model_dump() for api_key in obj.apiKeys.all() + ] + [ApiKey.from_orm(api_key) for api_key in obj] return cls(**user_dict) + def model_dump(self, **kwargs): + """Override model_dump to handle UUID serialization.""" + data = super().model_dump(**kwargs) + if isinstance(data.get("id"), UUID): + data["id"] = str(data["id"]) # Convert UUID to string + return data + class Config: - orm_mode = True from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py b/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py index b217d257..60276615 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py +++ b/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py @@ -36,6 +36,9 @@ class Vulnerability(BaseModel): domainId: UUID serviceId: UUID + class Config: + from_attributes = True + class VulnerabilityFilters(BaseModel): """VulnerabilityFilters schema.""" @@ -51,6 +54,9 @@ class VulnerabilityFilters(BaseModel): tag: Optional[UUID] isKev: Optional[bool] + class Config: + from_attributes = True + class VulnerabilitySearch(BaseModel): """VulnerabilitySearch schema.""" @@ -61,3 +67,6 @@ class VulnerabilitySearch(BaseModel): filters: Optional[VulnerabilityFilters] pageSize: Optional[int] groupBy: Optional[str] + + class Config: + from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schemas.py b/backend/src/xfd_django/xfd_api/schemas.py deleted file mode 100644 index fa5fe9b8..00000000 --- a/backend/src/xfd_django/xfd_api/schemas.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Schemas.py.""" -# Third-Party Libraries -# from pydantic.types import UUID1, UUID -# Standard Python Libraries -from datetime import datetime -from typing import Any, List, Optional -from uuid import UUID - -# Third-Party Libraries -from pydantic import BaseModel, Json - -from .schema_models.assessment import Assessment -from .schema_models.category import Category -from .schema_models.cpe import Cpe -from .schema_models.cve import Cve -from .schema_models.domain import Domain, DomainFilters, DomainSearch -from .schema_models.notification import Notification -from .schema_models.organization import Organization -from .schema_models.question import Question -from .schema_models.resource import Resource -from .schema_models.response import Response -from .schema_models.role import Role -from .schema_models.scan import Scan -from .schema_models.searchbody import SearchBody -from .schema_models.service import Service -from .schema_models.user import User -from .schema_models.vulnerability import ( - Vulnerability, - VulnerabilityFilters, - VulnerabilitySearch, -) diff --git a/backend/src/xfd_django/xfd_api/tests/conftest.py b/backend/src/xfd_django/xfd_api/tests/conftest.py new file mode 100644 index 00000000..0ff48bf9 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests/conftest.py @@ -0,0 +1,9 @@ +# Third-Party Libraries +from django.core.management import call_command +import pytest + + +@pytest.fixture(scope="session") +def django_db_setup(django_db_blocker): + with django_db_blocker.unblock(): + call_command("migrate") # Apply migrations before running the tests diff --git a/backend/src/xfd_django/xfd_api/tests/pytest.ini b/backend/src/xfd_django/xfd_api/tests/pytest.ini new file mode 100644 index 00000000..91833ec4 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +DJANGO_SETTINGS_MODULE = xfd_django.test_settings +# django_find_project = true +# addopts = --migrations --reuse-db +python_files = tests.py test_*.py *_tests.py diff --git a/backend/src/xfd_django/xfd_api/tests/test_api_key.py b/backend/src/xfd_django/xfd_api/tests/test_api_key.py new file mode 100644 index 00000000..b7b240cf --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests/test_api_key.py @@ -0,0 +1,203 @@ +# Standard Python Libraries +from datetime import datetime +import secrets + +# Third-Party Libraries +from fastapi.testclient import TestClient +import pytest +from xfd_api.auth import ( # Assuming this function generates a user token + create_jwt_token, +) +from xfd_api.models import ( # Adjust the import based on your project structure + ApiKey, + User, +) +from xfd_django.asgi import app # Import the FastAPI app + +client = TestClient(app) + + +@pytest.mark.django_db +def test_generate_api_key(): + user = User.objects.create( + firstName="Test", + lastName="User", + email=f"{secrets.token_hex(4)}@example.com", + userType="STANDARD", + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + response = client.post( + "/api-keys", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + assert response.status_code == 200 + + response2 = client.get( + "/users/me", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + assert response2.status_code == 200 + + assert response.json()["hashedKey"] == response2.json()["apiKeys"][0]["hashedKey"] + assert response.json()["key"][-4:] == response2.json()["apiKeys"][0]["lastFour"] + assert len(response2.json()["apiKeys"]) == 1 + assert "key" not in response2.json()["apiKeys"][0] + + +@pytest.mark.django_db +def test_delete_own_api_key(): + user = User.objects.create( + firstName="Test", + lastName="User", + email=f"{secrets.token_hex(4)}@example.com", + userType="STANDARD", + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + api_key = ApiKey.objects.create( + hashedKey="1234", + lastFour="1234", + userId=user, + ) + response = client.delete( + f"/api-keys/{api_key.id}", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + assert response.status_code == 200 + + response2 = client.get( + "/users/me", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + assert response2.status_code == 200 + assert len(response2.json()["apiKeys"]) == 0 + + +@pytest.mark.django_db +def test_delete_other_users_api_key_fails(): + user1 = User.objects.create( + firstName="Test1", + lastName="User1", + email=f"{secrets.token_hex(4)}@example.com", + userType="STANDARD", + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + user2 = User.objects.create( + firstName="Test2", + lastName="User2", + email=f"{secrets.token_hex(4)}@example.com", + userType="GLOBAL_ADMIN", + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + api_key = ApiKey.objects.create( + hashedKey="1234", + lastFour="1234", + userId=user1, + ) + + # Try to delete user1's API key as user2 + response = client.delete( + f"/api-keys/{api_key.id}", + headers={ + "Authorization": create_jwt_token( + {"id": user2.id, "userType": "GLOBAL_ADMIN"} + ) + }, + ) + assert response.status_code == 404 + + # Verify user1's API key still exists + response2 = client.get( + "/users/me", + headers={ + "Authorization": create_jwt_token({"id": user1.id, "userType": "STANDARD"}) + }, + ) + assert response2.status_code == 200 + assert len(response2.json()["apiKeys"]) == 1 + + +@pytest.mark.django_db +def test_using_valid_api_key(): + user = User.objects.create( + firstName="Test", + lastName="User", + email=f"{secrets.token_hex(4)}@example.com", + userType="STANDARD", + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + response = client.post( + "/api-keys", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + assert response.status_code == 200 + api_key = response.json()["key"] + + # Verify user info with API key + response_with_api_key = client.get("/users/me", headers={"Authorization": api_key}) + assert response_with_api_key.status_code == 200 + + +@pytest.mark.django_db +def test_using_invalid_api_key(): + User.objects.create( + firstName="Test", + lastName="User", + email=f"{secrets.token_hex(4)}@example.com", + userType="STANDARD", + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.get("/users/me", headers={"Authorization": "invalid_key"}) + assert response.status_code == 401 + + +@pytest.mark.django_db +def test_using_revoked_api_key(): + user = User.objects.create( + firstName="Test", + lastName="User", + email=f"{secrets.token_hex(4)}@example.com", + userType="STANDARD", + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.post( + "/api-keys", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + assert response.status_code == 200 + api_key_id = response.json()["id"] + + # Revoke the API key + response = client.delete( + f"/api-keys/{api_key_id}", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + assert response.status_code == 200 + + # Verify revoked API key fails + response_with_revoked_key = client.get( + "/users/me", headers={"Authorization": response.json()["key"]} + ) + assert response_with_revoked_key.status_code == 401 diff --git a/backend/src/xfd_django/xfd_api/tests/test_auth.py b/backend/src/xfd_django/xfd_api/tests/test_auth.py new file mode 100644 index 00000000..536cd312 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests/test_auth.py @@ -0,0 +1,119 @@ +# Standard Python Libraries +from datetime import datetime + +# Third-Party Libraries +from fastapi.testclient import TestClient +import pytest +from xfd_api.models import User +from xfd_django.asgi import app + +client = TestClient(app) + + +@pytest.mark.django_db +def test_login_success(): + # Mock login request + response = client.post("/auth/login") + assert response.status_code == 200 + assert response.json() + + +@pytest.mark.django_db +def test_callback_success_login_gov(): + # Simulate login via login.gov + response = client.post( + "/auth/callback", + json={ + "code": "CODE", + "state": "STATE", + "origState": "ORIGSTATE", + "nonce": "NONCE", + }, + ) + assert response.status_code == 200 + assert response.json()["token"] + assert response.json()["user"] + assert response.json()["user"]["email"] == "test@crossfeed.cisa.gov" + + # Verify user in the database + user = User.objects.get(id=response.json()["user"]["id"]) + assert user.firstName == "" + assert user.lastName == "" + + +@pytest.mark.django_db +def test_callback_success_cognito(): + # Simulate Cognito login + response = client.post( + "/auth/callback", json={"token": "TOKEN_test2@crossfeed.cisa.gov"} + ) + assert response.status_code == 200 + assert response.json()["token"] + assert response.json()["user"] + assert response.json()["user"]["email"] == "test2@crossfeed.cisa.gov" + + # Verify user in the database + user = User.objects.get(id=response.json()["user"]["id"]) + assert user.firstName == "" + assert user.lastName == "" + + +@pytest.mark.django_db +def test_callback_cognito_overwrite_cognito_id(): + # Simulate Cognito login with two different IDs + response = client.post( + "/auth/callback", json={"token": "TOKEN_test3@crossfeed.cisa.gov"} + ) + assert response.status_code == 200 + user_id = response.json()["user"]["id"] + cognito_id = response.json()["user"]["cognitoId"] + + response = client.post( + "/auth/callback", json={"token": "TOKEN_test3@crossfeed.cisa.gov"} + ) + user = User.objects.get(id=response.json()["user"]["id"]) + assert user.id == user_id + assert user.cognitoId != cognito_id + + +@pytest.mark.django_db +def test_login_gov_then_cognito_preserves_ids(): + # Simulate login via login.gov + response = client.post( + "/auth/callback", + json={ + "code": "CODE", + "state": "STATE", + "origState": "ORIGSTATE", + "nonce": "NONCE", + }, + ) + assert response.status_code == 200 + user_id = response.json()["user"]["id"] + login_gov_id = response.json()["user"]["loginGovId"] + + # Simulate subsequent Cognito login + response = client.post( + "/auth/callback", json={"token": "TOKEN_test@crossfeed.cisa.gov"} + ) + assert response.status_code == 200 + + user = User.objects.get(id=response.json()["user"]["id"]) + assert user.id == user_id + assert user.loginGovId == login_gov_id + assert user.cognitoId is not None + + +@pytest.mark.django_db +def test_last_logged_in_is_updated(): + # Simulate Cognito login and check lastLoggedIn timestamp + time_1 = datetime.now() + response = client.post( + "/auth/callback", json={"token": "TOKEN_test4@crossfeed.cisa.gov"} + ) + assert response.status_code == 200 + time_2 = datetime.now() + + assert response.json()["token"] + assert response.json()["user"]["lastLoggedIn"] + assert time_1 <= response.json()["user"]["lastLoggedIn"] <= time_2 diff --git a/backend/src/xfd_django/xfd_api/tests/test_notification.py b/backend/src/xfd_django/xfd_api/tests/test_notification.py new file mode 100644 index 00000000..560ac900 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests/test_notification.py @@ -0,0 +1,146 @@ +# Third-Party Libraries +from fastapi.testclient import TestClient +import pytest +from xfd_api.auth import ( # Adjust import based on your auth implementation + create_jwt_token, +) +from xfd_api.models import ( # Adjust import based on your project structure + Notification, + User, +) +from xfd_django.asgi import app # Import your FastAPI app + +client = TestClient(app) + + +@pytest.mark.django_db +def test_create_notification(): + # Create a test user + user = User.objects.create( + firstName="Test", + lastName="User", + email="testuser@example.com", + userType="STANDARD", + ) + + # Define the notification data + notification_data = {"message": "Test notification", "status": "unread"} + + # Send the POST request + response = client.post( + "/notifications", + json=notification_data, + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + + # Assert the response status code + assert response.status_code == 200 + + # Assert the notification was created correctly + response_data = response.json() + assert response_data["message"] == "Test notification" + assert response_data["status"] == "unread" + assert "id" in response_data # Ensure that ID was assigned + + +@pytest.mark.django_db +def test_delete_notification(): + # Create a test user + user = User.objects.create( + firstName="Test", + lastName="User", + email="testuser@example.com", + userType="STANDARD", + ) + + # Create a test notification + notification = Notification.objects.create( + message="Test notification", status="unread", userId=user + ) + + # Send the DELETE request + response = client.delete( + f"/notifications/{notification.id}", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + + # Assert the response status code + assert response.status_code == 200 + + # Assert the response content + response_data = response.json() + assert response_data["status"] == "success" + assert response_data["message"] == "Item deleted successfully" + + # Assert the notification no longer exists + with pytest.raises(Notification.DoesNotExist): + Notification.objects.get(id=notification.id) + + +@pytest.mark.django_db +def test_get_all_notifications(): + # Create a test user + user = User.objects.create( + firstName="Test", + lastName="User", + email="testuser@example.com", + userType="STANDARD", + ) + + # Create two test notifications + Notification.objects.create(message="Notification 1", status="unread", userId=user) + Notification.objects.create(message="Notification 2", status="read", userId=user) + + # Send the GET request + response = client.get( + "/notifications", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + + # Assert the response status code + assert response.status_code == 200 + + # Assert that two notifications are returned + notifications = response.json() + assert len(notifications) == 2 + assert notifications[0]["message"] == "Notification 1" + assert notifications[1]["message"] == "Notification 2" + + +@pytest.mark.django_db +def test_get_notification_by_id(): + # Create a test user + user = User.objects.create( + firstName="Test", + lastName="User", + email="testuser@example.com", + userType="STANDARD", + ) + + # Create a test notification + notification = Notification.objects.create( + message="Test notification", status="unread", userId=user + ) + + # Send the GET request + response = client.get( + f"/notifications/{notification.id}", + headers={ + "Authorization": create_jwt_token({"id": user.id, "userType": "STANDARD"}) + }, + ) + + # Assert the response status code + assert response.status_code == 200 + + # Assert the correct notification is returned + response_data = response.json() + assert response_data["message"] == "Test notification" + assert response_data["status"] == "unread" + assert response_data["id"] == str(notification.id) diff --git a/backend/src/xfd_django/xfd_api/tests/test_user.py b/backend/src/xfd_django/xfd_api/tests/test_user.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 5947ab4b..ca1f9a17 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -17,26 +17,26 @@ - .models """ # Standard Python Libraries -import hashlib -import secrets from typing import List, Optional -import uuid # Third-Party Libraries -from fastapi import APIRouter, Depends, HTTPException, Query, Request, status -from fastapi.responses import JSONResponse -from pydantic import model_serializer - -from .auth import get_current_active_user, get_jwt_from_code, process_user -from .login_gov import login -from .models import ApiKey, Cpe, Cve, Domain, Organization, User, Vulnerability -from .schemas import Cpe as CpeSchema -from .schemas import Cve as CveSchema -from .schemas import Domain as DomainSchema -from .schemas import DomainSearch -from .schemas import Organization as OrganizationSchema -from .schemas import User as UserSchema -from .schemas import Vulnerability as VulnerabilitySchema +from fastapi import APIRouter, Depends, HTTPException, Query, Request + +from .api_methods import api_key as api_key_methods +from .api_methods import auth as auth_methods +from .api_methods import notification as notification_methods +from .auth import get_current_active_user +from .login_gov import callback, login +from .models import Cpe, Cve, Domain, Organization, User, Vulnerability +from .schema_models.api_key import ApiKey as ApiKeySchema +from .schema_models.cpe import Cpe as CpeSchema +from .schema_models.cve import Cve as CveSchema +from .schema_models.domain import Domain as DomainSchema +from .schema_models.domain import DomainSearch +from .schema_models.notification import Notification as NotificationSchema +from .schema_models.organization import Organization as OrganizationSchema +from .schema_models.user import User as UserSchema +from .schema_models.vulnerability import Vulnerability as VulnerabilitySchema api_router = APIRouter() @@ -256,8 +256,52 @@ async def update_vulnerability(vuln_id, data: VulnerabilitySchema): raise HTTPException(status_code=500, detail=str(e)) +###################### +# Auth +###################### + + +# Okta Callback +@api_router.post("/auth/okta-callback", tags=["auth"]) +async def okta_callback(request: Request): + """Handle Okta Callback.""" + return auth_methods.handle_okta_callback(request) + + +# Login +@api_router.get("/login", tags=["auth"]) +async def login_route(): + """Handle V1 Login.""" + return login() + + +# V1 Callback +@api_router.post("/auth/callback", tags=["auth"]) +async def callback_route(request: Request): + """Handle V1 Callback.""" + body = await request.json() + try: + user_info = callback(body) + return user_info + except Exception as error: + raise HTTPException(status_code=400, detail=str(error)) + + +###################### +# Users +###################### + + +# GET Current User +@api_router.get("/users/me", tags=["users"]) +async def read_users_me(current_user: User = Depends(get_current_active_user)): + return current_user + + +# V2 GET Users @api_router.get( "/v2/users", + tags=["users"], response_model=List[UserSchema], # dependencies=[Depends(get_current_active_user)], ) @@ -304,191 +348,6 @@ async def get_users( return [UserSchema.from_orm(user) for user in users] -@api_router.get("/login") -async def login_route(): - login_data = login() - return login_data - - -@api_router.post("/auth/callback") -async def callback_route(request: Request): - body = await request.json() - try: - user_info = callback(body) - return user_info - except Exception as e: - raise HTTPException(status_code=400, detail=str(e)) - - -# @api_router.post("/auth/callback") -# async def callback_endpoint(request: Request): -# body = request.json() -# print(f"body: {body}") -# try: -# if os.getenv("USE_COGNITO"): -# token, user = await handle_cognito_callback(body) -# else: -# user_info = await handle_callback(body) -# user = await update_or_create_user(user_info) -# token = create_jwt_token(user) -# return JSONResponse(content={"token": token, "user": user}) -# except Exception as error: -# raise HTTPException( -# status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(error) -# ) from error - - -@api_router.post("/auth/okta-callback") -async def callback(request: Request): - print(f"Request from /auth/okta-callback: {str(request)}") - body = await request.json() - print(f"Request json from callback: {str(request)}") - print(f"Request json from callback: {body}") - print(f"Body type: {type(body)}") - code = body.get("code") - print(f"Code: {code}") - if not code: - return HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Code not found in request body", - ) - jwt_data = await get_jwt_from_code(code) - print(f"JWT Data: {jwt_data}") - if jwt_data is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Invalid authorization code or failed to retrieve tokens", - ) - access_token = jwt_data.get("access_token") - refresh_token = jwt_data.get("refresh_token") - decoded_token = jwt_data.get("decoded_token") - - resp = await process_user(decoded_token, access_token, refresh_token) - token = resp.get("token") - - # print(f"Response from process_user: {json.dumps(resp)}") - # Response(status_code=200, set("crossfeed-token", resp.get("token")) - # return json.dumps(resp) - # Create a JSONResponse object to return the response and set the cookie - response = JSONResponse( - content={"message": "User authenticated", "data": resp, "token": token} - ) - response.body = resp - # response.body = resp - response.set_cookie(key="token", value=token) - - # Set the 'crossfeed-token' cookie - response.set_cookie( - key="crossfeed-token", - value=token, - # httponly=True, # This makes the cookie inaccessible to JavaScript - # secure=True, # Ensures the cookie is only sent over HTTPS - # samesite="Lax" # Restricts when cookies are sent, adjust as necessary (e.g., "Strict" or "None") - ) - return response - - -# @api_router.post("/auth/login") -# async def login_endpoint(): -# print(f"Returning auth/login response") -# # result = await get_login_gov() -# # return JSONResponse(content=result) - - -# @api_router.get("/login") -# async def login_route(): -# login_data = login() -# return login_data - - -# @api_router.post("/auth/callback") -# async def callback_route(request: Request): -# body = await request.json() -# try: -# user_info = callback(body) -# return user_info -# except Exception as e: -# raise HTTPException(status_code=400, detail=str(e)) - - -# @api_router.post("/auth/callback") -# async def callback_endpoint(request: Request): -# body = request.json() -# print(f"body: {body}") -# try: -# if os.getenv("USE_COGNITO"): -# token, user = await handle_cognito_callback(body) -# else: -# user_info = await handle_callback(body) -# user = await update_or_create_user(user_info) -# token = create_jwt_token(user) -# return JSONResponse(content={"token": token, "user": user}) -# except Exception as error: -# raise HTTPException( -# status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(error) -# ) from error - - -# @api_router.post("/auth/okta-callback") -# async def callback(request: Request): -# print(f"Request from /auth/okta-callback: {str(request)}") -# body = await request.json() -# print(f"Request json from callback: {str(request)}") -# print(f"Request json from callback: {body}") -# print(f"Body type: {type(body)}") -# code = body.get("code") -# print(f"Code: {code}") -# if not code: -# return HTTPException( -# status_code=status.HTTP_400_BAD_REQUEST, -# detail="Code not found in request body", -# ) -# jwt_data = await get_jwt_from_code(code) -# print(f"JWT Data: {jwt_data}") -# if jwt_data is None: -# raise HTTPException( -# status_code=status.HTTP_400_BAD_REQUEST, -# detail="Invalid authorization code or failed to retrieve tokens", -# ) -# access_token = jwt_data.get("access_token") -# refresh_token = jwt_data.get("refresh_token") -# decoded_token = jwt_data.get("decoded_token") - -# resp = await process_user(decoded_token, access_token, refresh_token) -# token = resp.get("token") - -# # print(f"Response from process_user: {json.dumps(resp)}") -# # Response(status_code=200, set("crossfeed-token", resp.get("token")) -# # return json.dumps(resp) -# # Create a JSONResponse object to return the response and set the cookie -# response = JSONResponse(content={"message": "User authenticated", "data": resp, "token": token}) -# response.body = resp -# # response.body = resp -# response.set_cookie(key="token", value=token) - -# # Set the 'crossfeed-token' cookie -# response.set_cookie( -# key="crossfeed-token", -# value=token, -# # httponly=True, # This makes the cookie inaccessible to JavaScript -# # secure=True, # Ensures the cookie is only sent over HTTPS -# # samesite="Lax" # Restricts when cookies are sent, adjust as necessary (e.g., "Strict" or "None") -# ) - -# return response - - -# @api_router.get("/users/me", response_model=User) -# async def get_me(request: Request): -# user = get_current_active_user(request) -# return user - - -@api_router.get("/users/me") -async def read_users_me(current_user: User = Depends(get_current_active_user)): - return current_user - - # @api_router.post("/users/me/acceptTerms") # async def accept_terms(request: Request): # user = await get_current_active_user(request) @@ -496,153 +355,109 @@ async def read_users_me(current_user: User = Depends(get_current_active_user)): # body = await request.json() # if not body: -# raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid request body") +# raise HTTPException( +# status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid request body" +# ) # user.dateAcceptedTerms = datetime.utcnow() -# user.acceptedTermsVersion = body.get('version') +# user.acceptedTermsVersion = body.get("version") # user.save() # return JSONResponse(content=user.to_dict(), status_code=status.HTTP_200_OK) -# -# -# @api_router.post("/users/me/acceptTerms") -# async def accept_terms( -# version: str, current_user: User = Depends(get_current_active_user) -# ): -# current_user.date_accepted_terms = datetime.utcnow() -# current_user.accepted_terms_version = version -# current_user.save() -# return current_user - - -# POST /apikeys/generate -@api_router.post("/api-keys") -async def generate_api_key(current_user: User = Depends(get_current_active_user)): - """ - Generate API key for user - Args: - current_user (User, optional): _description_. Defaults to Depends(get_current_active_user). - Returns: - dict: ApiKey model dict - """ - # Generate a random 16-byte API key - key = secrets.token_hex(16) - - # Hash the API key - hashed_key = hashlib.sha256(key.encode()).hexdigest() - - # Create the ApiKey record in the database - api_key = ApiKey.objects.create( - id=uuid.uuid4(), - hashedKey=hashed_key, - lastFour=key[-4:], # Store the last four characters of the key - userId=current_user, - ) - - # Return the API key to the user (Do NOT store the plain key in the database) - return { - "id": api_key.id, - "status": "success", - "api_key": key, - "last_four": api_key.lastFour, - } - - -# DELETE /apikeys/{keyId} -@api_router.delete("/api-keys/{key_id}") +###################### +# API-Keys +###################### + + +# POST +@api_router.post("/api-keys", response_model=ApiKeySchema, tags=["api-keys"]) +async def create_api_key(current_user: User = Depends(get_current_active_user)): + """Create api key.""" + return api_key_methods.post(current_user) + + +# DELETE +@api_router.delete("/api-keys/{id}", tags=["api-keys"]) async def delete_api_key( - key_id: str, current_user: User = Depends(get_current_active_user) + id: str, current_user: User = Depends(get_current_active_user) ): - try: - # Validate that key_id is a valid UUID - uuid.UUID(key_id) - except ValueError: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Invalid API Key ID" - ) - - # Try to find the ApiKey by key_id and current user - try: - api_key = ApiKey.objects.get(id=key_id, userId=current_user) - except ApiKey.DoesNotExist: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="API Key not found" - ) + """Delete api key by id.""" + return api_key_methods.delete(id, current_user) - # Delete the API Key - api_key.delete() - return {"status": "success", "message": "API Key deleted successfully"} +# GET ALL +@api_router.get("/api-keys", response_model=List[ApiKeySchema], tags=["api-keys"]) +async def get_all_api_keys(current_user: User = Depends(get_current_active_user)): + """Get all api keys.""" + return api_key_methods.get_all(current_user) -@api_router.get("/api-keys") -async def get_api_keys(): - """ - Get all API keys. - Returns: - list: A list of all API keys. - """ - try: - api_keys = ApiKey.objects.all() - return [ - { - "id": api_key.id, - "created_at": api_key.createdAt, - "updated_at": api_key.updatedAt, - "last_used": api_key.lastUsed, - "hashed_key": api_key.hashedKey, - "last_four": api_key.lastFour, - "user_id": api_key.userId.id if api_key.userId else None, - } - for api_key in api_keys - ] - except Exception as error: - raise HTTPException(status_code=500, detail=str(error)) +# GET BY ID +@api_router.get("/api-keys/{id}", response_model=ApiKeySchema, tags=["api-keys"]) +async def get_api_key(id: str, current_user: User = Depends(get_current_active_user)): + """Get api key by id.""" + return api_key_methods.get_by_id(id, current_user) -@api_router.post("/notifications") -async def create_notification(): - """ - Create Notification - """ - return [] +######################### +# Notifications +######################### -@api_router.get("/notifications") -async def get_all_notifications(): - """ - Get All Notifications - """ +# POST +@api_router.post( + "/notifications", response_model=NotificationSchema, tags=["notifications"] +) +async def create_notification(current_user: User = Depends(get_current_active_user)): + """Create notification key.""" + # return notification_handler.post(current_user) return [] -@api_router.get("/notifications/{id}") -async def get_notification(): - """ - Get Notification by id - """ - return [] +# DELETE +@api_router.delete( + "/notifications/{id}", response_model=NotificationSchema, tags=["notifications"] +) +async def delete_notification( + id: str, current_user: User = Depends(get_current_active_user) +): + """Delete notification by id.""" + return notification_methods.delete(id, current_user) -@api_router.put("/notifications/{id}") -async def update_notifications(): - """ - Update Notification - """ - return [] +# GET ALL +@api_router.get( + "/notifications", response_model=List[NotificationSchema], tags=["notifications"] +) +async def get_all_notifications(current_user: User = Depends(get_current_active_user)): + """Get all notifications.""" + return notification_methods.get_all(current_user) -@api_router.delete("/notifications/{id}") -async def delete_notifications(): - """ - Delete Notification - """ - return [] +# GET BY ID +@api_router.get( + "/notifications/{id}", response_model=NotificationSchema, tags=["notifications"] +) +async def get_notification( + id: str, current_user: User = Depends(get_current_active_user) +): + """Get notification by id.""" + return notification_methods.get_by_id(id, current_user) + + +# UPDATE BY ID +@api_router.put("/notifications/{id}", tags=["notifications"]) +async def update_notification( + id: str, current_user: User = Depends(get_current_active_user) +): + """Update notification key by id.""" + return notification_methods.delete(id, current_user) -@api_router.get("/notifications/508-banner") -async def notification_banner(): - """ """ - return "" +# GET 508 Banner +@api_router.get("/notifications/508-banner", tags=["notifications"]) +async def get_508_banner(current_user: User = Depends(get_current_active_user)): + """Get notification by id.""" + return notification_methods.get_508_banner(current_user) diff --git a/backend/src/xfd_django/xfd_django/test_settings.py b/backend/src/xfd_django/xfd_django/test_settings.py new file mode 100644 index 00000000..eec8e80d --- /dev/null +++ b/backend/src/xfd_django/xfd_django/test_settings.py @@ -0,0 +1,136 @@ +# test_settings.py +# Standard Python Libraries +import mimetypes + +# Python built-in +from pathlib import Path +from re import A + +# Third-Party Libraries +from django.contrib.messages import constants as messages + +mimetypes.add_type("text/css", ".css", True) +mimetypes.add_type("text/html", ".html", True) + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +# TODO: GET THAT LATER +SECRET_KEY = "django-insecure-255j80npx26z%x0@-7p@(qs9(yvtuuln#xuhxt_x$bbevvxnm!" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [ + "http://localhost", + "http://localhost:3000", + ".execute-api.us-east-1.amazonaws.com", + "https://api.staging-cd.crossfeed.cyber.dhs.gov", +] + +MESSAGE_TAGS = { + messages.DEBUG: "alert-secondary", + messages.INFO: "alert-info", + messages.SUCCESS: "alert-success", + messages.WARNING: "alert-warning", + messages.ERROR: "alert-danger", +} + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "xfd_api.apps.XfdApiConfig", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "xfd_django.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + + +# Use an in-memory SQLite database for testing +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "DIR": ":memory:", + } +} + +# Password validation +# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +PROJECT_NAME = "XFD Python API" + +DJANGO_SETTINGS_MODULE = "xfd_django.test_settings" From 7ba7d0de00ec9e187b83317e62cf717a45eb5f53 Mon Sep 17 00:00:00 2001 From: nickviola Date: Tue, 1 Oct 2024 09:38:41 -0500 Subject: [PATCH 036/314] Update docker-compose to fix matomodb build errors --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4b205f95..a749f504 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,7 +128,7 @@ services: environment: - MYSQL_ROOT_PASSWORD=password logging: - driver: none + driver: "json-file" matomo: image: matomo:3.14.1 From 7c3eee3362fdc09d33431e65e08dffb3a9171ae7 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 1 Oct 2024 16:16:28 -0400 Subject: [PATCH 037/314] Add scan task endpoints --- .../xfd_django/xfd_api/api_methods/scan.py | 2 +- .../xfd_api/api_methods/scan_tasks.py | 187 ++++++++++++++++++ backend/src/xfd_django/xfd_api/auth.py | 6 +- .../xfd_api/schema_models/scan_tasks.py | 52 +++++ .../xfd_django/xfd_api/tasks/ecs_client.py | 6 +- backend/src/xfd_django/xfd_api/views.py | 46 +++++ 6 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/scan.py b/backend/src/xfd_django/xfd_api/api_methods/scan.py index 2cb934d2..112d2722 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/scan.py +++ b/backend/src/xfd_django/xfd_api/api_methods/scan.py @@ -1,4 +1,4 @@ -"""API methods to support Scan enpoints.""" +"""API methods to support Scan endpoints.""" # Standard Python Libraries import os diff --git a/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py b/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py new file mode 100644 index 00000000..839b97cb --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py @@ -0,0 +1,187 @@ +"""API methods to support scan task endpoints.""" + +# Standard Python Libraries +from datetime import datetime, timezone + +# Third-Party Libraries +from fastapi import HTTPException, Response, status + +from ..auth import is_global_view_admin, is_global_write_admin, get_tag_organizations +from ..models import ScanTask +from ..schema_models.scan_tasks import ScanTaskSearch +from ..tasks.ecs_client import ECSClient + + +PAGE_SIZE = 15; + +def list_scan_tasks(search_data: ScanTaskSearch, current_user): + """List scans tasks based on search filter.""" + try: + # Check if the user is a GlobalViewAdmin + if not is_global_view_admin(current_user): + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + + # Validate and parse the request body + pageSize = search_data.pageSize or PAGE_SIZE + + # Determine the correct ordering based on the 'order' field + ordering_field = f"-{search_data.sort}" if search_data.order.upper() == "DESC" else search_data.sort + + # Construct query based on filters + qs = ScanTask.objects.select_related("scanId").prefetch_related("organizations").order_by(ordering_field) + + # Apply filters to queryset safely + filters = search_data.filters + if filters: + if filters.get("name"): + qs = qs.filter(scanId__name__icontains=filters["name"]) + if filters.get("status"): + qs = qs.filter(status__icontains=filters["status"]) + if filters.get("organization"): + qs = qs.filter(organizations__id=filters["organization"]) + if filters.get("tag"): + print("We are in tags") + orgs = get_tag_organizations(current_user, filters["tag"]) + print(orgs) + qs = qs.filter(organizations__id__in=orgs) + print(qs) + + # Paginate results + if pageSize != -1: + qs = qs[(search_data.page - 1) * pageSize: search_data.page * pageSize] + + # Convert queryset into a serialized response + results = [] + for task in qs: + # Ensure scanId is not None before accessing its properties + if task.scanId is None: + print(f"Warning: ScanTask {task.id} has no scanId associated.") + scan_data = None # or some default values, depending on how you want to handle this case + else: + scan_data = { + "id": str(task.scanId.id), + "createdAt": task.scanId.createdAt.isoformat() + 'Z', + "updatedAt": task.scanId.updatedAt.isoformat() + 'Z', + "name": task.scanId.name, + "arguments": task.scanId.arguments, + "frequency": task.scanId.frequency, + "lastRun": task.scanId.lastRun.isoformat() + 'Z' if task.scanId.lastRun else None, + "isGranular": task.scanId.isGranular, + "isUserModifiable": task.scanId.isUserModifiable, + "isSingleScan": task.scanId.isSingleScan, + "manualRunPending": task.scanId.manualRunPending + } + results.append({ + "id": str(task.id), + "createdAt": task.createdAt.isoformat() + 'Z', + "updatedAt": task.updatedAt.isoformat() + 'Z', + "status": task.status, + "type": task.type, + "fargateTaskArn": task.fargateTaskArn, + "input": task.input, + "output": task.output, + "requestedAt": task.requestedAt.isoformat() + 'Z' if task.requestedAt else None, + "startedAt": task.startedAt.isoformat() + 'Z' if task.startedAt else None, + "finishedAt": task.finishedAt.isoformat() + 'Z' if task.finishedAt else None, + "queuedAt": task.queuedAt.isoformat() + 'Z' if task.queuedAt else None, + "scan": scan_data, + "organizations": [ + { + "id": str(org.id), + "createdAt": org.createdAt.isoformat() + 'Z', + "updatedAt": org.updatedAt.isoformat() + 'Z', + "acronym": org.acronym, + "name": org.name, + "rootDomains": org.rootDomains, + "ipBlocks": org.ipBlocks, + "isPassive": org.isPassive, + "pendingDomains": org.pendingDomains, + "country": org.country, + "state": org.state, + "regionId": org.regionId, + "stateFips": org.stateFips, + "stateName": org.stateName, + "county": org.county, + "countyFips": org.countyFips, + "type": org.type + } + for org in task.organizations.all() + ] + }) + + count = qs.count() + response = {"result": results, "count": count} + + return response + + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + + + +def kill_scan_task(scan_task_id, current_user): + """Kill a particular scan task.""" + try: + # Check if the user is a GlobalWriteAdmin + if not is_global_write_admin(current_user): + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + # Check if scan_task_id is valid + try: + scan_task = ScanTask.objects.get(id=scan_task_id) + except ScanTask.DoesNotExist: + raise HTTPException(status_code=404, detail="ScanTask not found.") + + # Check if scan task is already finished or failed + if scan_task.status in ['failed', 'finished']: + raise HTTPException(status_code=400, detail="ScanTask has already finished.") + + # Update scan task status to 'failed' + utc_now = datetime.now(timezone.utc) + scan_task.status = 'failed' + scan_task.finishedAt = utc_now + scan_task.output = f"Manually stopped at {utc_now.isoformat()}" + scan_task.save() + + return {"statusCode": 200, "message": "ScanTask successfully marked as failed."} + + + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + + +def get_scan_task_logs(scan_task_id, current_user): + """Get scan task logs.""" + try: + # Check if the user is a GlobalViewAdmin + if not is_global_view_admin(current_user): + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + + # Check if scan_task_id is valid + try: + scan_task = ScanTask.objects.get(id=scan_task_id) + except ScanTask.DoesNotExist: + raise HTTPException(status_code=404, detail="ScanTask not found.") + + # Ensure fargateTaskArn exists + if not scan_task.fargateTaskArn: + raise HTTPException(status_code=404, detail="No logs available for this ScanTask.") + + # Retrieve logs from the ECSClient + ecs_client = ECSClient() + logs = ecs_client.get_logs(scan_task.fargateTaskArn) + + return Response(content=logs or '', status_code=status.HTTP_200_OK) + + + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 6b256b89..dd864e83 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -81,7 +81,11 @@ def get_tag_organizations(current_user, tag_id: str) -> list[str]: return [] # Fetch the OrganizationTag and its related organizations - tag = OrganizationTag.objects.prefetch_related("organizations").filter(id=tag_id).first() + try: + tag = OrganizationTag.objects.prefetch_related("organizations").filter(id=tag_id).first() + except Exception: + return [] + if tag: # Return a list of organization IDs return [org.id for org in tag.organizations.all()] diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py b/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py new file mode 100644 index 00000000..7518218c --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py @@ -0,0 +1,52 @@ +"""Schemas to support scan task endpoints.""" + +# Standard Python Libraries +from datetime import datetime +from typing import Any, List, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + +from .organization import Organization +from .scan import Scan + + +class ScanTaskSearch(BaseModel): + """Scan-task search schema.""" + + page: int + pageSize: int + sort: str + order: str + filters: Any + +class ScanTaskList(BaseModel): + """Single scan-task schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + status: str + type: str + fargateTaskArn: str + input: str + output: Optional[str] + requestedAt: Optional[datetime] + startedAt: Optional[datetime] + finishedAt: Optional[datetime] + queuedAt: Optional[datetime] + scan: Optional[Scan] + organization: Optional[List[Organization]] = [] + +class ScanTaskListResponse(BaseModel): + """Scan-task list schema.""" + + result: List[ScanTaskList] + count: int + +class GenericResponse(BaseModel): + """Generic scan task response.""" + + statusCode: int + message: str \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/tasks/ecs_client.py b/backend/src/xfd_django/xfd_api/tasks/ecs_client.py index a2a47f7a..7d3381b6 100644 --- a/backend/src/xfd_django/xfd_api/tasks/ecs_client.py +++ b/backend/src/xfd_django/xfd_api/tasks/ecs_client.py @@ -145,13 +145,15 @@ async def run_command(self, command_options): ) return response - async def get_logs(self, fargate_task_arn): + def get_logs(self, fargate_task_arn): """Gets logs for a specific Fargate or Docker task.""" if self.is_local: + # Retrieve logs as bytes from the Docker container log_stream = self.docker.containers.get(fargate_task_arn).logs( stdout=True, stderr=True, timestamps=True ) - return "".join(line[8:] for line in log_stream.split("\n")) + # Process and return the logs + return "\n".join(line for line in log_stream.decode("utf-8").splitlines()) else: log_stream_name = f"worker/main/{fargate_task_arn.split('/')[-1]}" response = self.cloudwatch_logs.get_log_events( diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 8b3e7f11..cc7f9fcb 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -22,8 +22,10 @@ # Third-Party Libraries from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import UUID4 from .api_methods import scan +from .api_methods import scan_tasks from .api_methods.api_keys import get_api_keys from .api_methods.cpe import get_cpes_by_id from .api_methods.cve import get_cves_by_id, get_cves_by_name @@ -34,6 +36,7 @@ from .auth import get_current_active_user from .models import Assessment, User from .schema_models import scan as scanSchema +from .schema_models import scan_tasks as scanTaskSchema from .schema_models.assessment import Assessment from .schema_models.cpe import Cpe as CpeSchema from .schema_models.cve import Cve as CveSchema @@ -413,3 +416,46 @@ async def invoke_scheduler(current_user: User = Depends(get_current_active_user) """Manually invoke the scan scheduler.""" response = await scan.invoke_scheduler(current_user) return response + + +# ======================================== +# Scan Task Endpoints +# ======================================== + + +@api_router.post( + "/scan-tasks/search", + dependencies=[Depends(get_current_active_user)], + response_model=scanTaskSchema.ScanTaskListResponse, + tags=["Scan Tasks"], +) +async def list_scan_tasks( + search_data: scanTaskSchema.ScanTaskSearch, current_user: User = Depends(get_current_active_user) +): + """List scan tasks based on filters.""" + return scan_tasks.list_scan_tasks(search_data, current_user) + + +@api_router.post( + "/scan-tasks/{scan_task_id}/kill", + dependencies=[Depends(get_current_active_user)], + tags=["Scan Tasks"], +) +async def kill_scan_tasks( + scan_task_id: UUID4, current_user: User = Depends(get_current_active_user) +): + """Kill a scan task.""" + return scan_tasks.kill_scan_task(scan_task_id, current_user) + + +@api_router.get( + "/scan-tasks/{scan_task_id}/logs", + dependencies=[Depends(get_current_active_user)], + # response_model=scanTaskSchema.GenericResponse, + tags=["Scan Tasks"], +) +async def get_scan_task_logs( + scan_task_id: UUID4, current_user: User = Depends(get_current_active_user) +): + """Get logs from a particular scan task.""" + return scan_tasks.get_scan_task_logs(scan_task_id, current_user) \ No newline at end of file From 6993405edc4d36d00f51ca5ccca01b54ad7ea7b6 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 1 Oct 2024 16:17:09 -0400 Subject: [PATCH 038/314] Run pre-commit --- .../xfd_api/api_methods/scan_tasks.py | 148 ++++++++++-------- .../xfd_api/schema_models/scan_tasks.py | 5 +- backend/src/xfd_django/xfd_api/views.py | 8 +- 3 files changed, 92 insertions(+), 69 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py b/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py index 839b97cb..a224fe7b 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py +++ b/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py @@ -6,13 +6,13 @@ # Third-Party Libraries from fastapi import HTTPException, Response, status -from ..auth import is_global_view_admin, is_global_write_admin, get_tag_organizations +from ..auth import get_tag_organizations, is_global_view_admin, is_global_write_admin from ..models import ScanTask from ..schema_models.scan_tasks import ScanTaskSearch from ..tasks.ecs_client import ECSClient +PAGE_SIZE = 15 -PAGE_SIZE = 15; def list_scan_tasks(search_data: ScanTaskSearch, current_user): """List scans tasks based on search filter.""" @@ -21,16 +21,24 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): if not is_global_view_admin(current_user): raise HTTPException( status_code=403, detail="Unauthorized access. View logs for details." - ) + ) # Validate and parse the request body pageSize = search_data.pageSize or PAGE_SIZE # Determine the correct ordering based on the 'order' field - ordering_field = f"-{search_data.sort}" if search_data.order.upper() == "DESC" else search_data.sort - + ordering_field = ( + f"-{search_data.sort}" + if search_data.order.upper() == "DESC" + else search_data.sort + ) + # Construct query based on filters - qs = ScanTask.objects.select_related("scanId").prefetch_related("organizations").order_by(ordering_field) + qs = ( + ScanTask.objects.select_related("scanId") + .prefetch_related("organizations") + .order_by(ordering_field) + ) # Apply filters to queryset safely filters = search_data.filters @@ -50,7 +58,7 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): # Paginate results if pageSize != -1: - qs = qs[(search_data.page - 1) * pageSize: search_data.page * pageSize] + qs = qs[(search_data.page - 1) * pageSize : search_data.page * pageSize] # Convert queryset into a serialized response results = [] @@ -62,54 +70,66 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): else: scan_data = { "id": str(task.scanId.id), - "createdAt": task.scanId.createdAt.isoformat() + 'Z', - "updatedAt": task.scanId.updatedAt.isoformat() + 'Z', + "createdAt": task.scanId.createdAt.isoformat() + "Z", + "updatedAt": task.scanId.updatedAt.isoformat() + "Z", "name": task.scanId.name, "arguments": task.scanId.arguments, "frequency": task.scanId.frequency, - "lastRun": task.scanId.lastRun.isoformat() + 'Z' if task.scanId.lastRun else None, + "lastRun": task.scanId.lastRun.isoformat() + "Z" + if task.scanId.lastRun + else None, "isGranular": task.scanId.isGranular, "isUserModifiable": task.scanId.isUserModifiable, "isSingleScan": task.scanId.isSingleScan, - "manualRunPending": task.scanId.manualRunPending + "manualRunPending": task.scanId.manualRunPending, } - results.append({ - "id": str(task.id), - "createdAt": task.createdAt.isoformat() + 'Z', - "updatedAt": task.updatedAt.isoformat() + 'Z', - "status": task.status, - "type": task.type, - "fargateTaskArn": task.fargateTaskArn, - "input": task.input, - "output": task.output, - "requestedAt": task.requestedAt.isoformat() + 'Z' if task.requestedAt else None, - "startedAt": task.startedAt.isoformat() + 'Z' if task.startedAt else None, - "finishedAt": task.finishedAt.isoformat() + 'Z' if task.finishedAt else None, - "queuedAt": task.queuedAt.isoformat() + 'Z' if task.queuedAt else None, - "scan": scan_data, - "organizations": [ - { - "id": str(org.id), - "createdAt": org.createdAt.isoformat() + 'Z', - "updatedAt": org.updatedAt.isoformat() + 'Z', - "acronym": org.acronym, - "name": org.name, - "rootDomains": org.rootDomains, - "ipBlocks": org.ipBlocks, - "isPassive": org.isPassive, - "pendingDomains": org.pendingDomains, - "country": org.country, - "state": org.state, - "regionId": org.regionId, - "stateFips": org.stateFips, - "stateName": org.stateName, - "county": org.county, - "countyFips": org.countyFips, - "type": org.type - } - for org in task.organizations.all() - ] - }) + results.append( + { + "id": str(task.id), + "createdAt": task.createdAt.isoformat() + "Z", + "updatedAt": task.updatedAt.isoformat() + "Z", + "status": task.status, + "type": task.type, + "fargateTaskArn": task.fargateTaskArn, + "input": task.input, + "output": task.output, + "requestedAt": task.requestedAt.isoformat() + "Z" + if task.requestedAt + else None, + "startedAt": task.startedAt.isoformat() + "Z" + if task.startedAt + else None, + "finishedAt": task.finishedAt.isoformat() + "Z" + if task.finishedAt + else None, + "queuedAt": task.queuedAt.isoformat() + "Z" + if task.queuedAt + else None, + "scan": scan_data, + "organizations": [ + { + "id": str(org.id), + "createdAt": org.createdAt.isoformat() + "Z", + "updatedAt": org.updatedAt.isoformat() + "Z", + "acronym": org.acronym, + "name": org.name, + "rootDomains": org.rootDomains, + "ipBlocks": org.ipBlocks, + "isPassive": org.isPassive, + "pendingDomains": org.pendingDomains, + "country": org.country, + "state": org.state, + "regionId": org.regionId, + "stateFips": org.stateFips, + "stateName": org.stateName, + "county": org.county, + "countyFips": org.countyFips, + "type": org.type, + } + for org in task.organizations.all() + ], + } + ) count = qs.count() response = {"result": results, "count": count} @@ -119,7 +139,6 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) - def kill_scan_task(scan_task_id, current_user): @@ -129,31 +148,32 @@ def kill_scan_task(scan_task_id, current_user): if not is_global_write_admin(current_user): raise HTTPException( status_code=403, detail="Unauthorized access. View logs for details." - ) + ) # Check if scan_task_id is valid try: scan_task = ScanTask.objects.get(id=scan_task_id) except ScanTask.DoesNotExist: raise HTTPException(status_code=404, detail="ScanTask not found.") - + # Check if scan task is already finished or failed - if scan_task.status in ['failed', 'finished']: - raise HTTPException(status_code=400, detail="ScanTask has already finished.") + if scan_task.status in ["failed", "finished"]: + raise HTTPException( + status_code=400, detail="ScanTask has already finished." + ) # Update scan task status to 'failed' utc_now = datetime.now(timezone.utc) - scan_task.status = 'failed' + scan_task.status = "failed" scan_task.finishedAt = utc_now scan_task.output = f"Manually stopped at {utc_now.isoformat()}" scan_task.save() return {"statusCode": 200, "message": "ScanTask successfully marked as failed."} - - + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) - + def get_scan_task_logs(scan_task_id, current_user): """Get scan task logs.""" @@ -162,26 +182,26 @@ def get_scan_task_logs(scan_task_id, current_user): if not is_global_view_admin(current_user): raise HTTPException( status_code=403, detail="Unauthorized access. View logs for details." - ) + ) # Check if scan_task_id is valid try: scan_task = ScanTask.objects.get(id=scan_task_id) except ScanTask.DoesNotExist: raise HTTPException(status_code=404, detail="ScanTask not found.") - + # Ensure fargateTaskArn exists if not scan_task.fargateTaskArn: - raise HTTPException(status_code=404, detail="No logs available for this ScanTask.") + raise HTTPException( + status_code=404, detail="No logs available for this ScanTask." + ) # Retrieve logs from the ECSClient ecs_client = ECSClient() logs = ecs_client.get_logs(scan_task.fargateTaskArn) - return Response(content=logs or '', status_code=status.HTTP_200_OK) - - + return Response(content=logs or "", status_code=status.HTTP_200_OK) + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) - \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py b/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py index 7518218c..3f4cfd16 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py +++ b/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py @@ -21,6 +21,7 @@ class ScanTaskSearch(BaseModel): order: str filters: Any + class ScanTaskList(BaseModel): """Single scan-task schema.""" @@ -39,14 +40,16 @@ class ScanTaskList(BaseModel): scan: Optional[Scan] organization: Optional[List[Organization]] = [] + class ScanTaskListResponse(BaseModel): """Scan-task list schema.""" result: List[ScanTaskList] count: int + class GenericResponse(BaseModel): """Generic scan task response.""" statusCode: int - message: str \ No newline at end of file + message: str diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index cc7f9fcb..891249e7 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -24,8 +24,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import UUID4 -from .api_methods import scan -from .api_methods import scan_tasks +from .api_methods import scan, scan_tasks from .api_methods.api_keys import get_api_keys from .api_methods.cpe import get_cpes_by_id from .api_methods.cve import get_cves_by_id, get_cves_by_name @@ -430,7 +429,8 @@ async def invoke_scheduler(current_user: User = Depends(get_current_active_user) tags=["Scan Tasks"], ) async def list_scan_tasks( - search_data: scanTaskSchema.ScanTaskSearch, current_user: User = Depends(get_current_active_user) + search_data: scanTaskSchema.ScanTaskSearch, + current_user: User = Depends(get_current_active_user), ): """List scan tasks based on filters.""" return scan_tasks.list_scan_tasks(search_data, current_user) @@ -458,4 +458,4 @@ async def get_scan_task_logs( scan_task_id: UUID4, current_user: User = Depends(get_current_active_user) ): """Get logs from a particular scan task.""" - return scan_tasks.get_scan_task_logs(scan_task_id, current_user) \ No newline at end of file + return scan_tasks.get_scan_task_logs(scan_task_id, current_user) From 7dc048da95a195639977088013a3f2f798cc3797 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Tue, 1 Oct 2024 16:17:26 -0400 Subject: [PATCH 039/314] run pre-commit again --- backend/src/xfd_django/xfd_api/auth.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index dd864e83..f8ca13d3 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -74,6 +74,7 @@ def is_regional_admin(current_user) -> bool: """Check if the user has regional admin permissions.""" return current_user and current_user.userType in ["regionalAdmin", "globalAdmin"] + def get_tag_organizations(current_user, tag_id: str) -> list[str]: """Returns the organizations belonging to a tag, if the user can access the tag.""" # Check if the user is a global view admin @@ -82,7 +83,11 @@ def get_tag_organizations(current_user, tag_id: str) -> list[str]: # Fetch the OrganizationTag and its related organizations try: - tag = OrganizationTag.objects.prefetch_related("organizations").filter(id=tag_id).first() + tag = ( + OrganizationTag.objects.prefetch_related("organizations") + .filter(id=tag_id) + .first() + ) except Exception: return [] @@ -94,8 +99,6 @@ def get_tag_organizations(current_user, tag_id: str) -> list[str]: return [] - - # TODO: Below is a template of what these could be nut isn't tested # RECREATE ALL THE FUNCTIONS IN AUTH.TS From 09cac1db077303ff43849c17d28fb8269e47f528 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 2 Oct 2024 09:19:48 -0400 Subject: [PATCH 040/314] fix auth.py --- backend/src/xfd_django/xfd_api/auth.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 45470581..f8ca13d3 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -74,23 +74,6 @@ def is_regional_admin(current_user) -> bool: """Check if the user has regional admin permissions.""" return current_user and current_user.userType in ["regionalAdmin", "globalAdmin"] -def get_tag_organizations(current_user, tag_id: str) -> list[str]: - """Returns the organizations belonging to a tag, if the user can access the tag.""" - # Check if the user is a global view admin - if not is_global_view_admin(current_user): - return [] - - # Fetch the OrganizationTag and its related organizations - tag = OrganizationTag.objects.prefetch_related("organizations").filter(id=tag_id).first() - if tag: - # Return a list of organization IDs - return [org.id for org in tag.organizations.all()] - - # Return an empty list if tag is not found - return [] - - - def get_tag_organizations(current_user, tag_id: str) -> list[str]: """Returns the organizations belonging to a tag, if the user can access the tag.""" From 61581cb50a8c3e8a1660b4994b43cbac03e6c9aa Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Wed, 2 Oct 2024 08:23:37 -0500 Subject: [PATCH 041/314] Refactor AssessmentSchema import. --- backend/src/xfd_django/xfd_api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 8b3e7f11..0b3c3ba5 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -34,7 +34,7 @@ from .auth import get_current_active_user from .models import Assessment, User from .schema_models import scan as scanSchema -from .schema_models.assessment import Assessment +from .schema_models.assessment import Assessment as AssessmentSchema from .schema_models.cpe import Cpe as CpeSchema from .schema_models.cve import Cve as CveSchema from .schema_models.domain import Domain as DomainSchema From bebac7446b9a304b6d3f88a53712465607184034 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 2 Oct 2024 09:27:52 -0400 Subject: [PATCH 042/314] Resolve conflict --- backend/src/xfd_django/xfd_api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 891249e7..58cc6bd7 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -36,7 +36,7 @@ from .models import Assessment, User from .schema_models import scan as scanSchema from .schema_models import scan_tasks as scanTaskSchema -from .schema_models.assessment import Assessment +from .schema_models.assessment import Assessment as AssessmentSchema from .schema_models.cpe import Cpe as CpeSchema from .schema_models.cve import Cve as CveSchema from .schema_models.domain import Domain as DomainSchema From 6f508a8ae029b707f5b150f6506c675963e48fc2 Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Wed, 2 Oct 2024 11:36:56 -0400 Subject: [PATCH 043/314] Initial commit: -Created saved_search schema -Created saved_search API --- .../xfd_api/api_methods/saved_search.py | 108 ++++++++++++++++++ .../xfd_api/schema_models/saved_search.py | 50 ++++++++ backend/src/xfd_django/xfd_api/views.py | 34 ++++++ 3 files changed, 192 insertions(+) create mode 100644 backend/src/xfd_django/xfd_api/api_methods/saved_search.py create mode 100644 backend/src/xfd_django/xfd_api/schema_models/saved_search.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py new file mode 100644 index 00000000..e13c1af7 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -0,0 +1,108 @@ +"""Saved Search API""" + + +# Standard Python Libraries +import json +import uuid + +# Third-Party Libraries +from django.http import JsonResponse +from fastapi import HTTPException + +from ..models import SavedSearch + +PAGE_SIZE = 20 + + +def create_saved_search(request): + data = json.loads(request.body) + search = SavedSearch.objects.create( + name=data["name"], + count=data["count"], + sort_direction=data["sortDirection"], + sort_field=data["sortField"], + search_term=data["searchTerm"], + search_path=data["searchPath"], + filters=data["filters"], + create_vulnerabilities=data["createVulnerabilities"], + vulnerability_template=data.get("vulnerabilityTemplate"), + created_by=request.user, + ) + return JsonResponse({"status": "Created", "search": search.id}, status=201) + + +def list_saved_searches(request): + """List all saved searches.""" + page_size = int(request.GET.get("pageSize", PAGE_SIZE)) + page = int(request.GET.get("page", 1)) + searches = SavedSearch.objects.filter(created_by=request.user) + total_count = searches.count() + searches = searches[(page - 1) * page_size : page * page_size] + data = list(searches.values()) + return JsonResponse({"result": data, "count": total_count}, safe=False) + + +def get_saved_search(request, search_id): + if not uuid.UUID(search_id): + raise HTTPException({"error": "Invalid UUID"}, status=404) + + try: + search = SavedSearch.objects.get(id=search_id, created_by=request.user) + data = { + "id": str(search.id), + "name": search.name, + "count": search.count, + "sort_direction": search.sort_direction, + "sort_field": search.sort_field, + "search_term": search.search_term, + "search_path": search.search_path, + "filters": search.filters, + "create_vulnerabilities": search.create_vulnerabilities, + "vulnerability_template": search.vulnerability_template, + "created_by": search.created_by.id, + } + return JsonResponse(data) + except SavedSearch.DoesNotExist as e: + raise HTTPException(status_code=404, detail=str(e)) + + +def update_saved_search(request, search_id): + if not uuid.UUID(search_id): + raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) + + try: + search = SavedSearch.objects.get(id=search_id, created_by=request.user) + except SavedSearch.DoesNotExist as e: + return HTTPException(status_code=404, detail=str(e)) + + data = json.loads(request.body) + search.name = data.get("name", search.name) + search.count = data.get("count", search.count) + search.sort_direction = data.get("sortDirection", search.sort_direction) + search.sort_field = data.get("sortField", search.sort_field) + search.search_term = data.get("searchTerm", search.search_term) + search.search_path = data.get("searchPath", search.search_path) + search.filters = data.get("filters", search.filters) + search.create_vulnerabilities = data.get( + "createVulnerabilities", search.create_vulnerabilities + ) + search.vulnerability_template = data.get( + "vulnerabilityTemplate", search.vulnerability_template + ) + search.save() + return JsonResponse({"status": "Updated", "search": search.id}, status=200) + + +def delete_saved_search(request, search_id): + """Delete saved search by id.""" + if not uuid.UUID(search_id): + raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) + + try: + search = SavedSearch.objects.get(id=search_id, created_by=request.user) + search.delete() + return JsonResponse( + {"status": "success", "message": f"Saved search id:{search_id} deleted."} + ) + except SavedSearch.DoesNotExist as e: + raise HTTPException(status_code=404, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py new file mode 100644 index 00000000..82ce7f0d --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py @@ -0,0 +1,50 @@ +"""Saved Search schemas.""" +# Standard Python Libraries +from datetime import datetime +from typing import Any, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel, Json + + +class SavedSearch(BaseModel): + """SavedSearch schema.""" + + id: UUID + name: str + count: int + sort_direction: str + sort_field: str + search_term: str + search_path: str + filters: Json[Any] + create_vulnerabilities: bool + vulnerability_template: Optional[Json[Any]] + created_by: UUID + created_at: datetime + updated_at: datetime + + +class SavedSearchFilters(BaseModel): + """SavedSearchFilters schema.""" + + id: Optional[UUID] + name: Optional[str] + sort_direction: Optional[str] + sort_field: Optional[str] + search_term: Optional[str] + search_path: Optional[str] + create_vulnerabilities: Optional[bool] + created_by: Optional[UUID] + + +class SavedSearchSearch(BaseModel): + """SavedSearchSearch schema.""" + + page: int + sort: Optional[str] + order: str + filters: Optional[SavedSearchFilters] + pageSize: Optional[int] + groupBy: Optional[str] diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 8b3e7f11..732d3c6f 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -316,6 +316,40 @@ async def call_get_organizations( return get_organizations(state, regionId) +# TODO: Typescript endpoints for reference, not implemented in FastAPI. +# Remove after implementation +# authenticatedRoute.get('/saved-searches', handlerToExpress(savedSearches.list)); +# authenticatedRoute.post( +# '/saved-searches', +# handlerToExpress(savedSearches.create) +# ); +# authenticatedRoute.get( +# '/saved-searches/:searchId', +# handlerToExpress(savedSearches.get) +# ); +# authenticatedRoute.put( +# '/saved-searches/:searchId', +# handlerToExpress(savedSearches.update) +# ); +# authenticatedRoute.delete( +# '/saved-searches/:searchId', +# handlerToExpress(savedSearches.del) +# ); + +# ======================================== +# Saved Search Endpoints +# ======================================== + + +# @api_router.get( +# "/saved-searches", +# dependencies=[Depends(get_current_active_user)], +# response_model=savedSearchSchema.GetSavedSearchesResponseModel, +# tags=["Saved Searches"], + +# ) + + # ======================================== # Scan Endpoints # ======================================== From a33cd8d4d5c32e84eaaec41d5f6ada1627fb4c43 Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Thu, 3 Oct 2024 16:28:18 -0400 Subject: [PATCH 044/314] Created skeleton for saved-searches endpoint --- .../xfd_api/api_methods/saved_search.py | 171 +++++++++--------- .../xfd_api/schema_models/saved_search.py | 2 +- backend/src/xfd_django/xfd_api/views.py | 58 +++++- 3 files changed, 139 insertions(+), 92 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py index e13c1af7..739865be 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -14,95 +14,98 @@ PAGE_SIZE = 20 -def create_saved_search(request): - data = json.loads(request.body) - search = SavedSearch.objects.create( - name=data["name"], - count=data["count"], - sort_direction=data["sortDirection"], - sort_field=data["sortField"], - search_term=data["searchTerm"], - search_path=data["searchPath"], - filters=data["filters"], - create_vulnerabilities=data["createVulnerabilities"], - vulnerability_template=data.get("vulnerabilityTemplate"), - created_by=request.user, - ) - return JsonResponse({"status": "Created", "search": search.id}, status=201) +# def create_saved_search(request): +# data = json.loads(request.body) +# search = SavedSearch.objects.create( +# name=data["name"], +# count=data["count"], +# sort_direction=data["sortDirection"], +# sort_field=data["sortField"], +# search_term=data["searchTerm"], +# search_path=data["searchPath"], +# filters=data["filters"], +# create_vulnerabilities=data["createVulnerabilities"], +# vulnerability_template=data.get("vulnerabilityTemplate"), +# created_by=request.user, +# ) +# return JsonResponse({"status": "Created", "search": search.id}, status=201) def list_saved_searches(request): """List all saved searches.""" - page_size = int(request.GET.get("pageSize", PAGE_SIZE)) - page = int(request.GET.get("page", 1)) - searches = SavedSearch.objects.filter(created_by=request.user) - total_count = searches.count() - searches = searches[(page - 1) * page_size : page * page_size] - data = list(searches.values()) - return JsonResponse({"result": data, "count": total_count}, safe=False) - - -def get_saved_search(request, search_id): - if not uuid.UUID(search_id): - raise HTTPException({"error": "Invalid UUID"}, status=404) - try: - search = SavedSearch.objects.get(id=search_id, created_by=request.user) - data = { - "id": str(search.id), - "name": search.name, - "count": search.count, - "sort_direction": search.sort_direction, - "sort_field": search.sort_field, - "search_term": search.search_term, - "search_path": search.search_path, - "filters": search.filters, - "create_vulnerabilities": search.create_vulnerabilities, - "vulnerability_template": search.vulnerability_template, - "created_by": search.created_by.id, - } - return JsonResponse(data) - except SavedSearch.DoesNotExist as e: + page_size = int(request.GET.get("pageSize", PAGE_SIZE)) + page = int(request.GET.get("page", 1)) + searches = SavedSearch.objects.filter(created_by=request.user) + total_count = searches.count() + searches = searches[(page - 1) * page_size : page * page_size] + data = list(searches.values()) + return JsonResponse({"result": data, "count": total_count}, safe=False) + except Exception as e: raise HTTPException(status_code=404, detail=str(e)) -def update_saved_search(request, search_id): - if not uuid.UUID(search_id): - raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) - - try: - search = SavedSearch.objects.get(id=search_id, created_by=request.user) - except SavedSearch.DoesNotExist as e: - return HTTPException(status_code=404, detail=str(e)) - - data = json.loads(request.body) - search.name = data.get("name", search.name) - search.count = data.get("count", search.count) - search.sort_direction = data.get("sortDirection", search.sort_direction) - search.sort_field = data.get("sortField", search.sort_field) - search.search_term = data.get("searchTerm", search.search_term) - search.search_path = data.get("searchPath", search.search_path) - search.filters = data.get("filters", search.filters) - search.create_vulnerabilities = data.get( - "createVulnerabilities", search.create_vulnerabilities - ) - search.vulnerability_template = data.get( - "vulnerabilityTemplate", search.vulnerability_template - ) - search.save() - return JsonResponse({"status": "Updated", "search": search.id}, status=200) - - -def delete_saved_search(request, search_id): - """Delete saved search by id.""" - if not uuid.UUID(search_id): - raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) - - try: - search = SavedSearch.objects.get(id=search_id, created_by=request.user) - search.delete() - return JsonResponse( - {"status": "success", "message": f"Saved search id:{search_id} deleted."} - ) - except SavedSearch.DoesNotExist as e: - raise HTTPException(status_code=404, detail=str(e)) +# def get_saved_search(request, search_id): +# if not uuid.UUID(search_id): +# raise HTTPException({"error": "Invalid UUID"}, status=404) + +# try: +# search = SavedSearch.objects.get(id=search_id, created_by=request.user) +# data = { +# "id": str(search.id), +# "name": search.name, +# "count": search.count, +# "sort_direction": search.sort_direction, +# "sort_field": search.sort_field, +# "search_term": search.search_term, +# "search_path": search.search_path, +# "filters": search.filters, +# "create_vulnerabilities": search.create_vulnerabilities, +# "vulnerability_template": search.vulnerability_template, +# "created_by": search.created_by.id, +# } +# return JsonResponse(data) +# except SavedSearch.DoesNotExist as e: +# raise HTTPException(status_code=404, detail=str(e)) + + +# def update_saved_search(request, search_id): +# if not uuid.UUID(search_id): +# raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) + +# try: +# search = SavedSearch.objects.get(id=search_id, created_by=request.user) +# except SavedSearch.DoesNotExist as e: +# return HTTPException(status_code=404, detail=str(e)) + +# data = json.loads(request.body) +# search.name = data.get("name", search.name) +# search.count = data.get("count", search.count) +# search.sort_direction = data.get("sortDirection", search.sort_direction) +# search.sort_field = data.get("sortField", search.sort_field) +# search.search_term = data.get("searchTerm", search.search_term) +# search.search_path = data.get("searchPath", search.search_path) +# search.filters = data.get("filters", search.filters) +# search.create_vulnerabilities = data.get( +# "createVulnerabilities", search.create_vulnerabilities +# ) +# search.vulnerability_template = data.get( +# "vulnerabilityTemplate", search.vulnerability_template +# ) +# search.save() +# return JsonResponse({"status": "Updated", "search": search.id}, status=200) + + +# def delete_saved_search(request, search_id): +# """Delete saved search by id.""" +# if not uuid.UUID(search_id): +# raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) + +# try: +# search = SavedSearch.objects.get(id=search_id, created_by=request.user) +# search.delete() +# return JsonResponse( +# {"status": "success", "message": f"Saved search id:{search_id} deleted."} +# ) +# except SavedSearch.DoesNotExist as e: +# raise HTTPException(status_code=404, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py index 82ce7f0d..95cd5e6e 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py +++ b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py @@ -19,7 +19,7 @@ class SavedSearch(BaseModel): search_term: str search_path: str filters: Json[Any] - create_vulnerabilities: bool + create_vulnerabilities: Optional[bool] vulnerability_template: Optional[Json[Any]] created_by: UUID created_at: datetime diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 732d3c6f..c0adf869 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -29,10 +29,11 @@ from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import get_domain_by_id from .api_methods.organization import get_organizations, read_orgs +from .api_methods.saved_search import list_saved_searches from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user -from .models import Assessment, User +from .models import Assessment, SavedSearch, User from .schema_models import scan as scanSchema from .schema_models.assessment import Assessment from .schema_models.cpe import Cpe as CpeSchema @@ -341,13 +342,56 @@ async def call_get_organizations( # ======================================== -# @api_router.get( -# "/saved-searches", -# dependencies=[Depends(get_current_active_user)], -# response_model=savedSearchSchema.GetSavedSearchesResponseModel, -# tags=["Saved Searches"], +@api_router.post( + "/saved-searches", + tags=["Testing"], +) +async def create_saved_search(): + """Create a new saved search.""" + return {"status": "ok"} + + +@api_router.get( + "/saved-searches", + # dependencies=[Depends(get_current_active_user)], + # response_model=savedSearchSchema.GetSavedSearchesResponseModel, + tags=["Testing"], +) +async def call_list_saved_searches(): + """Retrieve a list of all saved searches.""" + return {"status": "ok"} + + +@api_router.get( + "/saved-searches/{saved_search_id}", + tags=["Testing"], +) +async def get_saved_search(saved_search_id: str): + """Retrieve a saved search by its ID.""" + return {"status": "ok"} + + +@api_router.put( + "/saved-searches/{saved_search_id}", + tags=["Testing"], +) +async def update_saved_search(saved_search_id: str): + """Update a saved search by its ID.""" + return {"status": "ok"} + + +@api_router.delete( + "/saved-searches/{saved_search_id}", + tags=["Testing"], +) +async def delete_saved_search(saved_search_id: str): + """Delete a saved search by its ID.""" + return {"status": "ok"} + -# ) +# async def call_list_saved_searches(current_user: User = Depends(get_current_active_user)): +# """Retrieve a list of all saved searches.""" +# return list_saved_searches(current_user) # ======================================== From 05e8bd23bb6bce1fc5e97dbf4b724cf9e2178d08 Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Fri, 4 Oct 2024 11:14:55 -0400 Subject: [PATCH 045/314] Updated List all saved searches endpoint --- backend/src/xfd_django/xfd_api/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index c0adf869..679b14c5 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -18,7 +18,7 @@ """ # Standard Python Libraries -from typing import List, Optional +from typing import Dict, List, Optional # Third-Party Libraries from fastapi import APIRouter, Depends, HTTPException, Query @@ -358,8 +358,10 @@ async def create_saved_search(): tags=["Testing"], ) async def call_list_saved_searches(): + saved_searches = SavedSearch.objects.all() + """Retrieve a list of all saved searches.""" - return {"status": "ok"} + return list(saved_searches) @api_router.get( From 6fa768f21340b6a4943864ebbe48e5fed15fbcb8 Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Mon, 7 Oct 2024 10:21:07 -0400 Subject: [PATCH 046/314] Updated GET request for a single saved search entry in views.py --- .../xfd_api/api_methods/saved_search.py | 61 ++++++++++--------- .../xfd_api/schema_models/saved_search.py | 3 - backend/src/xfd_django/xfd_api/views.py | 15 ++--- 3 files changed, 40 insertions(+), 39 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py index 739865be..3f3acf8e 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -31,42 +31,45 @@ # return JsonResponse({"status": "Created", "search": search.id}, status=201) -def list_saved_searches(request): +def list_saved_searches(): """List all saved searches.""" try: - page_size = int(request.GET.get("pageSize", PAGE_SIZE)) - page = int(request.GET.get("page", 1)) - searches = SavedSearch.objects.filter(created_by=request.user) - total_count = searches.count() - searches = searches[(page - 1) * page_size : page * page_size] - data = list(searches.values()) - return JsonResponse({"result": data, "count": total_count}, safe=False) + saved_searches = SavedSearch.objects.all() + count = saved_searches.count() + for search in saved_searches: + print(str(search.id)) + return {"saved_searches": list(saved_searches), "Saved Search Count": count} except Exception as e: raise HTTPException(status_code=404, detail=str(e)) -# def get_saved_search(request, search_id): -# if not uuid.UUID(search_id): -# raise HTTPException({"error": "Invalid UUID"}, status=404) +def get_saved_search(search_id): + if not uuid.UUID(search_id): + raise HTTPException({"error": "Invalid UUID"}, status=404) -# try: -# search = SavedSearch.objects.get(id=search_id, created_by=request.user) -# data = { -# "id": str(search.id), -# "name": search.name, -# "count": search.count, -# "sort_direction": search.sort_direction, -# "sort_field": search.sort_field, -# "search_term": search.search_term, -# "search_path": search.search_path, -# "filters": search.filters, -# "create_vulnerabilities": search.create_vulnerabilities, -# "vulnerability_template": search.vulnerability_template, -# "created_by": search.created_by.id, -# } -# return JsonResponse(data) -# except SavedSearch.DoesNotExist as e: -# raise HTTPException(status_code=404, detail=str(e)) + try: + saved_search = SavedSearch.objects.all() + for search in saved_search: + if str(search.id) == search_id: + return search + + # search = SavedSearch.objects.get(id=search_id, created_by=request.user) + # data = { + # "id": str(search.id), + # "name": search.name, + # "count": search.count, + # "sort_direction": search.sort_direction, + # "sort_field": search.sort_field, + # "search_term": search.search_term, + # "search_path": search.search_path, + # "filters": search.filters, + # "create_vulnerabilities": search.create_vulnerabilities, + # "vulnerability_template": search.vulnerability_template, + # "created_by": search.created_by.id, + # } + # return JsonResponse(data) + except SavedSearch.DoesNotExist as e: + raise HTTPException(status_code=404, detail=str(e)) # def update_saved_search(request, search_id): diff --git a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py index 95cd5e6e..50357bdb 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py +++ b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py @@ -19,9 +19,6 @@ class SavedSearch(BaseModel): search_term: str search_path: str filters: Json[Any] - create_vulnerabilities: Optional[bool] - vulnerability_template: Optional[Json[Any]] - created_by: UUID created_at: datetime updated_at: datetime diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 679b14c5..0808e7db 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -29,7 +29,7 @@ from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import get_domain_by_id from .api_methods.organization import get_organizations, read_orgs -from .api_methods.saved_search import list_saved_searches +from .api_methods.saved_search import get_saved_search, list_saved_searches from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user @@ -41,6 +41,7 @@ from .schema_models.domain import Domain as DomainSchema from .schema_models.domain import DomainSearch from .schema_models.organization import Organization as OrganizationSchema +from .schema_models.saved_search import SavedSearch as savedSearchSchema from .schema_models.user import User as UserSchema from .schema_models.vulnerability import Vulnerability as VulnerabilitySchema @@ -351,26 +352,26 @@ async def create_saved_search(): return {"status": "ok"} +# Get all existing saved searches is implemented in the following function @api_router.get( "/saved-searches", # dependencies=[Depends(get_current_active_user)], - # response_model=savedSearchSchema.GetSavedSearchesResponseModel, + # response_model=savedSearchSchema, tags=["Testing"], ) async def call_list_saved_searches(): - saved_searches = SavedSearch.objects.all() - """Retrieve a list of all saved searches.""" - return list(saved_searches) + return list_saved_searches() +# Get individual saved search is implemented in the following function @api_router.get( "/saved-searches/{saved_search_id}", tags=["Testing"], ) -async def get_saved_search(saved_search_id: str): +async def call_get_saved_search(saved_search_id: str): """Retrieve a saved search by its ID.""" - return {"status": "ok"} + return get_saved_search(saved_search_id) @api_router.put( From 2a5192434ec3cfac6128b106a9d3336b88052a82 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Mon, 7 Oct 2024 14:02:13 -0500 Subject: [PATCH 047/314] Add s3-client. Testing new domain endpoints. --- backend/requirements.txt | 1 + .../xfd_django/xfd_api/api_methods/domain.py | 97 ++++++++++++++- .../xfd_django/xfd_api/helpers/s3_client.py | 117 ++++++++++++++++++ .../xfd_api/schema_models/domain.py | 7 +- backend/src/xfd_django/xfd_api/views.py | 10 +- 5 files changed, 223 insertions(+), 9 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/helpers/s3_client.py diff --git a/backend/requirements.txt b/backend/requirements.txt index de640b24..cf7da47a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,6 +3,7 @@ django docker fastapi==0.111.0 mangum==0.17.0 +minio psycopg2-binary PyJWT uvicorn==0.30.1 diff --git a/backend/src/xfd_django/xfd_api/api_methods/domain.py b/backend/src/xfd_django/xfd_api/api_methods/domain.py index 5d78ee90..b8b78337 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/domain.py +++ b/backend/src/xfd_django/xfd_api/api_methods/domain.py @@ -3,13 +3,67 @@ """ +# Standard Python Libraries +import csv + # Third-Party Libraries +from django.core.paginator import Paginator from fastapi import HTTPException -from ..models import Domain +from ..helpers.s3_client import export_to_csv +from ..models import Domain, Organization, Service, Vulnerability from ..schema_models.domain import DomainFilters, DomainSearch +def filter_domains(domains, domain_filters: DomainFilters): + """ + Filter domains + Arguments: + domains: A list of all domains, sorted + domain_filters: Value to filter the domains table by + Returns: + object: a list of Domain objects + """ + try: + if domain_filters.port is not None: + services_by_port = Service.objects.values("domainId").filter( + port=domain_filters.port + ) + domains = domains.filter(id__in=services_by_port) + if domain_filters.service != "": + service_by_id = Service.objects.values("domainId").get( + id=domain_filters.service + ) + domains = domains.filter(id=service_by_id["domainId"]) + if domain_filters.reverseName != "": + domains_by_reverse_name = Domain.objects.values("id").filter( + reverseName=domain_filters.reverseName + ) + domains = domains.filter(id__in=domains_by_reverse_name) + if domain_filters.ip != "": + domains_by_ip = Domain.objects.values("id").filter(ip=domain_filters.ip) + domains = domains.filter(id__in=domains_by_ip) + if domain_filters.organization != "": + domains_by_org = Domain.objects.values("id").filter( + organizationId_id=domain_filters.organization + ) + domains = domains.filter(id__in=domains_by_org) + if domain_filters.organizationName != "": + organization_by_name = Organization.objects.values("id").filter( + name=domain_filters.organizationName + ) + domains = domains.filter(organizationId_id__in=organization_by_name) + if domain_filters.vulnerabilities != "": + vulnerabilities_by_id = Vulnerability.objects.values("domainId").filter( + id=domain_filters.vulnerabilities + ) + domains = domains.filter(id__in=vulnerabilities_by_id) + + return domains + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + def get_domain_by_id(domain_id: str): """ Get domain by id. @@ -25,22 +79,57 @@ def get_domain_by_id(domain_id: str): raise HTTPException(status_code=500, detail=str(e)) +def sort_direction(sort, order): + try: + # Fetch all domains in list + if sort == "ASC": + return sort + elif sort == "DSC": + return "-" + order + else: + raise ValueError + except ValueError as e: + raise HTTPException(status_code=500, detail="Invalid sort direction supplied") + + def search_domains(domain_search: DomainSearch): """ List domains by search filter Arguments: domain_search: A DomainSearch object to filter by. Returns: - object: a list of Domain objects + object: A paginated list of Domain objects """ try: - pass + # Fetch all domains in list + sort_direction(domain_search.sort, domain_search.order) + + domains = Domain.objects.all().order_by( + sort_direction(domain_search.sort, domain_search.order) + ) + if domain_search.filters is not None: + results = filter_domains(domains, domain_search.filters) + paginator = Paginator(results, domain_search.pageSize) + + return paginator.get_page(domain_search.page) + else: + raise ValueError("DomainFilters cannot be NoneType") except Exception as e: raise HTTPException(status_code=500, detail=str(e)) def export_domains(domain_search: DomainSearch): try: - pass + domains = Domain.objects.all().order_by( + sort_direction(domain_search.sort, domain_search.order) + ) + + if domain_search.filters is not None: + results = filter_domains(domains, domain_search.filters) + paginator = Paginator(results, domain_search.pageSize) + + return export_to_csv(paginator, domains, "testing", True) + else: + raise ValueError("DomainFilters cannot be NoneType") except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/helpers/s3_client.py b/backend/src/xfd_django/xfd_api/helpers/s3_client.py new file mode 100644 index 00000000..3bbac400 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/helpers/s3_client.py @@ -0,0 +1,117 @@ +# Standard Python Libraries +import csv +from datetime import datetime +import os +from random import randint + +# Third-Party Libraries +import boto3 +import boto3.s3 +from django.core.paginator import Paginator +from django.http import HttpResponse +from fastapi import HTTPException +from minio import Minio + + +def get_minio_client(): + return Minio( + "localhost:9000", + access_key=os.environ["MINIO_ACCESS_KEY"], + secret_key=os.environ["MINIO_SECRET_KEY"], + secure=False, + ) + + +async def save_minio(data, key: str): + client = get_minio_client() + try: + if not client.bucket_exists(os.environ["EXPORT_BUCKET_NAME"]): + client.make_bucket(os.environ["EXPORT_BUCKET_NAME"]) + + client.put_object(os.environ["EXPORT_BUCKET_NAME"], data, key) + print("File uploaded successfully") + except Exception as e: + print(f"Error uploading file to Minio: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +async def export_to_csv(pages: Paginator, queryset, name, is_local: bool = True): + try: + filename = f"{randint(0, 1000000000)}/{name}-{datetime.now()}.csv" + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f"attachment; filename={filename}" + + writer = csv.writer(response) + if queryset.count() > 0: + writer.writerow( + [ + "id", + "createdAt", + "updatedAt", + "syncedAt", + "ip", + "fromRootDomain", + "subdomainSource", + "ipOnly", + "reverseName", + "name", + "screenshot", + "country", + "asn", + "cloudHosted", + "ssl", + "censysCertificatesResults", + "trustymailResults", + "discoveredById_id", + "organizationId_id", + ] + ) + + for page_number in pages.page_range: + page = pages.page(page_number) + for obj in page.object_list: + print(obj) + writer.writerow( + [ + obj.id, + obj.createdAt, + obj.updatedAt, + obj.syncedAt, + obj.ip, + obj.fromRootDomain, + obj.subdomainSource, + obj.ipOnly, + obj.reverseName, + obj.name, + obj.screnshot, + obj.country, + obj.asn, + obj.cloudHosted, + obj.ssl, + obj.censysCertificatesResults, + obj.trustymailResults, + obj.discoveredById_id, + obj.organizationId_id, + ] + ) + # if not is_local: + # s3_client = boto3.client("s3") + # s3_client.put_object( + # Body=response.content, + # Bucket=os.environ['EXPORT_BUCKET_NAME'], + # Key=filename + # ) + # else: + # client = get_minio_client() + # csv_values = response.getvalue() + # if not client.bucket_exists(os.environ['EXPORT_BUCKET_NAME']): + # client.make_bucket(os.environ['EXPORT_BUCKET_NAME']) + # client.put_object( + # os.environ['EXPORT_BUCKET_NAME'], + # filename, + # len(csv_values), + # content_type='text/csv' + # ) + return response + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/domain.py b/backend/src/xfd_django/xfd_api/schema_models/domain.py index 3bf05405..d509de03 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/domain.py +++ b/backend/src/xfd_django/xfd_api/schema_models/domain.py @@ -3,12 +3,15 @@ # from pydantic.types import UUID1, UUID # Standard Python Libraries from datetime import datetime -from typing import Any, Optional +from typing import Any, List, Optional from uuid import UUID # Third-Party Libraries from pydantic import BaseModel +from ..schema_models.service import Service +from ..schema_models.vulnerability import Vulnerability + class Domain(BaseModel): """Domain schema.""" @@ -42,7 +45,7 @@ class Domain(BaseModel): class DomainFilters(BaseModel): """DomainFilters schema.""" - ports: Optional[str] = None + port: Optional[str] = None service: Optional[str] = None reverseName: Optional[str] = None ip: Optional[str] = None diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 4e546e72..ea2cea74 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -196,10 +196,14 @@ async def call_search_domains(domain_search: DomainSearch): raise HTTPException(status_code=500, detail=str(e)) -@api_router.post("/domain/export") -async def call_export_domains(): +@api_router.post( + "/domain/export", + # dependencies=[Depends(get_current_active_user)], + tags=["Domains"], +) +async def call_export_domains(domain_search: DomainSearch): try: - pass + return export_domains(domain_search) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) From 937a9090190d9b8c3a45e7403e2a32870eb93528 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 8 Oct 2024 09:20:07 -0500 Subject: [PATCH 048/314] Add docstring, minor bug fix to sort_direction(). Remove implementation of s3-client methods. --- backend/src/xfd_django/xfd_api/api_methods/domain.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/domain.py b/backend/src/xfd_django/xfd_api/api_methods/domain.py index b8b78337..34f60e46 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/domain.py +++ b/backend/src/xfd_django/xfd_api/api_methods/domain.py @@ -80,10 +80,16 @@ def get_domain_by_id(domain_id: str): def sort_direction(sort, order): + """ + Adds the sort direction modifier. + If sort = + ASC - return order field unmodified to sort in ascending order. + DSC - returns & prepend '-' to the order field to sort in descending order. + """ try: # Fetch all domains in list if sort == "ASC": - return sort + return order elif sort == "DSC": return "-" + order else: @@ -128,7 +134,8 @@ def export_domains(domain_search: DomainSearch): results = filter_domains(domains, domain_search.filters) paginator = Paginator(results, domain_search.pageSize) - return export_to_csv(paginator, domains, "testing", True) + return paginator.get_page(domain_search.page) + # return export_to_csv(paginator, domains, "testing", True) else: raise ValueError("DomainFilters cannot be NoneType") except Exception as e: From adccb64827d678a07dc2298b2297b652743e4808 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 8 Oct 2024 09:21:29 -0500 Subject: [PATCH 049/314] Add docstring for s3-client implementation line that is commented out. --- backend/src/xfd_django/xfd_api/api_methods/domain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/xfd_django/xfd_api/api_methods/domain.py b/backend/src/xfd_django/xfd_api/api_methods/domain.py index 34f60e46..b6029f64 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/domain.py +++ b/backend/src/xfd_django/xfd_api/api_methods/domain.py @@ -135,6 +135,7 @@ def export_domains(domain_search: DomainSearch): paginator = Paginator(results, domain_search.pageSize) return paginator.get_page(domain_search.page) + # TODO: Implement S3 client methods after collab with entire team. # return export_to_csv(paginator, domains, "testing", True) else: raise ValueError("DomainFilters cannot be NoneType") From 4a9532d77080aad9a7ee00a9f94075f369755368 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Tue, 8 Oct 2024 14:01:47 -0500 Subject: [PATCH 050/314] Moved sort_direction() to a helper file, removed duplicated call to sort_direction() in domains.py --- .../xfd_django/xfd_api/api_methods/domain.py | 22 +------------------ .../xfd_api/helpers/filter_helpers.py | 21 ++++++++++++++++++ 2 files changed, 22 insertions(+), 21 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/helpers/filter_helpers.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/domain.py b/backend/src/xfd_django/xfd_api/api_methods/domain.py index b6029f64..5bf3f4ba 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/domain.py +++ b/backend/src/xfd_django/xfd_api/api_methods/domain.py @@ -10,7 +10,7 @@ from django.core.paginator import Paginator from fastapi import HTTPException -from ..helpers.s3_client import export_to_csv +from ..helpers.filter_helpers import sort_direction from ..models import Domain, Organization, Service, Vulnerability from ..schema_models.domain import DomainFilters, DomainSearch @@ -79,25 +79,6 @@ def get_domain_by_id(domain_id: str): raise HTTPException(status_code=500, detail=str(e)) -def sort_direction(sort, order): - """ - Adds the sort direction modifier. - If sort = - ASC - return order field unmodified to sort in ascending order. - DSC - returns & prepend '-' to the order field to sort in descending order. - """ - try: - # Fetch all domains in list - if sort == "ASC": - return order - elif sort == "DSC": - return "-" + order - else: - raise ValueError - except ValueError as e: - raise HTTPException(status_code=500, detail="Invalid sort direction supplied") - - def search_domains(domain_search: DomainSearch): """ List domains by search filter @@ -108,7 +89,6 @@ def search_domains(domain_search: DomainSearch): """ try: # Fetch all domains in list - sort_direction(domain_search.sort, domain_search.order) domains = Domain.objects.all().order_by( sort_direction(domain_search.sort, domain_search.order) diff --git a/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py new file mode 100644 index 00000000..e592a0e7 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py @@ -0,0 +1,21 @@ +# Third-Party Libraries +from fastapi import HTTPException + + +def sort_direction(sort, order): + """ + Adds the sort direction modifier. + If sort = + ASC - return order field unmodified to sort in ascending order. + DSC - returns & prepend '-' to the order field to sort in descending order. + """ + try: + # Fetch all domains in list + if sort == "ASC": + return order + elif sort == "DSC": + return "-" + order + else: + raise ValueError + except ValueError as e: + raise HTTPException(status_code=500, detail="Invalid sort direction supplied") From ac3204fdd1c38744ccbe040871263a25435b3875 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 9 Oct 2024 12:57:05 -0400 Subject: [PATCH 051/314] Merge AL-python-serverless --- backend/requirements.txt | 1 + .../xfd_django/xfd_api/api_methods/domain.py | 101 ++++++++++++++- .../xfd_api/api_methods/organization.py | 31 ++--- .../xfd_django/xfd_api/api_methods/user.py | 39 ++---- .../xfd_api/helpers/filter_helpers.py | 21 ++++ .../xfd_django/xfd_api/helpers/s3_client.py | 117 ++++++++++++++++++ .../xfd_api/schema_models/domain.py | 7 +- backend/src/xfd_django/xfd_api/views.py | 59 ++++++--- 8 files changed, 300 insertions(+), 76 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/helpers/filter_helpers.py create mode 100644 backend/src/xfd_django/xfd_api/helpers/s3_client.py diff --git a/backend/requirements.txt b/backend/requirements.txt index 5a46a04e..2fe1a318 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,6 +5,7 @@ django docker fastapi==0.111.0 mangum==0.17.0 +minio psycopg2-binary PyJWT pytest diff --git a/backend/src/xfd_django/xfd_api/api_methods/domain.py b/backend/src/xfd_django/xfd_api/api_methods/domain.py index 92a0c1dd..5bf3f4ba 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/domain.py +++ b/backend/src/xfd_django/xfd_api/api_methods/domain.py @@ -3,10 +3,65 @@ """ +# Standard Python Libraries +import csv + # Third-Party Libraries +from django.core.paginator import Paginator from fastapi import HTTPException -from ..models import Domain +from ..helpers.filter_helpers import sort_direction +from ..models import Domain, Organization, Service, Vulnerability +from ..schema_models.domain import DomainFilters, DomainSearch + + +def filter_domains(domains, domain_filters: DomainFilters): + """ + Filter domains + Arguments: + domains: A list of all domains, sorted + domain_filters: Value to filter the domains table by + Returns: + object: a list of Domain objects + """ + try: + if domain_filters.port is not None: + services_by_port = Service.objects.values("domainId").filter( + port=domain_filters.port + ) + domains = domains.filter(id__in=services_by_port) + if domain_filters.service != "": + service_by_id = Service.objects.values("domainId").get( + id=domain_filters.service + ) + domains = domains.filter(id=service_by_id["domainId"]) + if domain_filters.reverseName != "": + domains_by_reverse_name = Domain.objects.values("id").filter( + reverseName=domain_filters.reverseName + ) + domains = domains.filter(id__in=domains_by_reverse_name) + if domain_filters.ip != "": + domains_by_ip = Domain.objects.values("id").filter(ip=domain_filters.ip) + domains = domains.filter(id__in=domains_by_ip) + if domain_filters.organization != "": + domains_by_org = Domain.objects.values("id").filter( + organizationId_id=domain_filters.organization + ) + domains = domains.filter(id__in=domains_by_org) + if domain_filters.organizationName != "": + organization_by_name = Organization.objects.values("id").filter( + name=domain_filters.organizationName + ) + domains = domains.filter(organizationId_id__in=organization_by_name) + if domain_filters.vulnerabilities != "": + vulnerabilities_by_id = Vulnerability.objects.values("domainId").filter( + id=domain_filters.vulnerabilities + ) + domains = domains.filter(id__in=vulnerabilities_by_id) + + return domains + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) def get_domain_by_id(domain_id: str): @@ -22,3 +77,47 @@ def get_domain_by_id(domain_id: str): raise HTTPException(status_code=404, detail="Domain not found.") except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +def search_domains(domain_search: DomainSearch): + """ + List domains by search filter + Arguments: + domain_search: A DomainSearch object to filter by. + Returns: + object: A paginated list of Domain objects + """ + try: + # Fetch all domains in list + + domains = Domain.objects.all().order_by( + sort_direction(domain_search.sort, domain_search.order) + ) + if domain_search.filters is not None: + results = filter_domains(domains, domain_search.filters) + paginator = Paginator(results, domain_search.pageSize) + + return paginator.get_page(domain_search.page) + else: + raise ValueError("DomainFilters cannot be NoneType") + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def export_domains(domain_search: DomainSearch): + try: + domains = Domain.objects.all().order_by( + sort_direction(domain_search.sort, domain_search.order) + ) + + if domain_search.filters is not None: + results = filter_domains(domains, domain_search.filters) + paginator = Paginator(results, domain_search.pageSize) + + return paginator.get_page(domain_search.page) + # TODO: Implement S3 client methods after collab with entire team. + # return export_to_csv(paginator, domains, "testing", True) + else: + raise ValueError("DomainFilters cannot be NoneType") + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/organization.py b/backend/src/xfd_django/xfd_api/api_methods/organization.py index 2701a968..e0594ee0 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/organization.py +++ b/backend/src/xfd_django/xfd_api/api_methods/organization.py @@ -48,16 +48,11 @@ def read_orgs(): raise HTTPException(status_code=500, detail=str(e)) -def get_organizations( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), -): +def get_organizations(regionId): """ List all organizations with query parameters. Args: - state (Optional[List[str]]): List of states to filter organizations by. - regionId (Optional[List[str]]): List of region IDs to filter organizations by. - + regionId : region IDs to filter organizations by. Raises: HTTPException: If the user is not authorized or no organizations are found. @@ -65,20 +60,8 @@ def get_organizations( List[Organizations]: A list of organizations matching the filter criteria. """ - # if not current_user: - # raise HTTPException(status_code=401, detail="Unauthorized") - - # Prepare filter parameters - filter_params = {} - if state: - filter_params["state__in"] = state - if regionId: - filter_params["regionId__in"] = regionId - - organizations = Organization.objects.filter(**filter_params) - - if not organizations.exists(): - raise HTTPException(status_code=404, detail="No organizations found") - - # Return the Pydantic models directly by calling from_orm - return organizations + try: + organizations = Organization.objects.filter(regionId=regionId) + return organizations + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/user.py b/backend/src/xfd_django/xfd_api/api_methods/user.py index 3052c012..fdb0f07a 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/user.py +++ b/backend/src/xfd_django/xfd_api/api_methods/user.py @@ -12,44 +12,21 @@ from ..schema_models.user import User as UserSchema -def get_users( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), - invitePending: Optional[List[str]] = Query(None), - # current_user: User = Depends(is_regional_admin) -): +def get_users(regionId): """ Retrieve a list of users based on optional filter parameters. Args: - state (Optional[List[str]]): List of states to filter users by. - regionId (Optional[List[str]]): List of region IDs to filter users by. - invitePending (Optional[List[str]]): List of invite pending statuses to filter users by. - current_user (User): The current authenticated user, must be a regional admin. - + regionId : Region ID to filter users by. Raises: HTTPException: If the user is not authorized or no users are found. Returns: List[User]: A list of users matching the filter criteria. """ - # if not current_user: - # raise HTTPException(status_code=401, detail="Unauthorized") - - # Prepare filter parameters - filter_params = {} - if state: - filter_params["state__in"] = state - if regionId: - filter_params["regionId__in"] = regionId - if invitePending: - filter_params["invitePending__in"] = invitePending - - # Query users with filter parameters and prefetch related roles - users = User.objects.filter(**filter_params).prefetch_related("roles") - - if not users.exists(): - raise HTTPException(status_code=404, detail="No users found") - - # Return the Pydantic models directly by calling from_orm - return [UserSchema.from_orm(user) for user in users] + + try: + users = User.objects.filter(regionId=regionId).prefetch_related("roles") + return [UserSchema.from_orm(user) for user in users] + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py new file mode 100644 index 00000000..e592a0e7 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py @@ -0,0 +1,21 @@ +# Third-Party Libraries +from fastapi import HTTPException + + +def sort_direction(sort, order): + """ + Adds the sort direction modifier. + If sort = + ASC - return order field unmodified to sort in ascending order. + DSC - returns & prepend '-' to the order field to sort in descending order. + """ + try: + # Fetch all domains in list + if sort == "ASC": + return order + elif sort == "DSC": + return "-" + order + else: + raise ValueError + except ValueError as e: + raise HTTPException(status_code=500, detail="Invalid sort direction supplied") diff --git a/backend/src/xfd_django/xfd_api/helpers/s3_client.py b/backend/src/xfd_django/xfd_api/helpers/s3_client.py new file mode 100644 index 00000000..3bbac400 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/helpers/s3_client.py @@ -0,0 +1,117 @@ +# Standard Python Libraries +import csv +from datetime import datetime +import os +from random import randint + +# Third-Party Libraries +import boto3 +import boto3.s3 +from django.core.paginator import Paginator +from django.http import HttpResponse +from fastapi import HTTPException +from minio import Minio + + +def get_minio_client(): + return Minio( + "localhost:9000", + access_key=os.environ["MINIO_ACCESS_KEY"], + secret_key=os.environ["MINIO_SECRET_KEY"], + secure=False, + ) + + +async def save_minio(data, key: str): + client = get_minio_client() + try: + if not client.bucket_exists(os.environ["EXPORT_BUCKET_NAME"]): + client.make_bucket(os.environ["EXPORT_BUCKET_NAME"]) + + client.put_object(os.environ["EXPORT_BUCKET_NAME"], data, key) + print("File uploaded successfully") + except Exception as e: + print(f"Error uploading file to Minio: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +async def export_to_csv(pages: Paginator, queryset, name, is_local: bool = True): + try: + filename = f"{randint(0, 1000000000)}/{name}-{datetime.now()}.csv" + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f"attachment; filename={filename}" + + writer = csv.writer(response) + if queryset.count() > 0: + writer.writerow( + [ + "id", + "createdAt", + "updatedAt", + "syncedAt", + "ip", + "fromRootDomain", + "subdomainSource", + "ipOnly", + "reverseName", + "name", + "screenshot", + "country", + "asn", + "cloudHosted", + "ssl", + "censysCertificatesResults", + "trustymailResults", + "discoveredById_id", + "organizationId_id", + ] + ) + + for page_number in pages.page_range: + page = pages.page(page_number) + for obj in page.object_list: + print(obj) + writer.writerow( + [ + obj.id, + obj.createdAt, + obj.updatedAt, + obj.syncedAt, + obj.ip, + obj.fromRootDomain, + obj.subdomainSource, + obj.ipOnly, + obj.reverseName, + obj.name, + obj.screnshot, + obj.country, + obj.asn, + obj.cloudHosted, + obj.ssl, + obj.censysCertificatesResults, + obj.trustymailResults, + obj.discoveredById_id, + obj.organizationId_id, + ] + ) + # if not is_local: + # s3_client = boto3.client("s3") + # s3_client.put_object( + # Body=response.content, + # Bucket=os.environ['EXPORT_BUCKET_NAME'], + # Key=filename + # ) + # else: + # client = get_minio_client() + # csv_values = response.getvalue() + # if not client.bucket_exists(os.environ['EXPORT_BUCKET_NAME']): + # client.make_bucket(os.environ['EXPORT_BUCKET_NAME']) + # client.put_object( + # os.environ['EXPORT_BUCKET_NAME'], + # filename, + # len(csv_values), + # content_type='text/csv' + # ) + return response + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/domain.py b/backend/src/xfd_django/xfd_api/schema_models/domain.py index 96a30464..2a48cd80 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/domain.py +++ b/backend/src/xfd_django/xfd_api/schema_models/domain.py @@ -3,12 +3,15 @@ # from pydantic.types import UUID1, UUID # Standard Python Libraries from datetime import datetime -from typing import Any, Optional +from typing import Any, List, Optional from uuid import UUID # Third-Party Libraries from pydantic import BaseModel +from ..schema_models.service import Service +from ..schema_models.vulnerability import Vulnerability + class Domain(BaseModel): """Domain schema.""" @@ -43,7 +46,7 @@ class Config: class DomainFilters(BaseModel): """DomainFilters schema.""" - ports: Optional[str] = None + port: Optional[str] = None service: Optional[str] = None reverseName: Optional[str] = None ip: Optional[str] = None diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index cc7314c8..4c149f2a 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -34,7 +34,7 @@ from .api_methods.api_keys import get_api_keys from .api_methods.cpe import get_cpes_by_id from .api_methods.cve import get_cves_by_id, get_cves_by_name -from .api_methods.domain import get_domain_by_id +from .api_methods.domain import export_domains, get_domain_by_id, search_domains from .api_methods.organization import get_organizations, read_orgs from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability @@ -193,18 +193,27 @@ async def call_get_cves_by_name(cve_name): return get_cves_by_name(cve_name) -@api_router.post("/domain/search") -async def search_domains(domain_search: DomainSearch): +@api_router.post( + "/domain/search", + # dependencies=[Depends(get_current_active_user)], + response_model=List[DomainSchema], + tags=["Domains"], +) +async def call_search_domains(domain_search: DomainSearch): try: - pass + return search_domains(domain_search) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) -@api_router.post("/domain/export") -async def export_domains(): +@api_router.post( + "/domain/export", + # dependencies=[Depends(get_current_active_user)], + tags=["Domains"], +) +async def call_export_domains(domain_search: DomainSearch): try: - pass + return export_domains(domain_search) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -315,25 +324,17 @@ async def read_users_me(current_user: User = Depends(get_current_active_user)): # V2 GET Users @api_router.get( - "/v2/users", + "/users/{regionId}", response_model=List[UserSchema], # dependencies=[Depends(get_current_active_user)], tags=["users"], ) -async def call_get_users( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), - invitePending: Optional[List[str]] = Query(None), - # current_user: User = Depends(is_regional_admin) -): +async def call_get_users(regionId): """ Call get_users() Args: - state (Optional[List[str]]): List of states to filter users by. - regionId (Optional[List[str]]): List of region IDs to filter users by. - invitePending (Optional[List[str]]): List of invite pending statuses to filter users by. - current_user (User): The current authenticated user, must be a regional admin. + regionId: Region IDs to filter users by. Raises: HTTPException: If the user is not authorized or no users are found. @@ -382,6 +383,28 @@ async def delete_api_key( ): """Delete api key by id.""" return api_key_methods.delete(id, current_user) + return get_users(regionId) + + +@api_router.get( + "/organizations/{regionId}", + response_model=List[OrganizationSchema], + # dependencies=[Depends(get_current_active_user)], + tags=["Organization"], +) +async def call_get_organizations(regionId): + """ + List all organizations with query parameters. + Args: + regionId : Region IDs to filter organizations by. + + Raises: + HTTPException: If the user is not authorized or no organizations are found. + + Returns: + List[Organizations]: A list of organizations matching the filter criteria. + """ + return get_organizations(regionId) # GET ALL From 49ca2cf798c63aa97437eaba4cbb30451a9fcf1d Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 9 Oct 2024 13:11:57 -0400 Subject: [PATCH 052/314] Make sure scans aren't overwritten --- backend/src/xfd_django/xfd_api/views.py | 100 ++++++++++++++++++ backend/worker/requirements.txt | 1 - docker-compose.yml | 4 +- .../src/pages/OktaCallback/OktaCallback.tsx | 22 ++-- 4 files changed, 114 insertions(+), 13 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 46ae9058..1c056bb3 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -41,6 +41,7 @@ from .auth import get_current_active_user from .login_gov import callback, login from .models import Assessment, User +from .schema_models import scan as scanSchema from .schema_models.api_key import ApiKey as ApiKeySchema from .schema_models.assessment import Assessment as AssessmentSchema from .schema_models.cpe import Cpe as CpeSchema @@ -486,3 +487,102 @@ async def update_notification( async def get_508_banner(current_user: User = Depends(get_current_active_user)): """Get notification by id.""" return notification_methods.get_508_banner(current_user) + + +# ======================================== +# Scan Endpoints +# ======================================== + + +@api_router.get( + "/scans", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.GetScansResponseModel, + tags=["Scans"], +) +async def list_scans(current_user: User = Depends(get_current_active_user)): + """Retrieve a list of all scans.""" + return scan.list_scans(current_user) + + +@api_router.get( + "/granularScans", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.GetGranularScansResponseModel, + tags=["Scans"], +) +async def list_granular_scans(current_user: User = Depends(get_current_active_user)): + """Retrieve a list of granular scans. User must be authenticated.""" + return scan.list_granular_scans(current_user) + + +@api_router.post( + "/scans", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.CreateScanResponseModel, + tags=["Scans"], +) +async def create_scan( + scan_data: scanSchema.NewScan, current_user: User = Depends(get_current_active_user) +): + """Create a new scan.""" + return scan.create_scan(scan_data, current_user) + + +@api_router.get( + "/scans/{scan_id}", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.GetScanResponseModel, + tags=["Scans"], +) +async def get_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): + """Get a scan by its ID. User must be authenticated.""" + return scan.get_scan(scan_id, current_user) + + +@api_router.put( + "/scans/{scan_id}", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.CreateScanResponseModel, + tags=["Scans"], +) +async def update_scan( + scan_id: str, + scan_data: scanSchema.NewScan, + current_user: User = Depends(get_current_active_user), +): + """Update a scan by its ID.""" + return scan.update_scan(scan_id, scan_data, current_user) + + +@api_router.delete( + "/scans/{scan_id}", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.GenericMessageResponseModel, + tags=["Scans"], +) +async def delete_scan( + scan_id: str, current_user: User = Depends(get_current_active_user) +): + """Delete a scan by its ID.""" + return scan.delete_scan(scan_id, current_user) + + +@api_router.post( + "/scans/{scan_id}/run", + dependencies=[Depends(get_current_active_user)], + response_model=scanSchema.GenericMessageResponseModel, + tags=["Scans"], +) +async def run_scan(scan_id: str, current_user: User = Depends(get_current_active_user)): + """Manually run a scan by its ID""" + return scan.run_scan(scan_id, current_user) + + +@api_router.post( + "/scheduler/invoke", dependencies=[Depends(get_current_active_user)], tags=["Scans"] +) +async def invoke_scheduler(current_user: User = Depends(get_current_active_user)): + """Manually invoke the scan scheduler.""" + response = await scan.invoke_scheduler(current_user) + return response \ No newline at end of file diff --git a/backend/worker/requirements.txt b/backend/worker/requirements.txt index 8aaaa44f..ebffb092 100644 --- a/backend/worker/requirements.txt +++ b/backend/worker/requirements.txt @@ -24,7 +24,6 @@ pytz==2023.3 pytz-deprecation-shim==0.1.0.post0 regex==2023.3.23 requests==2.32.3 -requests==2.32.3 requests-http-signature==0.2.0 scikit-learn==1.2.2 Scrapy==2.11.2 diff --git a/docker-compose.yml b/docker-compose.yml index 4352e71b..a6585a7f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ --- -# version: '3.4' +version: '3.4' services: db: @@ -129,7 +129,7 @@ services: environment: - MYSQL_ROOT_PASSWORD=password logging: - driver: "json-file" + driver: none matomo: image: matomo:3.14.1 diff --git a/frontend/src/pages/OktaCallback/OktaCallback.tsx b/frontend/src/pages/OktaCallback/OktaCallback.tsx index 479b90e3..f8a9c421 100644 --- a/frontend/src/pages/OktaCallback/OktaCallback.tsx +++ b/frontend/src/pages/OktaCallback/OktaCallback.tsx @@ -1,9 +1,13 @@ import React, { useCallback, useEffect } from 'react'; import { parse } from 'query-string'; import { useAuthContext } from 'context'; +import { User } from 'types'; import { useHistory } from 'react-router-dom'; -type OktaCallbackResponse = any; +type OktaCallbackResponse = { + token: string; + user: User; +}; export const OktaCallback: React.FC = () => { const { apiPost, login } = useAuthContext(); @@ -12,6 +16,8 @@ export const OktaCallback: React.FC = () => { const handleOktaCallback = useCallback(async () => { const { code } = parse(window.location.search); console.log('Code: ', code); + const nonce = localStorage.getItem('nonce'); + console.log('Nonce: ', nonce); try { // Pass request to backend callback endpoint @@ -20,21 +26,17 @@ export const OktaCallback: React.FC = () => { { body: { code: code - }, - headers: { - 'Content-Type': 'application/json' } } ); + console.log('Response: ', response); + console.log('token ', response.token); - console.log('Response: ', response.body); - console.log('token ', response.body.token); - const nonce = localStorage.getItem('nonce'); - console.log('Nonce: ', nonce); // Login - login(response.token); + await login(response.token); + // Storage Management - localStorage.setItem('token', response.body.token); + localStorage.setItem('token', response.token); localStorage.removeItem('nonce'); localStorage.removeItem('state'); From 4f0fd132400d98509fe3fb1bf7fb991fec5b78ed Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 9 Oct 2024 13:27:11 -0400 Subject: [PATCH 053/314] Run pre-commit checks. Remove helper.py since not used --- backend/requirements.txt | 1 - backend/src/xfd_django/xfd_api/helpers.py | 57 ----------------------- backend/src/xfd_django/xfd_api/views.py | 7 ++- 3 files changed, 3 insertions(+), 62 deletions(-) delete mode 100644 backend/src/xfd_django/xfd_api/helpers.py diff --git a/backend/requirements.txt b/backend/requirements.txt index 2fe1a318..4043de6a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,7 +1,6 @@ boto3 cryptography==38.0.0 django -django docker fastapi==0.111.0 mangum==0.17.0 diff --git a/backend/src/xfd_django/xfd_api/helpers.py b/backend/src/xfd_django/xfd_api/helpers.py deleted file mode 100644 index 592f5986..00000000 --- a/backend/src/xfd_django/xfd_api/helpers.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -This module provides various helper functions for the api endpoints. - -""" -# Standard Python Libraries -from datetime import datetime -import uuid - -# Third-Party Libraries -# Third Party Libraries -from django.forms.models import model_to_dict -from fastapi import HTTPException, status -from xfd_api.models import User - - -def user_to_dict(user): - """Takes a user model object from django and - sanitizes fields for output. - - Args: - user (django model): Django User model object - - Returns: - dict: Returns sanitized and formated dict - """ - user_dict = model_to_dict(user) # Convert model to dict - # Convert any UUID fields to strings - if isinstance(user_dict.get("id"), uuid.UUID): - user_dict["id"] = str(user_dict["id"]) - for key, val in user_dict.items(): - if isinstance(val, datetime): - user_dict[key] = str(val) - return user_dict - - -async def update_or_create_user(user_info): - try: - print(f"User info from update_or_create: {user_info}") - user, created = User.objects.get_or_create(email=user_info["email"]) - if created: - user.oktaId = user_info["sub"] - user.firstName = user_info["given_name"] - user.lastName = user_info["family_name"] - user.invitePending = True - user.userType = "standard" - user.save() - else: - if user.cognitoId != user_info["sub"]: - user.cognitoId = user_info["sub"] - user.lastLoggedIn = datetime.utcnow() - user.save() - return user - except Exception as error: - print(f"Error from update_or_create: {str(error)}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(error) - ) from error diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 1c056bb3..863b3175 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -330,7 +330,7 @@ async def read_users_me(current_user: User = Depends(get_current_active_user)): # dependencies=[Depends(get_current_active_user)], tags=["users"], ) -async def async def call_get_users( +async def call_get_users( state: Optional[List[str]] = Query(None), regionId: Optional[List[str]] = Query(None), invitePending: Optional[List[str]] = Query(None), @@ -367,7 +367,7 @@ async def async def call_get_users( raise HTTPException(status_code=404, detail="No users found") # Return the Pydantic models directly by calling from_orm - return [UserSchema.from_orm(user) for user in users] + return [UserSchema.model_validate(user) for user in users] ###################### @@ -389,7 +389,6 @@ async def delete_api_key( ): """Delete api key by id.""" return api_key_methods.delete(id, current_user) - return get_users(regionId) @api_router.get( @@ -585,4 +584,4 @@ async def run_scan(scan_id: str, current_user: User = Depends(get_current_active async def invoke_scheduler(current_user: User = Depends(get_current_active_user)): """Manually invoke the scan scheduler.""" response = await scan.invoke_scheduler(current_user) - return response \ No newline at end of file + return response From 8b323e0b7b70d60efa86e5342ae20be45c40bfc1 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 9 Oct 2024 13:41:45 -0400 Subject: [PATCH 054/314] Update auth to also accept api key --- backend/src/xfd_django/xfd_api/auth.py | 146 +++++++----------------- backend/src/xfd_django/xfd_api/views.py | 31 +---- 2 files changed, 45 insertions(+), 132 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 019b62b2..618dbc8b 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -1,23 +1,5 @@ -""" -This module provides authentication utilities for the FastAPI application. - -It includes functions to: -- Decode JWT tokens and retrieve the current user. -- Retrieve a user by their API key. -- Ensure the current user is authenticated and active. - -Functions: - - get_current_user: Decodes a JWT token to retrieve the current user. - - get_user_by_api_key: Retrieves a user by their API key. - - get_current_active_user: Ensures the current user is authenticated and active. - -Dependencies: - - fastapi - - django - - hashlib - - .jwt_utils - - .models -""" +"""Authentication utilities for the FastAPI application.""" + # Standard Python Libraries from datetime import datetime, timedelta import hashlib @@ -30,7 +12,7 @@ from django.conf import settings from django.forms.models import model_to_dict from django.utils import timezone -from fastapi import Depends, HTTPException, status +from fastapi import Depends, HTTPException, Security, status from fastapi.security import APIKeyHeader, OAuth2PasswordBearer import jwt from jwt import ExpiredSignatureError, InvalidTokenError @@ -193,15 +175,7 @@ def hash_key(key: str) -> str: # TODO: Confirm still needed async def get_user_info_from_cognito(token): - """ - Get user info from cognito - - Args: - token (_type_): _description_ - - Returns: - _type_: _description_ - """ + """Get user info from cognito.""" jwks_url = f"https://cognito-idp.us-east-1.amazonaws.com/{os.getenv('REACT_APP_USER_POOL_ID')}/.well-known/jwks.json" response = requests.get(jwks_url) jwks = response.json() @@ -221,101 +195,65 @@ async def get_user_info_from_cognito(token): return user_info -def get_current_user(token: str = Depends(oauth2_scheme)): - """ - Decode a JWT token to retrieve the current user. - - Args: - token (str): The JWT token. - - Raises: - HTTPException: If the token is invalid or expired. - - Returns: - User: The user object decoded from the token. - """ - user = decode_jwt_token(token) - if user is None: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid or expired token", - ) - return user - - def get_user_by_api_key(api_key: str): - """ - Retrieve a user by their API key. - - Args: - api_key (str): The API key. - - Returns: - User: The user object associated with the API key, or None if not found. - """ + """Get a user by their API key.""" hashed_key = sha256(api_key.encode()).hexdigest() try: api_key_instance = ApiKey.objects.get(hashedKey=hashed_key) - api_key_instance.lastused = timezone.now() + api_key_instance.lastUsed = timezone.now() api_key_instance.save(update_fields=["lastUsed"]) - return api_key_instance.userid + return api_key_instance.userId except ApiKey.DoesNotExist: print("API Key not found") return None -# TODO: enable usage of X-API-KEY header if needed -# adding arg api_key: str = Security(api_key_header), -def get_current_active_user(token: str = Depends(oauth2_scheme)): - """ - Ensure the current user is authenticated and active. - - Args: - token (str): The Authorization token header. - - Raises: - HTTPException: If the user is not authenticated. - - Returns: - User: The authenticated user object. - """ - try: - # Decode token in Authorization header to get user - payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) - user_id = payload.get("id") - - if user_id is None: - print("No user ID found in token") +def get_current_active_user( + api_key: str = Security(api_key_header), + token: str = Depends(oauth2_scheme), +): + """Ensure the current user is authenticated and active.""" + user = None + if api_key: + user = get_user_by_api_key(api_key) + return user + else: + try: + # Decode token in Authorization header to get user + payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) + user_id = payload.get("id") + + if user_id is None: + print("No user ID found in token") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token", + headers={"WWW-Authenticate": "Bearer"}, + ) + # Fetch the user by ID from the database + user = User.objects.get(id=user_id) + print(f"User found: {user_to_dict(user)}") + except jwt.ExpiredSignatureError: + print("Token has expired") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid token", + detail="Token has expired", headers={"WWW-Authenticate": "Bearer"}, ) - # Fetch the user by ID from the database - user = User.objects.get(id=user_id) - print(f"User found: {user_to_dict(user)}") - if user is None: - print("User not found") + except jwt.InvalidTokenError: + print("Invalid token") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, - detail="User not found", + detail="Invalid token", headers={"WWW-Authenticate": "Bearer"}, ) - return user - except jwt.ExpiredSignatureError: - print("Token has expired") - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Token has expired", - headers={"WWW-Authenticate": "Bearer"}, - ) - except jwt.InvalidTokenError: - print("Invalid token") + if user is None: + print("User not authenticated") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid token", - headers={"WWW-Authenticate": "Bearer"}, + detail="Invalid authentication credentials" ) + return user async def process_user(decoded_token, access_token, refresh_token): diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 863b3175..54a9a887 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -323,19 +323,13 @@ async def read_users_me(current_user: User = Depends(get_current_active_user)): return current_user -# V2 GET Users @api_router.get( "/users/{regionId}", response_model=List[UserSchema], # dependencies=[Depends(get_current_active_user)], - tags=["users"], + tags=["User"], ) -async def call_get_users( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), - invitePending: Optional[List[str]] = Query(None), - # current_user: User = Depends(is_regional_admin) -): +async def call_get_users(regionId): """ Call get_users() @@ -348,26 +342,7 @@ async def call_get_users( Returns: List[User]: A list of users matching the filter criteria. """ - # if not current_user: - # raise HTTPException(status_code=401, detail="Unauthorized") - - # Prepare filter parameters - filter_params = {} - if state: - filter_params["state__in"] = state - if regionId: - filter_params["regionId__in"] = regionId - if invitePending: - filter_params["invitePending__in"] = invitePending - - # Query users with filter parameters and prefetch related roles - users = User.objects.filter(**filter_params).prefetch_related("roles") - - if not users.exists(): - raise HTTPException(status_code=404, detail="No users found") - - # Return the Pydantic models directly by calling from_orm - return [UserSchema.model_validate(user) for user in users] + return get_users(regionId) ###################### From 30a091c74ebdfcf8923ecd6f57c3392034c64d06 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 9 Oct 2024 13:43:41 -0400 Subject: [PATCH 055/314] run pre-commit --- backend/src/xfd_django/xfd_api/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 618dbc8b..f06ff3ae 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -251,7 +251,7 @@ def get_current_active_user( print("User not authenticated") raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid authentication credentials" + detail="Invalid authentication credentials", ) return user From f1722d89f414d1b47efb591a1a3bf4587cc12645 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 9 Oct 2024 14:00:21 -0400 Subject: [PATCH 056/314] fix auth.py --- backend/src/xfd_django/xfd_api/auth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index f06ff3ae..8981e746 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -216,7 +216,6 @@ def get_current_active_user( user = None if api_key: user = get_user_by_api_key(api_key) - return user else: try: # Decode token in Authorization header to get user From 017d442da9a7193c48e34d62e4469ba9037b372b Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Thu, 10 Oct 2024 10:29:41 -0500 Subject: [PATCH 057/314] Add delete_user endpoint. --- .../xfd_django/xfd_api/api_methods/user.py | 21 +++++++++++++++++++ backend/src/xfd_django/xfd_api/views.py | 21 ++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/user.py b/backend/src/xfd_django/xfd_api/api_methods/user.py index fdb0f07a..55b3c3b2 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/user.py +++ b/backend/src/xfd_django/xfd_api/api_methods/user.py @@ -30,3 +30,24 @@ def get_users(regionId): return [UserSchema.from_orm(user) for user in users] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +def delete_user(user_id: str): + """ + Delete a user by ID. + + Args: + user_id : The ID of the user to delete. + Raises: + HTTPException: If the user is not authorized or the user is not found. + + Returns: + User: The user that was deleted. + """ + + try: + user = User.objects.get(id=user_id) + user.delete() + return UserSchema.from_orm(user) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 54a9a887..85a2fd88 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -36,7 +36,7 @@ from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import export_domains, get_domain_by_id, search_domains from .api_methods.organization import get_organizations, read_orgs -from .api_methods.user import get_users +from .api_methods.user import delete_user, get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user from .login_gov import callback, login @@ -318,7 +318,7 @@ async def callback_route(request: Request): # GET Current User -@api_router.get("/users/me", tags=["users"]) +@api_router.get("/users/me", tags=["Users"]) async def read_users_me(current_user: User = Depends(get_current_active_user)): return current_user @@ -327,7 +327,7 @@ async def read_users_me(current_user: User = Depends(get_current_active_user)): "/users/{regionId}", response_model=List[UserSchema], # dependencies=[Depends(get_current_active_user)], - tags=["User"], + tags=["Users"], ) async def call_get_users(regionId): """ @@ -345,6 +345,21 @@ async def call_get_users(regionId): return get_users(regionId) +@api_router.delete("/users/{userId}", tags=["Users"]) +async def call_delete_user(userId: str): + """ + call delete_user() + Args: + userId: UUID of the user to delete. + + Returns: + User: The user that was deleted. + + """ + + return delete_user(userId) + + ###################### # API-Keys ###################### From e27db5d5668dc765bfbc1e7d84dd3a5f26cbabd2 Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Thu, 10 Oct 2024 15:21:20 -0400 Subject: [PATCH 058/314] Added delete saved search endpoint; Formatted initial response model for get all saved searches --- .../xfd_api/api_methods/saved_search.py | 47 +++++++++++++------ backend/src/xfd_django/xfd_api/auth.py | 9 ++-- .../xfd_api/schema_models/saved_search.py | 1 - backend/src/xfd_django/xfd_api/views.py | 18 +++---- 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py index 3f3acf8e..62d25426 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -36,9 +36,28 @@ def list_saved_searches(): try: saved_searches = SavedSearch.objects.all() count = saved_searches.count() + saved_search_list = [] for search in saved_searches: - print(str(search.id)) - return {"saved_searches": list(saved_searches), "Saved Search Count": count} + print(str(saved_search_list)) + response = { + "id": str(search.id), + "name": search.name, + "count": search.count, + "sort_direction": search.sortDirection, + "sort_field": search.sortField, + "search_term": search.searchTerm, + "search_path": search.searchPath, + "filters": search.filters, + "create_vulnerabilities": search.createVulnerabilities, + "vulnerability_template": search.vulnerabilityTemplate, + "created_by": { + "id": search.createdById.id, + "user_name": search.createdById.fullName, + }, + } + saved_search_list.append(response) + + return {"saved_searches": list(saved_search_list), "Saved Search Count": count} except Exception as e: raise HTTPException(status_code=404, detail=str(e)) @@ -99,16 +118,16 @@ def get_saved_search(search_id): # return JsonResponse({"status": "Updated", "search": search.id}, status=200) -# def delete_saved_search(request, search_id): -# """Delete saved search by id.""" -# if not uuid.UUID(search_id): -# raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) +def delete_saved_search(search_id): + """Delete saved search by id.""" + if not uuid.UUID(search_id): + raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) -# try: -# search = SavedSearch.objects.get(id=search_id, created_by=request.user) -# search.delete() -# return JsonResponse( -# {"status": "success", "message": f"Saved search id:{search_id} deleted."} -# ) -# except SavedSearch.DoesNotExist as e: -# raise HTTPException(status_code=404, detail=str(e)) + try: + search = SavedSearch.objects.get(id=search_id) + search.delete() + return JsonResponse( + {"status": "success", "message": f"Saved search id:{search_id} deleted."} + ) + except SavedSearch.DoesNotExist as e: + raise HTTPException(status_code=404, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 6b256b89..fe33d7df 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -74,6 +74,7 @@ def is_regional_admin(current_user) -> bool: """Check if the user has regional admin permissions.""" return current_user and current_user.userType in ["regionalAdmin", "globalAdmin"] + def get_tag_organizations(current_user, tag_id: str) -> list[str]: """Returns the organizations belonging to a tag, if the user can access the tag.""" # Check if the user is a global view admin @@ -81,7 +82,11 @@ def get_tag_organizations(current_user, tag_id: str) -> list[str]: return [] # Fetch the OrganizationTag and its related organizations - tag = OrganizationTag.objects.prefetch_related("organizations").filter(id=tag_id).first() + tag = ( + OrganizationTag.objects.prefetch_related("organizations") + .filter(id=tag_id) + .first() + ) if tag: # Return a list of organization IDs return [org.id for org in tag.organizations.all()] @@ -90,8 +95,6 @@ def get_tag_organizations(current_user, tag_id: str) -> list[str]: return [] - - # TODO: Below is a template of what these could be nut isn't tested # RECREATE ALL THE FUNCTIONS IN AUTH.TS diff --git a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py index 50357bdb..28e2ff13 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py +++ b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py @@ -32,7 +32,6 @@ class SavedSearchFilters(BaseModel): sort_field: Optional[str] search_term: Optional[str] search_path: Optional[str] - create_vulnerabilities: Optional[bool] created_by: Optional[UUID] diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index c7246994..2ffdd7ef 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -29,7 +29,11 @@ from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import export_domains, get_domain_by_id, search_domains from .api_methods.organization import get_organizations, read_orgs -from .api_methods.saved_search import get_saved_search, list_saved_searches +from .api_methods.saved_search import ( + delete_saved_search, + get_saved_search, + list_saved_searches, +) from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user @@ -340,6 +344,7 @@ async def call_get_organizations(regionId): # ======================================== +# TODO: Implement the following functions @api_router.post( "/saved-searches", tags=["Testing"], @@ -371,6 +376,7 @@ async def call_get_saved_search(saved_search_id: str): return get_saved_search(saved_search_id) +# TODO: Implement the following functions @api_router.put( "/saved-searches/{saved_search_id}", tags=["Testing"], @@ -380,18 +386,14 @@ async def update_saved_search(saved_search_id: str): return {"status": "ok"} +# Delete saved search is implemented in the following function @api_router.delete( "/saved-searches/{saved_search_id}", tags=["Testing"], ) -async def delete_saved_search(saved_search_id: str): +async def call_delete_saved_search(saved_search_id: str): """Delete a saved search by its ID.""" - return {"status": "ok"} - - -# async def call_list_saved_searches(current_user: User = Depends(get_current_active_user)): -# """Retrieve a list of all saved searches.""" -# return list_saved_searches(current_user) + return delete_saved_search(saved_search_id) # ======================================== From bb4c6d1124f73e911e80025e592ab2315069df4b Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Thu, 10 Oct 2024 15:58:39 -0400 Subject: [PATCH 059/314] Updated Saved Search Base Schema --- .../xfd_api/api_methods/saved_search.py | 17 +++++++---------- .../xfd_api/schema_models/saved_search.py | 13 ++++++------- backend/src/xfd_django/xfd_api/views.py | 4 ++-- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py index 62d25426..e2b2e8d4 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -41,23 +41,20 @@ def list_saved_searches(): print(str(saved_search_list)) response = { "id": str(search.id), + "created_at": search.createdAt, + "updated_at": search.updatedAt, "name": search.name, - "count": search.count, + "search_term": search.searchTerm, "sort_direction": search.sortDirection, "sort_field": search.sortField, - "search_term": search.searchTerm, - "search_path": search.searchPath, + "count": search.count, "filters": search.filters, - "create_vulnerabilities": search.createVulnerabilities, - "vulnerability_template": search.vulnerabilityTemplate, - "created_by": { - "id": search.createdById.id, - "user_name": search.createdById.fullName, - }, + "search_path": search.searchPath, + "createdBy_id": search.createdById.id, } saved_search_list.append(response) - return {"saved_searches": list(saved_search_list), "Saved Search Count": count} + return list(saved_search_list) except Exception as e: raise HTTPException(status_code=404, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py index 28e2ff13..abad2934 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py +++ b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py @@ -1,7 +1,7 @@ """Saved Search schemas.""" # Standard Python Libraries from datetime import datetime -from typing import Any, Optional +from typing import Any, Dict, List, Optional from uuid import UUID # Third-Party Libraries @@ -12,15 +12,14 @@ class SavedSearch(BaseModel): """SavedSearch schema.""" id: UUID + created_at: datetime + updated_at: datetime name: str - count: int + search_term: str sort_direction: str sort_field: str - search_term: str - search_path: str - filters: Json[Any] - created_at: datetime - updated_at: datetime + count: int + filters: List[Dict[str, Any]] class SavedSearchFilters(BaseModel): diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 2ffdd7ef..3a7e6b41 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -45,7 +45,7 @@ from .schema_models.domain import Domain as DomainSchema from .schema_models.domain import DomainSearch from .schema_models.organization import Organization as OrganizationSchema -from .schema_models.saved_search import SavedSearch as savedSearchSchema +from .schema_models.saved_search import SavedSearch as SavedSearchSchema from .schema_models.user import User as UserSchema from .schema_models.vulnerability import Vulnerability as VulnerabilitySchema @@ -358,7 +358,7 @@ async def create_saved_search(): @api_router.get( "/saved-searches", # dependencies=[Depends(get_current_active_user)], - # response_model=savedSearchSchema, + response_model=List[SavedSearchSchema], tags=["Testing"], ) async def call_list_saved_searches(): From f61785200d36b81421daeb035e36fcf3e6d3e23b Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Fri, 11 Oct 2024 16:38:37 -0400 Subject: [PATCH 060/314] Updated response model for retrieving a single saved search --- .../xfd_api/api_methods/saved_search.py | 16 ++++++++++++++-- .../xfd_api/schema_models/saved_search.py | 2 +- backend/src/xfd_django/xfd_api/views.py | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py index e2b2e8d4..c46c045c 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -35,7 +35,6 @@ def list_saved_searches(): """List all saved searches.""" try: saved_searches = SavedSearch.objects.all() - count = saved_searches.count() saved_search_list = [] for search in saved_searches: print(str(saved_search_list)) @@ -67,7 +66,20 @@ def get_saved_search(search_id): saved_search = SavedSearch.objects.all() for search in saved_search: if str(search.id) == search_id: - return search + response = { + "id": str(search.id), + "created_at": search.createdAt, + "updated_at": search.updatedAt, + "name": search.name, + "search_term": search.searchTerm, + "sort_direction": search.sortDirection, + "sort_field": search.sortField, + "count": search.count, + "filters": search.filters, + "search_path": search.searchPath, + "createdBy_id": search.createdById.id, + } + return response # search = SavedSearch.objects.get(id=search_id, created_by=request.user) # data = { diff --git a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py index abad2934..994b94f8 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py +++ b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py @@ -5,7 +5,7 @@ from uuid import UUID # Third-Party Libraries -from pydantic import BaseModel, Json +from pydantic import BaseModel class SavedSearch(BaseModel): diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 3a7e6b41..6fc66af8 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -369,6 +369,7 @@ async def call_list_saved_searches(): # Get individual saved search is implemented in the following function @api_router.get( "/saved-searches/{saved_search_id}", + response_model=SavedSearchSchema, tags=["Testing"], ) async def call_get_saved_search(saved_search_id: str): From c4f518f4ac6c8dfa1d04a16de54f67b3a2bf42e6 Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Wed, 16 Oct 2024 15:18:07 -0400 Subject: [PATCH 061/314] Updated get_saved_Search and update_saved_search; Created response models for get and update saved search --- .../xfd_api/api_methods/saved_search.py | 137 ++++++++---------- backend/src/xfd_django/xfd_api/views.py | 31 +++- 2 files changed, 87 insertions(+), 81 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py index c46c045c..16294176 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -2,6 +2,7 @@ # Standard Python Libraries +from datetime import datetime import json import uuid @@ -11,24 +12,22 @@ from ..models import SavedSearch -PAGE_SIZE = 20 - -# def create_saved_search(request): -# data = json.loads(request.body) -# search = SavedSearch.objects.create( -# name=data["name"], -# count=data["count"], -# sort_direction=data["sortDirection"], -# sort_field=data["sortField"], -# search_term=data["searchTerm"], -# search_path=data["searchPath"], -# filters=data["filters"], -# create_vulnerabilities=data["createVulnerabilities"], -# vulnerability_template=data.get("vulnerabilityTemplate"), -# created_by=request.user, -# ) -# return JsonResponse({"status": "Created", "search": search.id}, status=201) +def create_saved_search(request): + data = json.loads(request.body) + search = SavedSearch.objects.create( + name=data["name"], + count=data["count"], + sort_direction=data["sortDirection"], + sort_field=data["sortField"], + search_term=data["searchTerm"], + search_path=data["searchPath"], + filters=data["filters"], + create_vulnerabilities=data["createVulnerabilities"], + vulnerability_template=data.get("vulnerabilityTemplate"), + created_by=request.user, + ) + return JsonResponse({"status": "Created", "search": search.id}, status=201) def list_saved_searches(): @@ -63,68 +62,56 @@ def get_saved_search(search_id): raise HTTPException({"error": "Invalid UUID"}, status=404) try: - saved_search = SavedSearch.objects.all() - for search in saved_search: - if str(search.id) == search_id: - response = { - "id": str(search.id), - "created_at": search.createdAt, - "updated_at": search.updatedAt, - "name": search.name, - "search_term": search.searchTerm, - "sort_direction": search.sortDirection, - "sort_field": search.sortField, - "count": search.count, - "filters": search.filters, - "search_path": search.searchPath, - "createdBy_id": search.createdById.id, - } - return response - - # search = SavedSearch.objects.get(id=search_id, created_by=request.user) - # data = { - # "id": str(search.id), - # "name": search.name, - # "count": search.count, - # "sort_direction": search.sort_direction, - # "sort_field": search.sort_field, - # "search_term": search.search_term, - # "search_path": search.search_path, - # "filters": search.filters, - # "create_vulnerabilities": search.create_vulnerabilities, - # "vulnerability_template": search.vulnerability_template, - # "created_by": search.created_by.id, - # } - # return JsonResponse(data) + saved_search = SavedSearch.objects.get(id=search_id) + response = { + "id": str(saved_search.id), + "created_at": saved_search.createdAt, + "updated_at": saved_search.updatedAt, + "name": saved_search.name, + "search_term": saved_search.searchTerm, + "sort_direction": saved_search.sortDirection, + "sort_field": saved_search.sortField, + "count": saved_search.count, + "filters": saved_search.filters, + "search_path": saved_search.searchPath, + "createdBy_id": saved_search.createdById.id, + } + return response except SavedSearch.DoesNotExist as e: raise HTTPException(status_code=404, detail=str(e)) -# def update_saved_search(request, search_id): -# if not uuid.UUID(search_id): -# raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) - -# try: -# search = SavedSearch.objects.get(id=search_id, created_by=request.user) -# except SavedSearch.DoesNotExist as e: -# return HTTPException(status_code=404, detail=str(e)) - -# data = json.loads(request.body) -# search.name = data.get("name", search.name) -# search.count = data.get("count", search.count) -# search.sort_direction = data.get("sortDirection", search.sort_direction) -# search.sort_field = data.get("sortField", search.sort_field) -# search.search_term = data.get("searchTerm", search.search_term) -# search.search_path = data.get("searchPath", search.search_path) -# search.filters = data.get("filters", search.filters) -# search.create_vulnerabilities = data.get( -# "createVulnerabilities", search.create_vulnerabilities -# ) -# search.vulnerability_template = data.get( -# "vulnerabilityTemplate", search.vulnerability_template -# ) -# search.save() -# return JsonResponse({"status": "Updated", "search": search.id}, status=200) +def update_saved_search(request): + if not uuid.UUID(request["saved_search_id"]): + raise HTTPException(status_code=404, detail={"error": "Invalid UUID"}) + + try: + saved_search = SavedSearch.objects.get(id=request["saved_search_id"]) + + # search = SavedSearch.objects.get(id=search_id, created_by=request.user) + except SavedSearch.DoesNotExist as e: + return HTTPException(status_code=404, detail=str(e)) + + saved_search.name = request["name"] + saved_search.updatedAt = datetime.now() + saved_search.searchTerm = request["search_term"] + + saved_search.save() + response = { + "id": saved_search.id, + "created_at": saved_search.createdAt, + "updated_at": saved_search.updatedAt, + "name": saved_search.name, + "search_term": saved_search.searchTerm, + "sort_direction": saved_search.sortDirection, + "sort_field": saved_search.sortField, + "count": saved_search.count, + "filters": saved_search.filters, + "search_path": saved_search.searchPath, + "createdBy_id": saved_search.createdById.id, + } + + return response def delete_saved_search(search_id): diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index c4e70bdd..0a6be309 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -37,16 +37,17 @@ from .api_methods.domain import export_domains, get_domain_by_id, search_domains from .api_methods.organization import get_organizations, read_orgs from .api_methods.saved_search import ( + create_saved_search, delete_saved_search, get_saved_search, list_saved_searches, + update_saved_search, ) from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user -from .models import Assessment, SavedSearch, User from .login_gov import callback, login -from .models import Assessment, User +from .models import Assessment, SavedSearch, User from .schema_models import scan as scanSchema from .schema_models.api_key import ApiKey as ApiKeySchema from .schema_models.assessment import Assessment as AssessmentSchema @@ -56,8 +57,8 @@ from .schema_models.domain import DomainFilters, DomainSearch from .schema_models.notification import Notification as NotificationSchema from .schema_models.organization import Organization as OrganizationSchema -from .schema_models.saved_search import SavedSearch as SavedSearchSchema from .schema_models.role import Role as RoleSchema +from .schema_models.saved_search import SavedSearch as SavedSearchSchema from .schema_models.user import User as UserSchema from .schema_models.vulnerability import Vulnerability as VulnerabilitySchema @@ -424,7 +425,11 @@ async def call_get_organizations(regionId): "/saved-searches", tags=["Testing"], ) -async def create_saved_search(): +async def call_create_saved_search( + name: str, + search_term: str, + region_id: int, +): """Create a new saved search.""" return {"status": "ok"} @@ -455,11 +460,23 @@ async def call_get_saved_search(saved_search_id: str): # TODO: Implement the following functions @api_router.put( "/saved-searches/{saved_search_id}", + response_model=SavedSearchSchema, tags=["Testing"], ) -async def update_saved_search(saved_search_id: str): +async def call_update_saved_search( + saved_search_id: str, + name: str, + search_term: str, +): """Update a saved search by its ID.""" - return {"status": "ok"} + + request = { + "name": name, + "saved_search_id": saved_search_id, + "search_term": search_term, + } + + return update_saved_search(request) # Delete saved search is implemented in the following function @@ -470,6 +487,8 @@ async def update_saved_search(saved_search_id: str): async def call_delete_saved_search(saved_search_id: str): """Delete a saved search by its ID.""" return delete_saved_search(saved_search_id) + + # GET ALL @api_router.get("/api-keys", response_model=List[ApiKeySchema], tags=["api-keys"]) async def get_all_api_keys(current_user: User = Depends(get_current_active_user)): From 3c406b6f0c486971c930a1e07dc06df4200dfbf7 Mon Sep 17 00:00:00 2001 From: nickviola Date: Thu, 17 Oct 2024 09:55:27 -0500 Subject: [PATCH 062/314] Add base file structure and code for python search endpoints --- backend/requirements.txt | 1 + .../xfd_django/xfd_api/api_methods/search.py | 141 ++++++++++++++++++ backend/src/xfd_django/xfd_api/auth.py | 10 +- .../xfd_api/helpers/elastic_search.py | 58 +++++++ .../xfd_django/xfd_api/tests/test_search.py | 0 backend/src/xfd_django/xfd_api/views.py | 33 ++-- 6 files changed, 232 insertions(+), 11 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/api_methods/search.py create mode 100644 backend/src/xfd_django/xfd_api/helpers/elastic_search.py create mode 100644 backend/src/xfd_django/xfd_api/tests/test_search.py diff --git a/backend/requirements.txt b/backend/requirements.txt index 4043de6a..0e1d1968 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,6 +2,7 @@ boto3 cryptography==38.0.0 django docker +elasticsearch fastapi==0.111.0 mangum==0.17.0 minio diff --git a/backend/src/xfd_django/xfd_api/api_methods/search.py b/backend/src/xfd_django/xfd_api/api_methods/search.py new file mode 100644 index 00000000..e7eadaf7 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/search.py @@ -0,0 +1,141 @@ +# api_methods/search.py +# Standard Python Libraries +import csv +from io import StringIO +from typing import Any, Dict, List + +# Third-Party Libraries +import boto3 +from elasticsearch import Elasticsearch +from fastapi import HTTPException +from pydantic import BaseModel +from xfd_api.auth import ( + get_org_memberships, + get_tag_organizations, + is_global_view_admin, +) +from xfd_api.helpers.elastic_search import build_request + + +class SearchBody(BaseModel): + current: int + resultsPerPage: int + searchTerm: str + sortDirection: str + sortField: str + filters: List[Dict[str, Any]] + organizationIds: List[str] = [] + organizationId: str = "" + tagId: str = "" + + +def get_options(search_body: SearchBody, event) -> Dict[str, Any]: + """Determine options for filtering based on organization ID or tag ID.""" + if search_body.organizationId and ( + search_body.organizationId in get_org_memberships(event) + or is_global_view_admin(event) + ): + options = { + "organizationIds": [search_body.organizationId], + "matchAllOrganizations": False, + } + elif search_body.tagId: + options = { + "organizationIds": get_tag_organizations(event, search_body.tagId), + "matchAllOrganizations": False, + } + else: + options = { + "organizationIds": get_org_memberships(event), + "matchAllOrganizations": is_global_view_admin(event), + } + return options + + +def fetch_all_results( + filters: Dict[str, Any], options: Dict[str, Any] +) -> List[Dict[str, Any]]: + """Fetch all search results from Elasticsearch.""" + client = Elasticsearch() + RESULTS_PER_PAGE = 1000 + results = [] + current = 1 + while True: + request = build_request( + {**filters, "current": current, "resultsPerPage": RESULTS_PER_PAGE}, options + ) + current += 1 + try: + search_results = client.search(index="domains", body=request) + except Exception as e: + print(f"Elasticsearch search error: {e}") + continue + if len(search_results["hits"]["hits"]) == 0: + break + results.extend([res["_source"] for res in search_results["hits"]["hits"]]) + return results + + +def export(search_body: SearchBody, event) -> Dict[str, Any]: + """Export the search results into a CSV and upload to S3.""" + options = get_options(search_body, event) + results = fetch_all_results(search_body.dict(), options) + + # Process results for CSV + for res in results: + res["organization"] = res.get("organization", {}).get("name", "") + res["ports"] = ", ".join( + [str(service["port"]) for service in res.get("services", [])] + ) + products = {} + for service in res.get("services", []): + for product in service.get("products", []): + if product.get("name"): + products[product["name"].lower()] = product["name"] + ( + f" {product['version']}" if product.get("version") else "" + ) + res["products"] = ", ".join(products.values()) + + # Create CSV + csv_buffer = StringIO() + fieldnames = [ + "name", + "ip", + "id", + "ports", + "products", + "createdAt", + "updatedAt", + "organization", + ] + writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(results) + + # Save to S3 + s3 = boto3.client("s3") + bucket_name = "your-bucket-name" + csv_key = "domains.csv" + s3.put_object(Bucket=bucket_name, Key=csv_key, Body=csv_buffer.getvalue()) + + # Generate a presigned URL to access the CSV + url = s3.generate_presigned_url( + "get_object", Params={"Bucket": bucket_name, "Key": csv_key}, ExpiresIn=3600 + ) + + return {"url": url} + + +def search(search_body: SearchBody, event) -> Dict[str, Any]: + """Perform a search on Elasticsearch and return results.""" + options = get_options(search_body, event) + request = build_request(search_body.dict(), options) + + client = Elasticsearch() + try: + search_results = client.search(index="domains", body=request) + except Exception as e: + print(f"Elasticsearch search error: {e}") + raise HTTPException(status_code=500, detail="Elasticsearch query failed") + + return search_results["hits"] diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 8981e746..db631da5 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -19,7 +19,7 @@ import requests # from .helpers import user_to_dict -from .models import ApiKey, OrganizationTag, User +from .models import ApiKey, OrganizationTag, Role, User # JWT_ALGORITHM = "RS256" JWT_SECRET = os.getenv("JWT_SECRET") @@ -442,3 +442,11 @@ def get_tag_organizations(current_user, tag_id: str) -> list[str]: # Return an empty list if tag is not found return [] + + +def get_org_memberships(current_user) -> list[str]: + """Returns the organization IDs that a user is a member of.""" + roles = Role.objects.filter(userId=current_user) + if not roles: + return [] + return [role.organizationId.id for role in roles if role.organizationId] diff --git a/backend/src/xfd_django/xfd_api/helpers/elastic_search.py b/backend/src/xfd_django/xfd_api/helpers/elastic_search.py new file mode 100644 index 00000000..ccd836ac --- /dev/null +++ b/backend/src/xfd_django/xfd_api/helpers/elastic_search.py @@ -0,0 +1,58 @@ +# Standard Python Libraries +from typing import Any, Dict + + +def build_request( + searchBody: Dict[str, Any], options: Dict[str, Any] +) -> Dict[str, Any]: + """ + Build an Elasticsearch query from the search body and options. + + :param searchBody: The body containing search parameters such as current page, resultsPerPage, + searchTerm, sortField, sortDirection, filters, etc. + :param options: Additional options for filtering by organizations, etc. + :return: The Elasticsearch query. + """ + # Pagination + current_page = searchBody.get("current", 1) + results_per_page = searchBody.get("resultsPerPage", 20) + + # Search term + search_term = searchBody.get("searchTerm", "") + + # Sorting + sort_field = searchBody.get("sortField", "createdAt") + sort_direction = searchBody.get("sortDirection", "desc") + + # Filters + filters = searchBody.get("filters", []) + + # Organization filtering + organization_ids = options.get("organizationIds", []) + + # Elasticsearch query + query = { + "from": (current_page - 1) * results_per_page, + "size": results_per_page, + "sort": [{sort_field: {"order": sort_direction}}], + "query": {"bool": {"must": [{"match": {"_all": search_term}}], "filter": []}}, + } + + # Apply filters + for filter_item in filters: + field = filter_item.get("field") + values = filter_item.get("values", []) + filter_type = filter_item.get("type", "any") + + if filter_type == "any": + query["query"]["bool"]["filter"].append({"terms": {field: values}}) + elif filter_type == "range": + query["query"]["bool"]["filter"].append({"range": {field: values}}) + + # Apply organization filters + if organization_ids: + query["query"]["bool"]["filter"].append( + {"terms": {"organization.Id": organization_ids}} + ) + + return query diff --git a/backend/src/xfd_django/xfd_api/tests/test_search.py b/backend/src/xfd_django/xfd_api/tests/test_search.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 54a9a887..09996fc0 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -36,6 +36,7 @@ from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import export_domains, get_domain_by_id, search_domains from .api_methods.organization import get_organizations, read_orgs +from .api_methods.search import SearchBody, export, search from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user @@ -139,16 +140,6 @@ async def list_assessments(): return list(assessments) -@api_router.post("/search") -async def search(): - pass - - -@api_router.post("/search/export") -async def export_search(): - pass - - @api_router.get( "/cpes/{cpe_id}", # dependencies=[Depends(get_current_active_user)], @@ -560,3 +551,25 @@ async def invoke_scheduler(current_user: User = Depends(get_current_active_user) """Manually invoke the scan scheduler.""" response = await scan.invoke_scheduler(current_user) return response + + +@api_router.post("/search") +async def search_endpoint(request: Request): + try: + body = await request.json() + search_body = SearchBody(**body) # Parse request body into SearchBody + result = search(search_body, request) + return result + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@api_router.post("/search/export") +async def export_endpoint(request: Request): + try: + body = await request.json() + search_body = SearchBody(**body) # Parse request body into SearchBody + result = export(search_body, request) + return result + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) From f518d6e7e293f24086abf6d1ba1ab39fd14fb1f7 Mon Sep 17 00:00:00 2001 From: nickviola Date: Thu, 17 Oct 2024 15:03:52 -0500 Subject: [PATCH 063/314] Update search endpoint format and response --- .../xfd_django/xfd_api/api_methods/search.py | 27 ++++++++++++------- backend/src/xfd_django/xfd_api/views.py | 22 ++++++++++----- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/search.py b/backend/src/xfd_django/xfd_api/api_methods/search.py index e7eadaf7..305b9dd1 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/search.py @@ -2,7 +2,8 @@ # Standard Python Libraries import csv from io import StringIO -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional +from uuid import UUID # Third-Party Libraries import boto3 @@ -17,16 +18,22 @@ from xfd_api.helpers.elastic_search import build_request +class Filter(BaseModel): + field: str + values: List[str] + type: str + + class SearchBody(BaseModel): current: int - resultsPerPage: int - searchTerm: str - sortDirection: str - sortField: str - filters: List[Dict[str, Any]] - organizationIds: List[str] = [] - organizationId: str = "" - tagId: str = "" + results_per_page: int + search_term: str + sort_direction: str + sort_field: str + filters: List[Filter] + organization_ids: Optional[List[UUID]] = None + organization_id: Optional[UUID] = None + tag_id: Optional[UUID] = None def get_options(search_body: SearchBody, event) -> Dict[str, Any]: @@ -112,7 +119,7 @@ def export(search_body: SearchBody, event) -> Dict[str, Any]: writer.writeheader() writer.writerows(results) - # Save to S3 + # Save to S3 # TODO: Replace with heler logic s3 = boto3.client("s3") bucket_name = "your-bucket-name" csv_key = "domains.csv" diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 09996fc0..9d0980cc 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -554,14 +554,24 @@ async def invoke_scheduler(current_user: User = Depends(get_current_active_user) @api_router.post("/search") -async def search_endpoint(request: Request): +async def search_endpoint(request: Request, body: SearchBody): try: - body = await request.json() - search_body = SearchBody(**body) # Parse request body into SearchBody - result = search(search_body, request) - return result + # Example of parsing UUIDs correctly + organization_id = body.organization_id + tag_id = body.tag_id + + # Search logic + # Using the parsed and validated UUIDs in the search + results = { + "current": body.current, + "organization_id": str(organization_id) if organization_id else None, + "tag_id": str(tag_id) if tag_id else None, + } + + return results + except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=400, detail=str(e)) @api_router.post("/search/export") From b0a0306a5ca04e4c0ef8d319d34ff611b23ece4f Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Mon, 21 Oct 2024 11:38:16 -0500 Subject: [PATCH 064/314] WIP in domain search. --- .../xfd_django/xfd_api/api_methods/domain.py | 65 ++------ .../xfd_api/api_methods/vulnerability.py | 37 +++++ .../xfd_api/helpers/filter_helpers.py | 139 ++++++++++++++++++ .../xfd_api/schema_models/domain.py | 4 +- .../xfd_api/schema_models/vulnerability.py | 8 +- backend/src/xfd_django/xfd_api/views.py | 11 +- 6 files changed, 200 insertions(+), 64 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/domain.py b/backend/src/xfd_django/xfd_api/api_methods/domain.py index 5bf3f4ba..8e919409 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/domain.py +++ b/backend/src/xfd_django/xfd_api/api_methods/domain.py @@ -10,58 +10,9 @@ from django.core.paginator import Paginator from fastapi import HTTPException -from ..helpers.filter_helpers import sort_direction -from ..models import Domain, Organization, Service, Vulnerability -from ..schema_models.domain import DomainFilters, DomainSearch - - -def filter_domains(domains, domain_filters: DomainFilters): - """ - Filter domains - Arguments: - domains: A list of all domains, sorted - domain_filters: Value to filter the domains table by - Returns: - object: a list of Domain objects - """ - try: - if domain_filters.port is not None: - services_by_port = Service.objects.values("domainId").filter( - port=domain_filters.port - ) - domains = domains.filter(id__in=services_by_port) - if domain_filters.service != "": - service_by_id = Service.objects.values("domainId").get( - id=domain_filters.service - ) - domains = domains.filter(id=service_by_id["domainId"]) - if domain_filters.reverseName != "": - domains_by_reverse_name = Domain.objects.values("id").filter( - reverseName=domain_filters.reverseName - ) - domains = domains.filter(id__in=domains_by_reverse_name) - if domain_filters.ip != "": - domains_by_ip = Domain.objects.values("id").filter(ip=domain_filters.ip) - domains = domains.filter(id__in=domains_by_ip) - if domain_filters.organization != "": - domains_by_org = Domain.objects.values("id").filter( - organizationId_id=domain_filters.organization - ) - domains = domains.filter(id__in=domains_by_org) - if domain_filters.organizationName != "": - organization_by_name = Organization.objects.values("id").filter( - name=domain_filters.organizationName - ) - domains = domains.filter(organizationId_id__in=organization_by_name) - if domain_filters.vulnerabilities != "": - vulnerabilities_by_id = Vulnerability.objects.values("domainId").filter( - id=domain_filters.vulnerabilities - ) - domains = domains.filter(id__in=vulnerabilities_by_id) - - return domains - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) +from ..helpers.filter_helpers import filter_domains, sort_direction +from ..models import Domain +from ..schema_models.domain import DomainSearch def get_domain_by_id(domain_id: str): @@ -89,10 +40,14 @@ def search_domains(domain_search: DomainSearch): """ try: # Fetch all domains in list + if domain_search.order is not None: + domains = Domain.objects.all().order_by( + sort_direction(domain_search.sort, domain_search.order) + ) + else: + # Default sort order behavior + domains = Domain.objects.all() - domains = Domain.objects.all().order_by( - sort_direction(domain_search.sort, domain_search.order) - ) if domain_search.filters is not None: results = filter_domains(domains, domain_search.filters) paginator = Paginator(results, domain_search.pageSize) diff --git a/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py b/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py index 1d9713a9..12ed0ff9 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py +++ b/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py @@ -4,10 +4,13 @@ """ # Third-Party Libraries +from django.core.paginator import Paginator from fastapi import HTTPException +from ..helpers.filter_helpers import filter_vulnerabilities, sort_direction from ..models import Vulnerability from ..schema_models.vulnerability import Vulnerability as VulnerabilitySchema +from ..schema_models.vulnerability import VulnerabilityFilters, VulnerabilitySearch def get_vulnerability_by_id(vuln_id): @@ -37,3 +40,37 @@ def update_vulnerability(vuln_id, data: VulnerabilitySchema): return vulnerability except Exception as e: raise HTTPException(status_code=500, detail=str(e)) + + +def search_vulnerabilities(vulnerability_search: VulnerabilitySearch): + """ + List vulnerabilities by search filter + Arguments: + vulnerability_search: A VulnerabilitySearch object to filter by. + Returns: + object: A paginated list of Vulnerability objects + """ + try: + # Fetch all domains in list + if vulnerability_search.order is not None: + vulnerabilities = Vulnerability.objects.all().order_by( + sort_direction(vulnerability_search.sort, vulnerability_search.order) + ) + else: + # Default sort order behavior + vulnerabilities = Vulnerability.objects.all() + + if vulnerability_search.filters is not None: + print(f"filters: {vulnerability_search.filters}") + results = filter_vulnerabilities( + vulnerabilities, vulnerability_search.filters + ) + if vulnerability_search.groupBy is not None: + results = results.values(vulnerability_search.groupBy).order_by() + + paginator = Paginator(results, vulnerability_search.pageSize) + return paginator.get_page(vulnerability_search.page) + else: + raise ValueError("DomainFilters cannot be NoneType") + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py index e592a0e7..ffc0315b 100644 --- a/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py +++ b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py @@ -1,6 +1,10 @@ # Third-Party Libraries from fastapi import HTTPException +from ..models import Domain, Organization, Service, Vulnerability +from ..schema_models.domain import Domain, DomainFilters +from ..schema_models.vulnerability import Vulnerability, VulnerabilityFilters + def sort_direction(sort, order): """ @@ -19,3 +23,138 @@ def sort_direction(sort, order): raise ValueError except ValueError as e: raise HTTPException(status_code=500, detail="Invalid sort direction supplied") + + +def filter_domains(domains, domain_filters: DomainFilters): + """ + Filter domains + Arguments: + domains: A list of all domains, sorted + domain_filters: Value to filter the domains table by + Returns: + object: a list of Domain objects + """ + try: + print("DEBUG") + if domain_filters.port: + print(f"port: {domain_filters.port}") + services_by_port = Service.objects.values("domainId").filter( + port=domain_filters.port + ) + if services_by_port.exists(): + domains = domains.filter(id__in=services_by_port) + if domain_filters.service: + print(f"service: {domain_filters.service}") + service_by_id = Service.objects.values("domainId").filter( + id__in=domain_filters.service + ) + if service_by_id.exists(): + domains = domains.filter(id=service_by_id) + if domain_filters.reverseName: + print(f"reverseName: {domain_filters.reverseName}") + domains_by_reverse_name = Domain.objects.values("id").filter( + reverseName=domain_filters.reverseName + ) + if domains_by_reverse_name.exists(): + domains = domains.filter(id__in=domains_by_reverse_name) + if domain_filters.ip: + print(f"ip: {domain_filters.ip}") + domains_by_ip = Domain.objects.values("id").filter(ip=domain_filters.ip) + if domains_by_ip.exists(): + domains = domains.filter(id__in=domains_by_ip) + if domain_filters.organization: + print(f"organization: {domain_filters.organization}") + domains_by_org = Domain.objects.values("id").filter( + organizationId_id=domain_filters.organization + ) + if domains_by_org.exists(): + domains = domains.filter(id__in=domains_by_org) + if domain_filters.organizationName: + print(f"organizationName: {domain_filters.organizationName}") + organization_by_name = Organization.objects.values("id").filter( + name=domain_filters.organizationName + ) + if organization_by_name.exists(): + domains = domains.filter(organizationId_id__in=organization_by_name) + if domain_filters.vulnerabilities: + print(f"vulnerabilities: {domain_filters.vulnerabilities}") + vulnerabilities_by_id = Vulnerability.objects.values("domainId").filter( + id=domain_filters.vulnerabilities + ) + if vulnerabilities_by_id.exists(): + domains = domains.filter(id__in=vulnerabilities_by_id) + + return domains + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def filter_vulnerabilities( + vulnerabilities, vulnerability_filters: VulnerabilityFilters +): + """ + Filter vulnerabilitie + Arguments: + vulnerabilities: A list of all vulnerabilities, sorted + vulnerability_filters: Value to filter the vulnberabilities table by + Returns: + object: a list of Vulnerability objects + """ + try: + if vulnerability_filters.id: + print(f"id: {vulnerability_filters.id}") + vulnerability_by_id = Vulnerability.objects.values("id").get( + id=vulnerability_filters.id + ) + vulnerabilities = vulnerabilities.filter(id=vulnerability_by_id("id")) + if vulnerability_filters.title: + print(f"title: {vulnerability_filters.title}") + vulnerabilities_by_title = Vulnerability.objects.values("id").filter( + title=vulnerability_filters.title + ) + vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_title) + if vulnerability_filters.domain: + print(f"domain: {vulnerability_filters.domain}") + vulnerabilities_by_domain = Vulnerability.objects.values("id").filters( + domainId=vulnerability_filters.domain + ) + vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_domain) + if vulnerability_filters.severity: + print(f"severity: {vulnerability_filters.severity}") + vulnerabilities_by_severity = Vulnerability.objects.values("id").filter( + severity=vulnerability_filters.severity + ) + vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_severity) + if vulnerability_filters.cpe: + print(f"cpe: {vulnerability_filters.cpe}") + vulnerabilities_by_cpe = Vulnerability.objects.values("id").filter( + cpe=vulnerability_filters.cpe + ) + vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_cpe) + if vulnerability_filters.state: + print(f"state: {vulnerability_filters.state}") + vulnerabilities_by_state = Vulnerability.objects.values("id").filter( + state=vulnerability_filters.state + ) + vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_state) + if vulnerability_filters.organization: + print(f"organization: {vulnerability_filters.organization}") + domains = Domain.objects.all() + domains_by_organization = Domain.objects.values("id").filter( + organizationId_id=vulnerability_filters.organization + ) + domains = domains.filter(id__in=domains_by_organization) + vulnerabilities_by_domain = Vulnerability.objects.values("id").filter( + id__in=domains + ) + vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_domain) + if vulnerability_filters.isKev: + print(f"isKev: {vulnerability_filters.isKev}") + vulnerabilities_by_is_kev = Vulnerability.objects.values("id").filter( + isKev=vulnerability_filters.isKev + ) + vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_is_kev) + print(f"vulnerabilities: {vulnerabilities}") + return vulnerabilities + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/domain.py b/backend/src/xfd_django/xfd_api/schema_models/domain.py index d509de03..21a75c01 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/domain.py +++ b/backend/src/xfd_django/xfd_api/schema_models/domain.py @@ -59,7 +59,7 @@ class DomainSearch(BaseModel): """DomainSearch schema.""" page: int = 1 - sort: str - order: str + sort: Optional[str] = "ASC" + order: Optional[str] = None filters: Optional[DomainFilters] pageSize: Optional[int] = None diff --git a/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py b/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py index b217d257..979fb8bd 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py +++ b/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py @@ -40,16 +40,16 @@ class Vulnerability(BaseModel): class VulnerabilityFilters(BaseModel): """VulnerabilityFilters schema.""" - id: Optional[UUID] + id: Optional[str] = None title: Optional[str] domain: Optional[str] severity: Optional[str] cpe: Optional[str] state: Optional[str] substate: Optional[str] - organization: Optional[UUID] - tag: Optional[UUID] - isKev: Optional[bool] + organization: Optional[str] = None + tag: Optional[str] + isKev: Optional[bool] = None class VulnerabilitySearch(BaseModel): diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 181e4110..b805cf57 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -30,7 +30,11 @@ from .api_methods.domain import export_domains, get_domain_by_id, search_domains from .api_methods.organization import get_organizations, read_orgs from .api_methods.user import get_users -from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability +from .api_methods.vulnerability import ( + get_vulnerability_by_id, + search_vulnerabilities, + update_vulnerability, +) from .auth import get_current_active_user from .models import Assessment, User from .schema_models import scan as scanSchema @@ -42,6 +46,7 @@ from .schema_models.organization import Organization as OrganizationSchema from .schema_models.user import User as UserSchema from .schema_models.vulnerability import Vulnerability as VulnerabilitySchema +from .schema_models.vulnerability import VulnerabilitySearch # Define API router api_router = APIRouter() @@ -224,9 +229,9 @@ async def call_get_domain_by_id(domain_id: str): @api_router.post("/vulnerabilities/search") -async def search_vulnerabilities(): +async def call_search_vulnerabilities(vulnerability_search: VulnerabilitySearch): try: - pass + return search_vulnerabilities(vulnerability_search) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) From 0222f9707bf438d72b460f66cac87dede3063997 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Tue, 22 Oct 2024 09:59:18 -0500 Subject: [PATCH 065/314] Add Accept Terms Endpoint. Add accept_terms function to api_methods/user.py Add /users/acceptTerms endpoint to views.py Add UpdateUser class to schema_models/user.py Add can_access_user function to auth.py --- .../xfd_django/xfd_api/api_methods/user.py | 22 +++++++++++++++++++ backend/src/xfd_django/xfd_api/auth.py | 7 ++++++ .../xfd_django/xfd_api/schema_models/user.py | 8 +++++++ backend/src/xfd_django/xfd_api/views.py | 18 ++++++++++++++- 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/user.py b/backend/src/xfd_django/xfd_api/api_methods/user.py index 55b3c3b2..da2ea6ff 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/user.py +++ b/backend/src/xfd_django/xfd_api/api_methods/user.py @@ -3,6 +3,7 @@ """ # Standard Python Libraries +from datetime import datetime from typing import List, Optional # Third-Party Libraries @@ -12,6 +13,27 @@ from ..schema_models.user import User as UserSchema +async def accept_terms(current_user: User, version: str): + """ + Accept the latest terms of service. + + Args: + current_user (User): The current authenticated user. + version (str): The version of the terms of service. + + Returns: + User: The updated user. + """ + if not current_user: + raise HTTPException(status_code=401, detail="User not authenticated.") + + current_user.dateAcceptedTerms = datetime.now() + current_user.acceptedTermsVersion = version + current_user.save() + + return UserSchema.from_orm(current_user) + + def get_users(regionId): """ Retrieve a list of users based on optional filter parameters. diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 8981e746..0ffc8105 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -5,6 +5,7 @@ import hashlib from hashlib import sha256 import os +from typing import Optional from urllib.parse import urlencode import uuid @@ -409,6 +410,12 @@ async def get_jwt_from_code(auth_code: str): # return user +def can_access_user(current_user, user_id: Optional[str]) -> bool: + return ( + user_id and (current_user.id == user_id) or is_global_write_admin(current_user) + ) + + def is_global_write_admin(current_user) -> bool: """Check if the user has global write admin permissions.""" return current_user and current_user.userType == "globalAdmin" diff --git a/backend/src/xfd_django/xfd_api/schema_models/user.py b/backend/src/xfd_django/xfd_api/schema_models/user.py index 40465fe1..25795a54 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/user.py +++ b/backend/src/xfd_django/xfd_api/schema_models/user.py @@ -58,3 +58,11 @@ def model_dump(self, **kwargs): class Config: from_attributes = True + + +class UpdateUser(BaseModel): + firstName: str + lastName: str + email: str + userType: str + state: Optional[str] diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 85a2fd88..6b55760c 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -36,7 +36,7 @@ from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import export_domains, get_domain_by_id, search_domains from .api_methods.organization import get_organizations, read_orgs -from .api_methods.user import delete_user, get_users +from .api_methods.user import accept_terms, delete_user, get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user from .login_gov import callback, login @@ -318,6 +318,22 @@ async def callback_route(request: Request): # GET Current User +@api_router.post("/users/acceptTerms", tags=["Users"]) +async def call_accept_terms( + version: str, current_user: User = Depends(get_current_active_user) +): + """ + Accept the latest terms of service. + + Args: + version (str): The version of the terms of service. + + Returns: + User: The updated user. + """ + return accept_terms(current_user, version) + + @api_router.get("/users/me", tags=["Users"]) async def read_users_me(current_user: User = Depends(get_current_active_user)): return current_user From 40b2c3034dfc91c43a03912465dd74507508bdda Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Tue, 22 Oct 2024 10:33:44 -0500 Subject: [PATCH 066/314] Modify and reorder delete user endpoint; add todos for authentication and permissions. --- .../xfd_django/xfd_api/api_methods/user.py | 34 ++++++++++--------- backend/src/xfd_django/xfd_api/views.py | 33 ++++++++++-------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/user.py b/backend/src/xfd_django/xfd_api/api_methods/user.py index da2ea6ff..ed9eb5c0 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/user.py +++ b/backend/src/xfd_django/xfd_api/api_methods/user.py @@ -7,7 +7,8 @@ from typing import List, Optional # Third-Party Libraries -from fastapi import HTTPException, Query +from fastapi import HTTPException +from fastapi.responses import JSONResponse from ..models import User from ..schema_models.user import User as UserSchema @@ -34,42 +35,43 @@ async def accept_terms(current_user: User, version: str): return UserSchema.from_orm(current_user) -def get_users(regionId): +# TODO: Add user context and permissions +def delete_user(user_id: str): """ - Retrieve a list of users based on optional filter parameters. + Delete a user by ID. Args: - regionId : Region ID to filter users by. + user_id : The ID of the user to delete. Raises: - HTTPException: If the user is not authorized or no users are found. + HTTPException: If the user is not authorized or the user is not found. Returns: - List[User]: A list of users matching the filter criteria. + JSONResponse: The result of the deletion. """ try: - users = User.objects.filter(regionId=regionId).prefetch_related("roles") - return [UserSchema.from_orm(user) for user in users] + user = User.objects.get(id=user_id) + result = user.delete() + return JSONResponse(status_code=200, content={"result": result}) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) -def delete_user(user_id: str): +def get_users(regionId): """ - Delete a user by ID. + Retrieve a list of users based on optional filter parameters. Args: - user_id : The ID of the user to delete. + regionId : Region ID to filter users by. Raises: - HTTPException: If the user is not authorized or the user is not found. + HTTPException: If the user is not authorized or no users are found. Returns: - User: The user that was deleted. + List[User]: A list of users matching the filter criteria. """ try: - user = User.objects.get(id=user_id) - user.delete() - return UserSchema.from_orm(user) + users = User.objects.filter(regionId=regionId).prefetch_related("roles") + return [UserSchema.from_orm(user) for user in users] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 6b55760c..bf94746c 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -334,6 +334,24 @@ async def call_accept_terms( return accept_terms(current_user, version) +# TODO: Add authentication and permissions +@api_router.delete("/users/{userId}", tags=["Users"]) +async def call_delete_user(userId: str): + """ + Delete a user by ID. + + Args: + userId : The ID of the user to delete. + + Raises: + HTTPException: If the user is not authorized or the user is not found. + + Returns: + JSONResponse: The result of the deletion. + """ + return delete_user(userId) + + @api_router.get("/users/me", tags=["Users"]) async def read_users_me(current_user: User = Depends(get_current_active_user)): return current_user @@ -361,21 +379,6 @@ async def call_get_users(regionId): return get_users(regionId) -@api_router.delete("/users/{userId}", tags=["Users"]) -async def call_delete_user(userId: str): - """ - call delete_user() - Args: - userId: UUID of the user to delete. - - Returns: - User: The user that was deleted. - - """ - - return delete_user(userId) - - ###################### # API-Keys ###################### From bfbc798bdb574723d3626bbe9b1f67e256d72cff Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Tue, 22 Oct 2024 12:31:43 -0400 Subject: [PATCH 067/314] Updated saved search response model and list all to drill down on filters; Updated saved search model to remove unnecessary fields --- .../xfd_api/api_methods/saved_search.py | 17 ++++++--- backend/src/xfd_django/xfd_api/models.py | 2 -- .../xfd_api/schema_models/saved_search.py | 36 ++++++------------- backend/src/xfd_django/xfd_api/views.py | 14 +------- 4 files changed, 25 insertions(+), 44 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py index 16294176..82c09537 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -11,6 +11,7 @@ from fastapi import HTTPException from ..models import SavedSearch +from ..schema_models.saved_search import SavedSearchFilters def create_saved_search(request): @@ -18,8 +19,8 @@ def create_saved_search(request): search = SavedSearch.objects.create( name=data["name"], count=data["count"], - sort_direction=data["sortDirection"], - sort_field=data["sortField"], + sort_direction="asc", + sort_field="name", search_term=data["searchTerm"], search_path=data["searchPath"], filters=data["filters"], @@ -36,7 +37,15 @@ def list_saved_searches(): saved_searches = SavedSearch.objects.all() saved_search_list = [] for search in saved_searches: - print(str(saved_search_list)) + filters_without_type = [] + for filter in search.filters: + # Exclude the "type" field by constructing a new dictionary without it + filter_without_type = { + "field": filter["field"], # Keep field + "values": filter["values"], # Keep values + } + filters_without_type.append(filter_without_type) + response = { "id": str(search.id), "created_at": search.createdAt, @@ -46,7 +55,7 @@ def list_saved_searches(): "sort_direction": search.sortDirection, "sort_field": search.sortField, "count": search.count, - "filters": search.filters, + "filters": filters_without_type, "search_path": search.searchPath, "createdBy_id": search.createdById.id, } diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 9642d31f..655b8607 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -460,8 +460,6 @@ class SavedSearch(models.Model): count = models.IntegerField() filters = models.JSONField() searchPath = models.CharField(db_column="searchPath") - createVulnerabilities = models.BooleanField(db_column="createVulnerabilities") - vulnerabilityTemplate = models.JSONField(db_column="vulnerabilityTemplate") createdById = models.ForeignKey( "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True ) diff --git a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py index 994b94f8..db7ab162 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py +++ b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py @@ -8,6 +8,13 @@ from pydantic import BaseModel +class SavedSearchFilters(BaseModel): + """SavedSearchFilters schema.""" + + field: str + values: List[Any] + + class SavedSearch(BaseModel): """SavedSearch schema.""" @@ -15,31 +22,10 @@ class SavedSearch(BaseModel): created_at: datetime updated_at: datetime name: str - search_term: str + search_term: str = "" sort_direction: str sort_field: str count: int - filters: List[Dict[str, Any]] - - -class SavedSearchFilters(BaseModel): - """SavedSearchFilters schema.""" - - id: Optional[UUID] - name: Optional[str] - sort_direction: Optional[str] - sort_field: Optional[str] - search_term: Optional[str] - search_path: Optional[str] - created_by: Optional[UUID] - - -class SavedSearchSearch(BaseModel): - """SavedSearchSearch schema.""" - - page: int - sort: Optional[str] - order: str - filters: Optional[SavedSearchFilters] - pageSize: Optional[int] - groupBy: Optional[str] + filters: List[SavedSearchFilters] + search_path: str + createdBy_id: UUID diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 0a6be309..3c51136a 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -397,23 +397,11 @@ async def call_get_organizations(regionId): # TODO: Typescript endpoints for reference, not implemented in FastAPI. # Remove after implementation -# authenticatedRoute.get('/saved-searches', handlerToExpress(savedSearches.list)); # authenticatedRoute.post( # '/saved-searches', # handlerToExpress(savedSearches.create) # ); -# authenticatedRoute.get( -# '/saved-searches/:searchId', -# handlerToExpress(savedSearches.get) -# ); -# authenticatedRoute.put( -# '/saved-searches/:searchId', -# handlerToExpress(savedSearches.update) -# ); -# authenticatedRoute.delete( -# '/saved-searches/:searchId', -# handlerToExpress(savedSearches.del) -# ); + # ======================================== # Saved Search Endpoints From 4bc644ee46da1f13b22a6b3962331980e03adbd9 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 23 Oct 2024 09:59:18 -0400 Subject: [PATCH 068/314] Add organization endpoints --- .../xfd_api/api_methods/organization.py | 950 ++++++++++++++++-- backend/src/xfd_django/xfd_api/auth.py | 49 +- .../xfd_api/helpers/regionStateMap.py | 62 ++ backend/src/xfd_django/xfd_api/models.py | 272 +++-- .../xfd_api/schema_models/organization.py | 184 +++- .../xfd_django/xfd_api/schema_models/scan.py | 2 +- backend/src/xfd_django/xfd_api/views.py | 263 ++++- 7 files changed, 1564 insertions(+), 218 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/helpers/regionStateMap.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/organization.py b/backend/src/xfd_django/xfd_api/api_methods/organization.py index 2701a968..1e14213f 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/organization.py +++ b/backend/src/xfd_django/xfd_api/api_methods/organization.py @@ -1,84 +1,910 @@ -""" -Organizations API. +"""API methods to support Organization endpoints.""" -""" # Standard Python Libraries -from typing import List, Optional +from typing import List +import uuid # Third-Party Libraries -from fastapi import HTTPException, Query +from django.db.models import Q +from fastapi import HTTPException -from ..models import Organization +from ..auth import ( + get_org_memberships, + is_global_view_admin, + is_global_write_admin, + is_org_admin, + is_regional_admin, + is_regional_admin_for_organization, +) +from ..helpers.regionStateMap import REGION_STATE_MAP +from ..models import Organization, OrganizationTag, Role, Scan, ScanTask, User +from ..schema_models import organization as organization_schemas -def read_orgs(): - """ - Call API endpoint to get all organizations. - Returns: - list: A list of all organizations. - """ +def is_valid_uuid(val: str) -> bool: + """Check if the given string is a valid UUID.""" try: - organizations = Organization.objects.all() - return [ + uuid_obj = uuid.UUID(val, version=4) + except ValueError: + return False + return str(uuid_obj) == val + + +def list_organizations(current_user): + """List organizations that the user is a member of or has access to.""" + try: + # Check if user is GlobalViewAdmin or has memberships + if not is_global_view_admin(current_user) and not get_org_memberships( + current_user + ): + return [] + + # Define filter for organizations based on admin status + org_filter = {} + if not is_global_view_admin(current_user): + org_filter["id__in"] = get_org_memberships(current_user) + org_filter["parent"] = None + + # Fetch organizations with related userRoles and tags + organizations = ( + Organization.objects.prefetch_related("tags", "userRoles") + .filter(**org_filter) + .order_by("name") + ) + + # Serialize organizations using list comprehension + organization_list = [ { - "id": organization.id, - "name": organization.name, - "acronym": organization.acronym, - "rootDomains": organization.rootDomains, - "ipBlocks": organization.ipBlocks, - "isPassive": organization.isPassive, - "country": organization.country, - "state": organization.state, - "regionId": organization.regionId, - "stateFips": organization.stateFips, - "stateName": organization.stateName, - "county": organization.county, - "countyFips": organization.countyFips, - "type": organization.type, - "parentId": organization.parentId.id if organization.parentId else None, - "createdById": organization.createdById.id - if organization.createdById - else None, - "createdAt": organization.createdAt, - "updatedAt": organization.updatedAt, + "id": str(org.id), + "createdAt": org.createdAt.isoformat(), + "updatedAt": org.updatedAt.isoformat(), + "acronym": org.acronym, + "name": org.name, + "rootDomains": org.rootDomains, + "ipBlocks": org.ipBlocks, + "isPassive": org.isPassive, + "pendingDomains": org.pendingDomains, + "country": org.country, + "state": org.state, + "regionId": org.regionId, + "stateFips": org.stateFips, + "stateName": org.stateName, + "county": org.county, + "countyFips": org.countyFips, + "type": org.type, + "userRoles": [ + {"id": str(role.id), "role": role.role, "approved": role.approved} + for role in org.userRoles.all() + ], + "tags": [ + { + "id": str(tag.id), + "createdAt": tag.createdAt.isoformat(), + "updatedAt": tag.updatedAt.isoformat(), + "name": tag.name, + } + for tag in org.tags.all() + ], } - for organization in organizations + for org in organizations ] + + return organization_list + + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + + +def get_tags(current_user): + """Fetches all possible organization tags.""" + try: + # Check if user is a global admin + if not is_global_view_admin(current_user): + return [] + + # Fetch organization tags + tags = OrganizationTag.objects.all().values("id", "name") + + # Return the list of tags + return list(tags) + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def get_organization(organization_id, current_user): + """Get information about a particular organization.""" + try: + # Authorization checks + if not ( + is_org_admin(current_user, organization_id) + or is_global_write_admin(current_user) + or is_regional_admin_for_organization(current_user, organization_id) + ): + raise HTTPException(status_code=403, detail="Unauthorized") + + # Fetch organization with relations + organization = ( + Organization.objects.select_related("parent") + .prefetch_related("userRoles__user", "granularScans", "tags", "children") + .filter(id=organization_id) + .first() + ) + + if not organization: + raise HTTPException(status_code=404, detail="Organization not found") + + # Fetch scan tasks related to the organization, limited to 10 most recent + scan_tasks = ( + ScanTask.objects.filter(organizations__id=organization_id) + .select_related("scan") + .order_by("-createdAt")[:10] + ) + + # Serialize organization details along with scan tasks + org_data = { + "id": str(organization.id), + "createdAt": organization.createdAt.isoformat(), + "updatedAt": organization.updatedAt.isoformat(), + "acronym": organization.acronym, + "name": organization.name, + "rootDomains": organization.rootDomains, + "ipBlocks": organization.ipBlocks, + "isPassive": organization.isPassive, + "pendingDomains": organization.pendingDomains, + "country": organization.country, + "state": organization.state, + "regionId": organization.regionId, + "stateFips": organization.stateFips, + "stateName": organization.stateName, + "county": organization.county, + "countyFips": organization.countyFips, + "type": organization.type, + "userRoles": [ + { + "id": str(role.id), + "role": role.role, + "approved": role.approved, + "user": { + "id": str(role.user.id), + "email": role.user.email, + "firstName": role.user.firstName, + "lastName": role.user.lastName, + }, + } + for role in organization.userRoles.all() + ], + "granularScans": [ + { + "id": str(scan.id), + "createdAt": scan.createdAt.isoformat(), + "updatedAt": scan.updatedAt.isoformat(), + "name": scan.name, + "arguments": scan.arguments, + "frequency": scan.frequency, + "lastRun": scan.lastRun.isoformat() if scan.lastRun else None, + "isGranular": scan.isGranular, + "isUserModifiable": scan.isUserModifiable, + "isSingleScan": scan.isSingleScan, + "manualRunPending": scan.manualRunPending, + } + for scan in organization.granularScans.all() + ], + "tags": [ + { + "id": str(tag.id), + "createdAt": tag.createdAt.isoformat(), + "updatedAt": tag.updatedAt.isoformat(), + "name": tag.name, + } + for tag in organization.tags.all() + ], + "parent": { + "id": str(organization.parent.id), + "name": organization.parent.name, + } + if organization.parent + else None, + "children": [ + {"id": str(child.id), "name": child.name} + for child in organization.children.all() + ], + "scanTasks": [ + { + "id": str(task.id), + "createdAt": task.createdAt.isoformat(), + "scan": {"id": str(task.scan.id), "name": task.scan.name} + if task.scan + else None, + } + for task in scan_tasks + ], + } + + return org_data + + except Exception as e: + print(f"An error occurred: {e}") + raise HTTPException(status_code=500, detail="An unexpected error occurred") + + +def get_by_state(state, current_user): + """List organizations with specific state.""" + # Check if the current user is a regional admin + if not is_regional_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized") + + # Fetch organizations based on the provided state + organizations = Organization.objects.filter(state=state).values( + "id", + "createdAt", + "updatedAt", + "acronym", + "name", + "rootDomains", + "ipBlocks", + "isPassive", + "pendingDomains", + "country", + "state", + "regionId", + "stateFips", + "stateName", + "county", + "countyFips", + "type", + ) + + if not organizations: + raise HTTPException( + status_code=404, detail="No organizations found for the given state" + ) + + # Return the serialized list of organizations + return list(organizations) + + +def get_by_region(regionId, current_user): + """List organizations with specific regionId.""" + # Check if the current user is a regional admin + if not is_regional_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized") + + # Fetch organizations based on the provided state + organizations = Organization.objects.filter(regionId=regionId).values( + "id", + "createdAt", + "updatedAt", + "acronym", + "name", + "rootDomains", + "ipBlocks", + "isPassive", + "pendingDomains", + "country", + "state", + "regionId", + "stateFips", + "stateName", + "county", + "countyFips", + "type", + ) + + if not organizations: + raise HTTPException( + status_code=404, detail="No organizations found for the given region" + ) + + # Return the serialized list of organizations + return list(organizations) + + +def get_all_regions(current_user): + """Get all regions.""" + try: + # Check if user is GlobalViewAdmin or has memberships + if not is_global_view_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized") + + # Fetch distinct regionId values + regions = ( + Organization.objects.exclude(regionId__isnull=True) + .values("regionId") + .distinct() + ) + + # Convert to a list and return the regions + return list(regions) + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def find_or_create_tags( + tags: List[organization_schemas.TagSchema], +) -> List[OrganizationTag]: + """Find or create organization tags.""" + final_tags = [] + + for tag_data in tags: + tag_name = tag_data.name + + # Check if a tag with the given name exists + existing_tag = OrganizationTag.objects.filter(name=tag_name).first() + if existing_tag: + final_tags.append(existing_tag) + else: + # If not found, create a new tag + created_tag = OrganizationTag.objects.create(name=tag_name) + final_tags.append(created_tag) + + return final_tags + + +def create_organization(organization_data, current_user): + """Create a new organization.""" + try: + # Check if the user is a GlobalWriteAdmin + if not is_global_write_admin(current_user): + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + + # Prepare the organization data for creation + organization_data_dict = organization_data.dict( + exclude_unset=True, exclude={"tags", "parent"} + ) + organization_data_dict["createdBy"] = current_user + + # Set regionId based on stateName if available + organization_data_dict["regionId"] = REGION_STATE_MAP.get( + organization_data.stateName, None + ) + + # Create the organization object + organization = Organization.objects.create(**organization_data_dict) + + # Link parent organization if provided + if organization_data.parent: + organization.parent_id = organization_data.parent + organization.save() + + # Link tags (using the find_or_create_tags function) + if organization_data.tags: + tags = find_or_create_tags(organization_data.tags) + organization.tags.add(*tags) + + # Return the organization details in response + return { + "id": str(organization.id), + "createdAt": organization.createdAt.isoformat(), + "updatedAt": organization.updatedAt.isoformat(), + "acronym": organization.acronym, + "name": organization.name, + "rootDomains": organization.rootDomains, + "ipBlocks": organization.ipBlocks, + "isPassive": organization.isPassive, + "pendingDomains": organization.pendingDomains, + "country": organization.country, + "state": organization.state, + "regionId": organization.regionId, + "stateFips": organization.stateFips, + "stateName": organization.stateName, + "county": organization.county, + "countyFips": organization.countyFips, + "type": organization.type, + "tags": [ + { + "id": str(tag.id), + "createdAt": tag.createdAt.isoformat(), + "updatedAt": tag.updatedAt.isoformat(), + "name": tag.name, + } + for tag in organization.tags.all() + ], + "parent": { + "id": str(organization.parent.id), + "name": organization.parent.name, + } + if organization.parent + else {}, + } + + except Organization.DoesNotExist: + raise HTTPException(status_code=404, detail="Parent organization not found") + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + + +def upsert_organization(organization_data, current_user): + """Create a new organization or update it if it already exists.""" + try: + # Check if the user is a GlobalWriteAdmin + if not is_global_write_admin(current_user): + raise HTTPException( + status_code=403, detail="Unauthorized access. View logs for details." + ) + + # Prepare the organization data for creation + organization_data_dict = organization_data.dict( + exclude_unset=True, exclude={"tags", "parent"} + ) + organization_data_dict["createdBy"] = current_user + + # Set regionId based on stateName if available + organization_data_dict["regionId"] = REGION_STATE_MAP.get( + organization_data.stateName, None + ) + + # Try to update or create a new organization + organization, created = Organization.objects.update_or_create( + acronym=organization_data.acronym, # Conflict target is the acronym + defaults=organization_data_dict, # Fields to update if organization exists + ) + + # Link parent organization if provided + if organization_data.parent: + organization.parent_id = organization_data.parent + organization.save() + + # Link tags (using the find_or_create_tags function) + if organization_data.tags: + tags = find_or_create_tags(organization_data.tags) + organization.tags.add(*tags) + + # Return the organization details in response + return { + "id": str(organization.id), + "createdAt": organization.createdAt.isoformat(), + "updatedAt": organization.updatedAt.isoformat(), + "acronym": organization.acronym, + "name": organization.name, + "rootDomains": organization.rootDomains, + "ipBlocks": organization.ipBlocks, + "isPassive": organization.isPassive, + "pendingDomains": organization.pendingDomains, + "country": organization.country, + "state": organization.state, + "regionId": organization.regionId, + "stateFips": organization.stateFips, + "stateName": organization.stateName, + "county": organization.county, + "countyFips": organization.countyFips, + "type": organization.type, + "tags": [ + { + "id": str(tag.id), + "createdAt": tag.createdAt.isoformat(), + "updatedAt": tag.updatedAt.isoformat(), + "name": tag.name, + } + for tag in organization.tags.all() + ], + "parent": { + "id": str(organization.parent.id), + "name": organization.parent.name, + } + if organization.parent + else {}, + } + + except Organization.DoesNotExist: + raise HTTPException(status_code=404, detail="Parent organization not found") + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + + +def update_organization(organization_id: str, organization_data, current_user): + """Update an organization by its ID.""" + try: + # Validate the organization ID and ensure it's a valid UUID + if not organization_id or not is_valid_uuid(organization_id): + raise HTTPException(status_code=404, detail="Organization not found") + + # Ensure the current user has permission to update the organization + if not is_org_admin(current_user, organization_id): + raise HTTPException(status_code=403, detail="Unauthorized access.") + + # Fetch the existing organization with userRoles and granularScans relations + try: + organization = Organization.objects.prefetch_related( + "userRoles", "granularScans" + ).get(id=organization_id) + except Organization.DoesNotExist: + raise HTTPException(status_code=404, detail="Organization not found") + + # Manually update each field + organization.name = organization_data.name + organization.acronym = organization_data.acronym + organization.rootDomains = organization_data.rootDomains + organization.ipBlocks = organization_data.ipBlocks + organization.stateName = organization_data.stateName + organization.state = organization_data.state + organization.isPassive = organization_data.isPassive + + # Handle parent organization if provided + if organization_data.parent: + organization.parent_id = organization_data.parent + + # Handle tags (using the find_or_create_tags function) + if organization_data.tags: + tags = find_or_create_tags(organization_data.tags) + organization.tags.set(tags) + + # Save the updated organization object + organization.save() + + # Return the updated organization details in response + return { + "id": str(organization.id), + "createdAt": organization.createdAt.isoformat(), + "updatedAt": organization.updatedAt.isoformat(), + "acronym": organization.acronym, + "name": organization.name, + "rootDomains": organization.rootDomains, + "ipBlocks": organization.ipBlocks, + "isPassive": organization.isPassive, + "pendingDomains": organization.pendingDomains, + "country": organization.country, + "state": organization.state, + "regionId": organization.regionId, + "stateFips": organization.stateFips, + "stateName": organization.stateName, + "county": organization.county, + "countyFips": organization.countyFips, + "type": organization.type, + "tags": [ + { + "id": str(tag.id), + "createdAt": tag.createdAt.isoformat(), + "updatedAt": tag.updatedAt.isoformat(), + "name": tag.name, + } + for tag in organization.tags.all() + ], + "userRoles": [ + { + "id": str(role.id), + "role": role.role, + "approved": role.approved, + "user": { + "id": str(role.user.id), + "email": role.user.email, + "firstName": role.user.firstName, + "lastName": role.user.lastName, + }, + } + for role in organization.userRoles.all() + ], + "granularScans": [ + { + "id": str(scan.id), + "createdAt": scan.createdAt.isoformat(), + "updatedAt": scan.updatedAt.isoformat(), + "name": scan.name, + "arguments": scan.arguments, + "frequency": scan.frequency, + "lastRun": scan.lastRun.isoformat() if scan.lastRun else None, + "isGranular": scan.isGranular, + "isUserModifiable": scan.isUserModifiable, + "isSingleScan": scan.isSingleScan, + "manualRunPending": scan.manualRunPending, + } + for scan in organization.granularScans.all() + ], + } + + except Organization.DoesNotExist: + raise HTTPException(status_code=404, detail="Organization not found") + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + + +def delete_organization(id: str, current_user): + """Delete a particular organization.""" + try: + # Validate the organization ID format (UUID) + if not is_valid_uuid(id): + raise HTTPException(status_code=404, detail="Invalid organization ID.") + + # Check if the current user is a GlobalWriteAdmin + if not is_global_write_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access.") + + # Fetch the organization by ID to ensure it exists + try: + organization = Organization.objects.get(id=id) + except Organization.DoesNotExist: + raise HTTPException(status_code=404, detail="Organization not found.") + + # Delete the organization + organization.delete() + + # Return success response + return { + "status": "success", + "message": f"Organization {id} has been deleted successfully.", + } + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) -def get_organizations( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), -): - """ - List all organizations with query parameters. - Args: - state (Optional[List[str]]): List of states to filter organizations by. - regionId (Optional[List[str]]): List of region IDs to filter organizations by. +def add_user_to_org_v2(organization_id: str, user_data, current_user): + """Add a user to a particular organization.""" + try: + # Check if the current user has regional admin permissions + if not is_regional_admin(current_user): + raise HTTPException(status_code=403, detail="Unauthorized access.") + + # Validate the organization ID format (UUID) + if not is_valid_uuid(organization_id): + raise HTTPException(status_code=404, detail="Invalid organization ID.") + + # Fetch the organization by ID + try: + organization = Organization.objects.get(id=organization_id) + except Organization.DoesNotExist: + raise HTTPException(status_code=404, detail="Organization not found.") + + # Validate the user ID in the body + user_id = user_data.userId + if not is_valid_uuid(user_id): + raise HTTPException(status_code=404, detail="Invalid user ID.") + + # Fetch the user by ID + try: + user = User.objects.get(id=user_id) + except User.DoesNotExist: + raise HTTPException(status_code=404, detail="User not found.") + + # Prepare the new role data + new_role_data = { + "user": user, + "organization": organization, + "approved": True, + "role": user_data.role, + "approvedBy": current_user, + "createdBy": current_user, + } + + # Create the new role object + new_role = Role.objects.create(**new_role_data) + + # Return the created role in the response + return { + "statusCode": 200, + "body": { + "id": str(new_role.id), + "user": { + "id": str(new_role.user.id), + "email": new_role.user.email, + "firstName": new_role.user.firstName, + "lastName": new_role.user.lastName, + }, + "organization": { + "id": str(new_role.organization.id), + "name": new_role.organization.name, + }, + "role": new_role.role, + "approved": new_role.approved, + "approvedBy": { + "id": str(new_role.approvedBy.id), + "email": new_role.approvedBy.email, + }, + "createdBy": { + "id": str(new_role.createdBy.id), + "email": new_role.createdBy.email, + }, + }, + } + + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) + + +def approve_role(organization_id: str, role_id, current_user): + """Approve a role within an organization.""" + + # Check if the current user is an org admin for the organization + if not is_org_admin(current_user, organization_id): + raise HTTPException(status_code=403, detail="Unauthorized access.") + + # Validate that the role_id is a valid UUID + if not is_valid_uuid(role_id): + raise HTTPException(status_code=404, detail="Role not found") - Raises: - HTTPException: If the user is not authorized or no organizations are found. + # Validate that the organization_id is a valid UUID + if not is_valid_uuid(organization_id): + raise HTTPException(status_code=404, detail="Organization not found") - Returns: - List[Organizations]: A list of organizations matching the filter criteria. - """ + try: + # Fetch the role within the organization + role = Role.objects.filter(organization_id=organization_id, id=role_id).first() + + if role: + # Approve the role and set the approvedBy field to the current user + role.approved = True + role.approvedBy = current_user + role.save() + + return {"status": "success", "message": "Role approved successfully"} - # if not current_user: - # raise HTTPException(status_code=401, detail="Unauthorized") + raise HTTPException(status_code=404, detail="Role not found") - # Prepare filter parameters - filter_params = {} - if state: - filter_params["state__in"] = state - if regionId: - filter_params["regionId__in"] = regionId + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) - organizations = Organization.objects.filter(**filter_params) - if not organizations.exists(): - raise HTTPException(status_code=404, detail="No organizations found") +def remove_role(organization_id: str, role_id, current_user): + """Remove a role within an organization.""" - # Return the Pydantic models directly by calling from_orm - return organizations + # Check if the current user is an org admin for the organization + if not is_org_admin(current_user, organization_id): + raise HTTPException(status_code=403, detail="Unauthorized access.") + + # Validate that the role_id is a valid UUID + if not is_valid_uuid(role_id): + raise HTTPException(status_code=404, detail="Role not found") + + # Validate that the organization_id is a valid UUID + if not is_valid_uuid(organization_id): + raise HTTPException(status_code=404, detail="Organization not found") + + try: + # Attempt to delete the role within the organization + result = Role.objects.filter( + organization_id=organization_id, id=role_id + ).delete() + + # If no role was deleted, raise a 404 + if result[0] == 0: + raise HTTPException(status_code=404, detail="Role not found") + + return {"status": "success", "message": "Role removed successfully"} + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def update_org_scan(organization_id: str, scan_id, scan_data, current_user): + """Enable or disable a scan for a particular organization.""" + + # Validate organization_id is a valid UUID + if not is_valid_uuid(organization_id): + raise HTTPException(status_code=404, detail="Organization not found") + + # Check if the current user is either an org admin or a global write admin + if not ( + is_org_admin(current_user, organization_id) + or is_global_write_admin(current_user) + ): + raise HTTPException(status_code=403, detail="Unauthorized access.") + + # Validate scan_id is a valid UUID + if not is_valid_uuid(scan_id): + raise HTTPException(status_code=404, detail="Scan not found") + + try: + # Fetch the scan that is granular and user-modifiable + scan = Scan.objects.filter( + id=scan_id, isGranular=True, isUserModifiable=True + ).first() + if not scan: + raise HTTPException( + status_code=404, detail="Scan not found or not modifiable" + ) + + # Fetch the organization and its related granular scans + organization = ( + Organization.objects.prefetch_related("granularScans") + .filter(id=organization_id) + .first() + ) + if not organization: + raise HTTPException(status_code=404, detail="Organization not found") + + # Check the "enabled" field in the request body + if not scan_data.enabled: + enabled = False + else: + enabled = scan_data.enabled + + # Add the scan to the organization's granular scans if enabled and not already present + if enabled: + if not organization.granularScans.filter(id=scan_id).exists(): + organization.granularScans.add(scan) + # Remove the scan from the organization's granular scans if disabled and present + else: + if organization.granularScans.filter(id=scan_id).exists(): + organization.granularScans.remove(scan) + + # Save the updated organization + organization.save() + + # Return a success response + return { + "status": "success", + "organization_id": organization_id, + "scan_id": scan_id, + "enabled": enabled, + } + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def list_organizations_v2(state, regionId, current_user): + """List organizations that the user is a member of or has access to.""" + try: + # Check if user is GlobalViewAdmin or has memberships + if not is_global_view_admin(current_user) and not get_org_memberships( + current_user + ): + return [] + + # Define filter for organizations based on admin status + # Prepare the filter criteria + filter_criteria = Q() + + if state: + filter_criteria &= Q(state__in=state) + + if regionId: + filter_criteria &= Q(regionId__in=regionId) + + # Fetch organizations with related userRoles and tags + organizations = ( + Organization.objects.filter(filter_criteria) + if filter_criteria + else Organization.objects.all() + ) + + # Serialize organizations using list comprehension + organization_list = [ + { + "id": str(org.id), + "createdAt": org.createdAt.isoformat(), + "updatedAt": org.updatedAt.isoformat(), + "acronym": org.acronym, + "name": org.name, + "rootDomains": org.rootDomains, + "ipBlocks": org.ipBlocks, + "isPassive": org.isPassive, + "pendingDomains": org.pendingDomains, + "country": org.country, + "state": org.state, + "regionId": org.regionId, + "stateFips": org.stateFips, + "stateName": org.stateName, + "county": org.county, + "countyFips": org.countyFips, + "type": org.type, + "userRoles": [ + {"id": str(role.id), "role": role.role, "approved": role.approved} + for role in org.userRoles.all() + ], + "tags": [ + { + "id": str(tag.id), + "createdAt": tag.createdAt.isoformat(), + "updatedAt": tag.updatedAt.isoformat(), + "name": tag.name, + } + for tag in org.tags.all() + ], + } + for org in organizations + ] + + return organization_list + + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index f8ca13d3..d5074ae7 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -9,7 +9,7 @@ from fastapi.security import APIKeyHeader, OAuth2PasswordBearer from .jwt_utils import decode_jwt_token -from .models import ApiKey, OrganizationTag +from .models import ApiKey, Organization, OrganizationTag, Role oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False) @@ -75,6 +75,45 @@ def is_regional_admin(current_user) -> bool: return current_user and current_user.userType in ["regionalAdmin", "globalAdmin"] +def is_org_admin(current_user, organization_id) -> bool: + """Check if the user is an admin of the given organization.""" + if not organization_id: + return False + + # Check if the user has an admin role in the given organization + for role in current_user.roles.all(): + if role.organization.id == organization_id and role.role == "admin": + return True + + # If the user is a global write admin, they are considered an org admin + return is_global_write_admin(current_user) + + +def is_regional_admin_for_organization(current_user, organization_id) -> bool: + """Check if user is a regional admin and if a selected organization belongs to their region.""" + if not organization_id: + return False + + # Check if the user is a regional admin + if is_regional_admin(current_user): + # Check if the organization belongs to the user's region + user_region_id = ( + current_user.regionId + ) # Assuming this is available in the user object + organization_region_id = get_organization_region( + organization_id + ) # Function to fetch the organization's region + return user_region_id == organization_region_id + + return False + + +def get_organization_region(organization_id: str) -> str: + """Fetch the region ID for the given organization.""" + organization = Organization.objects.get(id=organization_id) + return organization.regionId + + def get_tag_organizations(current_user, tag_id: str) -> list[str]: """Returns the organizations belonging to a tag, if the user can access the tag.""" # Check if the user is a global view admin @@ -99,6 +138,14 @@ def get_tag_organizations(current_user, tag_id: str) -> list[str]: return [] +def get_org_memberships(current_user) -> list[str]: + """Returns the organization IDs that a user is a member of.""" + roles = Role.objects.filter(userId=current_user) + if not roles: + return [] + return [role.organizationId.id for role in roles if role.organizationId] + + # TODO: Below is a template of what these could be nut isn't tested # RECREATE ALL THE FUNCTIONS IN AUTH.TS diff --git a/backend/src/xfd_django/xfd_api/helpers/regionStateMap.py b/backend/src/xfd_django/xfd_api/helpers/regionStateMap.py new file mode 100644 index 00000000..390c83b3 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/helpers/regionStateMap.py @@ -0,0 +1,62 @@ +"""Region state mapping dictionary.""" + +REGION_STATE_MAP = { + "Alabama": "4", + "Alaska": "10", + "American Samoa": "9", + "Arkansas": "6", + "Arizona": "9", + "California": "9", + "Colorado": "8", + "Commonwealth Northern Mariana Islands": "9", + "Connecticut": "1", + "Delaware": "3", + "District of Columbia": "3", + "Federal States of Micronesia": "9", + "Florida": "4", + "Georgia": "4", + "Guam": "9", + "Hawaii": "9", + "Idaho": "10", + "Illinois": "5", + "Indiana": "5", + "Iowa": "7", + "Kansas": "7", + "Kentucky": "4", + "Louisiana": "6", + "Maine": "1", + "Maryland": "3", + "Massachusetts": "1", + "Michigan": "5", + "Minnesota": "5", + "Mississippi": "4", + "Missouri": "7", + "Montana": "8", + "Nebraska": "7", + "Nevada": "9", + "New Hampshire": "1", + "New Jersey": "2", + "New Mexico": "6", + "New York": "2", + "North Carolina": "4", + "North Dakota": "8", + "Ohio": "5", + "Oklahoma": "6", + "Oregon": "10", + "Pennsylvania": "3", + "Puerto Rico": "2", + "Republic of Marshall Islands": "9", + "Rhode Island": "1", + "South Carolina": "4", + "South Dakota": "8", + "Tennessee": "4", + "Texas": "6", + "Utah": "8", + "Vermont": "1", + "Virgin Islands": "2", + "Virginia": "3", + "Washington": "10", + "West Virginia": "3", + "Wisconsin": "5", + "Wyoming": "8", +} diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 9642d31f..89e73055 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -12,14 +12,19 @@ class ApiKey(models.Model): """The ApiKey model.""" - id = models.UUIDField(primary_key=True) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) createdAt = models.DateTimeField(auto_now_add=True, db_column="createdAt") updatedAt = models.DateTimeField(auto_now=True, db_column="updatedAt") lastUsed = models.DateTimeField(db_column="lastUsed", blank=True, null=True) hashedKey = models.TextField(db_column="hashedKey") lastFour = models.TextField(db_column="lastFour") userId = models.ForeignKey( - "User", models.CASCADE, db_column="userId", blank=True, null=True + "User", + models.CASCADE, + db_column="userId", + blank=True, + null=True, + related_name="apiKeys", ) class Meta: @@ -32,13 +37,19 @@ class Meta: class Assessment(models.Model): """The Assessment model.""" - id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column="createdAt") - updatedAt = models.DateTimeField(db_column="updatedAt") - rscId = models.CharField(db_column="rscId", unique=True) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + createdAt = models.DateTimeField(auto_now_add=True, db_column="createdAt") + updatedAt = models.DateTimeField(auto_now=True, db_column="updatedAt") + rscId = models.CharField(max_length=255, db_column="rscId", unique=True) type = models.CharField(max_length=255) - userId = models.ForeignKey( - "User", db_column="userId", blank=True, null=True, on_delete=models.CASCADE + + user = models.ForeignKey( + "User", + db_column="userId", + blank=True, + null=True, + on_delete=models.CASCADE, + related_name="assessments", ) class Meta: @@ -51,7 +62,7 @@ class Meta: class Category(models.Model): """The Category model.""" - id = models.UUIDField(primary_key=True) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=255) number = models.CharField(max_length=255, unique=True) shortName = models.CharField( @@ -68,7 +79,7 @@ class Meta: class Cpe(models.Model): """The Cpe model.""" - id = models.UUIDField(primary_key=True) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(max_length=255) version = models.CharField(max_length=255) vendor = models.CharField(max_length=255) @@ -85,7 +96,7 @@ class Meta: class Cve(models.Model): """The Cve model.""" - id = models.UUIDField(primary_key=True) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) name = models.CharField(unique=True, blank=True, null=True) publishedAt = models.DateTimeField(db_column="publishedAt", blank=True, null=True) modifiedAt = models.DateTimeField(db_column="modifiedAt", blank=True, null=True) @@ -172,35 +183,38 @@ class Meta: class Domain(models.Model): """The Domain model.""" - id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column="createdAt") - updatedAt = models.DateTimeField(db_column="updatedAt") + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + createdAt = models.DateTimeField(auto_now_add=True, db_column="createdAt") + updatedAt = models.DateTimeField(auto_now=True, db_column="updatedAt") + syncedAt = models.DateTimeField(db_column="syncedAt", blank=True, null=True) ip = models.CharField(max_length=255, blank=True, null=True) - fromRootDomain = models.CharField(db_column="fromRootDomain", blank=True, null=True) + fromRootDomain = models.CharField( + max_length=255, db_column="fromRootDomain", blank=True, null=True + ) subdomainSource = models.CharField( db_column="subdomainSource", max_length=255, blank=True, null=True ) ipOnly = models.BooleanField(db_column="ipOnly", default=False) + reverseName = models.CharField(db_column="reverseName", max_length=512) name = models.CharField(max_length=512) + screenshot = models.CharField(max_length=512, blank=True, null=True) country = models.CharField(max_length=255, blank=True, null=True) asn = models.CharField(max_length=255, blank=True, null=True) cloudHosted = models.BooleanField(db_column="cloudHosted", default=False) + ssl = models.JSONField(blank=True, null=True) censysCertificatesResults = models.JSONField( db_column="censysCertificatesResults", default=dict ) trustymailResults = models.JSONField(db_column="trustymailResults", default=dict) - discoveredById = models.ForeignKey( - "Scan", - on_delete=models.SET_NULL, - db_column="discoveredById", - blank=True, - null=True, + + discoveredBy = models.ForeignKey( + "Scan", on_delete=models.SET_NULL, null=True, blank=True ) - organizationId = models.ForeignKey( + organization = models.ForeignKey( "Organization", on_delete=models.CASCADE, db_column="organizationId" ) @@ -209,7 +223,7 @@ class Meta: db_table = "domain" managed = False # This ensures Django does not manage the table - unique_together = (("name", "organizationId"),) # Unique constraint + unique_together = (("name", "organization"),) # Unique constraint def save(self, *args, **kwargs): self.name = self.name.lower() @@ -247,40 +261,44 @@ class Organization(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) + acronym = models.CharField(unique=True, blank=True, null=True, max_length=255) - name = models.CharField() + name = models.CharField(max_length=255) rootDomains = ArrayField(models.CharField(max_length=255), db_column="rootDomains") ipBlocks = ArrayField(models.CharField(max_length=255), db_column="ipBlocks") - isPassive = models.BooleanField(db_column="isPassive") - pendingDomains = models.TextField(db_column="pendingDomains", default=list) - country = models.CharField(blank=True, null=True) - state = models.CharField(blank=True, null=True) - regionId = models.CharField(db_column="regionId", blank=True, null=True) - stateFips = models.IntegerField(db_column="stateFips", blank=True, null=True) - stateName = models.CharField(db_column="stateName", blank=True, null=True) - county = models.CharField(blank=True, null=True) - countyFips = models.IntegerField(db_column="countyFips", blank=True, null=True) - type = models.CharField(blank=True, null=True) - parentId = models.ForeignKey( - "self", models.DO_NOTHING, db_column="parentId", blank=True, null=True - ) - createdById = models.ForeignKey( - "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True + isPassive = models.BooleanField(db_column="isPassive", default=False) + + pendingDomains = models.TextField( + db_column="pendingDomains", default="[]" + ) # ******* Had to change this from JSON TO TEXT********** + country = models.CharField(max_length=255, blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + regionId = models.CharField( + max_length=255, db_column="regionId", blank=True, null=True ) - # TODO: Consider geting rid of this, don't need Many To Many in both tables - # Relationships with other models (Scan, OrganizationTag) - granularScans = models.ManyToManyField( - "Scan", related_name="organizations", db_table="scan_organizations_organization" + stateFips = models.IntegerField(db_column="stateFips", blank=True, null=True) + stateName = models.CharField( + max_length=255, db_column="stateName", blank=True, null=True ) - tags = models.ManyToManyField( - "OrganizationTag", - related_name="organizations", - db_table="organization_tag_organizations_organization", + county = models.CharField(max_length=255, blank=True, null=True) + countyFips = models.IntegerField(db_column="countyFips", blank=True, null=True) + type = models.CharField(max_length=255, blank=True, null=True) + + parent = models.ForeignKey( + "self", + db_column="parentId", + on_delete=models.CASCADE, + related_name="children", + null=True, + blank=True, ) - allScanTasks = models.ManyToManyField( - "ScanTask", - related_name="organizations", - db_table="scan_task_organizations_organization", + + createdBy = models.ForeignKey( + "User", + db_column="createdById", + on_delete=models.SET_NULL, + null=True, + blank=True, ) class Meta: @@ -296,7 +314,8 @@ class OrganizationTag(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) - name = models.CharField(unique=True) + + name = models.CharField(max_length=255, unique=True) organizations = models.ManyToManyField( "Organization", related_name="tags", @@ -407,34 +426,38 @@ class Meta: class Role(models.Model): """The Role model.""" - id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column="createdAt") - updatedAt = models.DateTimeField(db_column="updatedAt") - role = models.CharField() - approved = models.BooleanField() - createdById = models.ForeignKey( - "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + createdAt = models.DateTimeField(auto_now_add=True, db_column="createdAt") + updatedAt = models.DateTimeField(auto_now=True, db_column="updatedAt") + + role = models.CharField(max_length=10, default="user") + approved = models.BooleanField(default=False) + + user = models.ForeignKey( + "User", on_delete=models.CASCADE, db_column="userId", related_name="roles" ) - approvedById = models.ForeignKey( + createdBy = models.ForeignKey( "User", models.DO_NOTHING, - db_column="approvedById", - related_name="role_approvedbyid_set", + db_column="createdById", + related_name="createdRoles", blank=True, null=True, ) - userId = models.ForeignKey( + approvedBy = models.ForeignKey( "User", - on_delete=models.CASCADE, - db_column="userId", - related_name="roles", + models.DO_NOTHING, + db_column="approvedById", + related_name="approvedRoles", blank=True, null=True, ) - organizationId = models.ForeignKey( - Organization, - models.DO_NOTHING, + + organization = models.ForeignKey( + "Organization", + on_delete=models.CASCADE, db_column="organizationId", + related_name="userRoles", blank=True, null=True, ) @@ -444,7 +467,7 @@ class Meta: managed = False db_table = "role" - unique_together = (("userId", "organizationId"),) + unique_together = (("user", "organization"),) class SavedSearch(models.Model): @@ -479,11 +502,13 @@ class Scan(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) createdAt = models.DateTimeField(db_column="createdAt", auto_now_add=True) updatedAt = models.DateTimeField(db_column="updatedAt", auto_now=True) + name = models.CharField() arguments = models.TextField( default="{}" ) # JSON in the database but fails: the JSON object must be str, bytes or bytearray, not dict frequency = models.IntegerField() + lastRun = models.DateTimeField(db_column="lastRun", blank=True, null=True) isGranular = models.BooleanField(db_column="isGranular", default=False) isUserModifiable = models.BooleanField( @@ -491,14 +516,17 @@ class Scan(models.Model): ) isSingleScan = models.BooleanField(db_column="isSingleScan", default=False) manualRunPending = models.BooleanField(db_column="manualRunPending", default=False) + createdBy = models.ForeignKey( - "User", models.DO_NOTHING, db_column="createdById", blank=True, null=True + "User", models.SET_NULL, db_column="createdById", blank=True, null=True ) tags = models.ManyToManyField( "OrganizationTag", related_name="scans", db_table="scan_tags_organization_tag" ) organizations = models.ManyToManyField( - "Organization", related_name="scans", db_table="scan_organizations_organization" + "Organization", + related_name="granularScans", + db_table="scan_organizations_organization", ) class Meta: @@ -523,8 +551,9 @@ class ScanTask(models.Model): startedAt = models.DateTimeField(db_column="startedAt", blank=True, null=True) finishedAt = models.DateTimeField(db_column="finishedAt", blank=True, null=True) queuedAt = models.DateTimeField(db_column="queuedAt", blank=True, null=True) - scanId = models.ForeignKey( - Scan, on_delete=models.DO_NOTHING, db_column="scanId", blank=True, null=True + + scan = models.ForeignKey( + Scan, on_delete=models.SET_NULL, db_column="scanId", blank=True, null=True ) organizations = models.ManyToManyField( "Organization", @@ -542,25 +571,39 @@ class Meta: class Service(models.Model): """The Service model.""" - id = models.UUIDField(primary_key=True) - createdAt = models.DateTimeField(db_column="createdAt") - updatedAt = models.DateTimeField(db_column="updatedAt") + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + createdAt = models.DateTimeField(auto_now_add=True, db_column="createdAt") + updatedAt = models.DateTimeField(auto_now=True, db_column="updatedAt") + serviceSource = models.TextField(db_column="serviceSource", blank=True, null=True) port = models.IntegerField() service = models.CharField(blank=True, null=True) lastSeen = models.DateTimeField(db_column="lastSeen", blank=True, null=True) banner = models.TextField(blank=True, null=True) - products = models.JSONField() - censysMetadata = models.JSONField(db_column="censysMetadata") - censysIpv4Results = models.JSONField(db_column="censysIpv4Results") - intrigueIdentResults = models.JSONField(db_column="intrigueIdentResults") - shodanResults = models.JSONField(db_column="shodanResults") - wappalyzerResults = models.JSONField(db_column="wappalyzerResults") - domainId = models.ForeignKey( - Domain, models.DO_NOTHING, db_column="domainId", blank=True, null=True + + products = models.JSONField(default=list) + censysMetadata = models.JSONField( + db_column="censysMetadata", null=True, blank=True, default=dict ) - discoveredById = models.ForeignKey( - Scan, models.DO_NOTHING, db_column="discoveredById", blank=True, null=True + censysIpv4Results = models.JSONField(db_column="censysIpv4Results", default=dict) + intrigueIdentResults = models.JSONField( + db_column="intrigueIdentResults", default=dict + ) + shodanResults = models.JSONField( + db_column="shodanResults", null=True, blank=True, default=dict + ) + wappalyzerResults = models.JSONField(db_column="wappalyzerResults", default=list) + + domain = models.ForeignKey( + Domain, db_column="domainId", on_delete=models.CASCADE, related_name="services" + ) + discoveredBy = models.ForeignKey( + Scan, + db_column="discoveredById", + blank=True, + null=True, + on_delete=models.SET_NULL, + related_name="services", ) class Meta: @@ -568,7 +611,7 @@ class Meta: managed = False db_table = "service" - unique_together = (("port", "domainId"),) + unique_together = (("port", "domain"),) class TypeormMetadata(models.Model): @@ -588,25 +631,38 @@ class Meta: db_table = "typeorm_metadata" +class UserType(models.TextChoices): + GLOBAL_ADMIN = "globalAdmin" + GLOBAL_VIEW = "globalView" + REGIONAL_ADMIN = "regionalAdmin" + READY_SET_CYBER = "readySetCyber" + STANDARD = "standard" + + class User(models.Model): """The User model.""" - id = models.UUIDField(primary_key=True) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) cognitoId = models.CharField( - db_column="cognitoId", unique=True, blank=True, null=True + max_length=255, db_column="cognitoId", unique=True, blank=True, null=True + ) + oktaId = models.CharField( + max_length=255, db_column="oktaId", null=True, blank=True, unique=True ) loginGovId = models.CharField( - db_column="loginGovId", unique=True, blank=True, null=True + max_length=255, db_column="loginGovId", unique=True, blank=True, null=True ) - createdAt = models.DateTimeField(db_column="createdAt") - updatedAt = models.DateTimeField(db_column="updatedAt") - firstName = models.CharField(db_column="firstName") - lastName = models.CharField(db_column="lastName") - fullName = models.CharField(db_column="fullName") + createdAt = models.DateTimeField(auto_now_add=True, db_column="createdAt") + updatedAt = models.DateTimeField(auto_now=True, db_column="updatedAt") + + firstName = models.CharField(max_length=255, db_column="firstName") + lastName = models.CharField(max_length=255, db_column="lastName") + fullName = models.CharField(max_length=255, db_column="fullName") email = models.CharField(unique=True) - invitePending = models.BooleanField(db_column="invitePending") + + invitePending = models.BooleanField(db_column="invitePending", default=False) loginBlockedByMaintenance = models.BooleanField( - db_column="loginBlockedByMaintenance" + db_column="loginBlockedByMaintenance", default=False ) dateAcceptedTerms = models.DateTimeField( db_column="dateAcceptedTerms", blank=True, null=True @@ -614,11 +670,23 @@ class User(models.Model): acceptedTermsVersion = models.TextField( db_column="acceptedTermsVersion", blank=True, null=True ) + lastLoggedIn = models.DateTimeField(db_column="lastLoggedIn", blank=True, null=True) - userType = models.TextField(db_column="userType") - regionId = models.CharField(db_column="regionId", blank=True, null=True) - state = models.CharField(blank=True, null=True) - oktaId = models.CharField(db_column="oktaId", unique=True, blank=True, null=True) + userType = models.CharField( + db_column="userType", + max_length=50, + choices=UserType.choices, + default=UserType.STANDARD, + ) + + regionId = models.CharField( + db_column="regionId", blank=True, null=True, max_length=255 + ) + state = models.CharField(blank=True, null=True, max_length=255) + + def save(self, *args, **kwargs): + self.full_name = f"{self.first_name} {self.last_name}" + super().save(*args, **kwargs) class Meta: """The Meta class for User.""" diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization.py b/backend/src/xfd_django/xfd_api/schema_models/organization.py index edf9191b..625d0ce5 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/organization.py +++ b/backend/src/xfd_django/xfd_api/schema_models/organization.py @@ -2,11 +2,13 @@ # Standard Python Libraries from datetime import datetime -from typing import List, Optional +from typing import Any, List, Optional from uuid import UUID # Third-Party Libraries -from pydantic import BaseModel, Json +from pydantic import BaseModel + +from .organization_tag import OrganizationalTags class Organization(BaseModel): @@ -29,3 +31,181 @@ class Organization(BaseModel): county: Optional[str] countyFips: Optional[int] type: Optional[str] + + +class UserRoleSchema(BaseModel): + """User role schema.""" + + id: UUID + role: str + approved: bool + + +class TagSchema(BaseModel): + """Tag schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + name: str + + +class GetTagSchema(BaseModel): + """Tag simplified schema.""" + + id: UUID + name: str + + +class SimpleScanSchema(BaseModel): + """Simple scan schema.""" + + id: UUID + name: str + + +class GranularScanSchema(BaseModel): + """Granular task schema.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + name: str + arguments: Any + frequency: int + lastRun: Optional[datetime] + isGranular: bool + isUserModifiable: Optional[bool] + isSingleScan: bool + manualRunPending: bool + tags: Optional[List[OrganizationalTags]] = [] + organizations: Optional[List[Organization]] = [] + + +class ScanTaskSchema(BaseModel): + """Scan task schema.""" + + id: UUID + createdAt: datetime + scan: SimpleScanSchema + + +class GetOrganizationSchema(BaseModel): + """Schema for listing an organization.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + acronym: Optional[str] + name: str + rootDomains: List[str] + ipBlocks: List[str] + isPassive: bool + pendingDomains: Optional[Any] = [] + country: Optional[str] = None + state: Optional[str] = None + regionId: Optional[str] = None + stateFips: Optional[int] = None + stateName: Optional[str] = None + county: Optional[str] = None + countyFips: Optional[int] = None + type: Optional[str] = None + userRoles: Optional[List[UserRoleSchema]] = [] + tags: Optional[List[TagSchema]] = [] + + +class GetSingleOrganizationSchema(BaseModel): + """Schema for listing an organization.""" + + id: UUID + createdAt: datetime + updatedAt: datetime + acronym: Optional[str] + name: str + rootDomains: List[str] + ipBlocks: List[str] + isPassive: bool + pendingDomains: Optional[Any] = [] + country: Optional[str] = None + state: Optional[str] = None + regionId: Optional[str] = None + stateFips: Optional[int] = None + stateName: Optional[str] = None + county: Optional[str] = None + countyFips: Optional[int] = None + type: Optional[str] = None + userRoles: Optional[List[UserRoleSchema]] = [] + tags: Optional[List[TagSchema]] = [] + parent: Optional[Any] = {} + children: Optional[Any] = {} + granularScans: Optional[List[GranularScanSchema]] = [] + scanTasks: Optional[List[ScanTaskSchema]] = [] + + +class NewTag(BaseModel): + """Schema for tag data.""" + + name: str # Adjust this if there could be an 'id' field + + +class NewOrganization(BaseModel): + """Create a new organization schema.""" + + acronym: Optional[str] + name: str + rootDomains: List[str] + ipBlocks: List[str] + isPassive: bool + pendingDomains: Optional[Any] = [] + country: Optional[str] = None + state: Optional[str] = None + regionId: Optional[str] = None + stateFips: Optional[int] = None + stateName: Optional[str] = None + county: Optional[str] = None + countyFips: Optional[int] = None + type: Optional[str] = None + parent: Optional[str] = None + tags: Optional[List[NewTag]] = None + + +class NewOrgUser(BaseModel): + """Add a user to organization schema.""" + + userId: str + role: str + + +class NewOrgScan(BaseModel): + """Update an organization scan schema.""" + + enabled: bool + + +class RegionSchema(BaseModel): + """Update an organization scan schema.""" + + regionId: str + + +class GenericMessageResponseModel(BaseModel): + """Generic response model.""" + + status: str + message: str + + +class GenericPostResponseModel(BaseModel): + """Generic response model.""" + + statusCode: int + body: Any + + +class UpdateOrgScanSchema(BaseModel): + """Update an org's scan model.""" + + status: str + organization_id: UUID + scan_id: UUID + enabled: bool diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan.py b/backend/src/xfd_django/xfd_api/schema_models/scan.py index 6c6d2817..1b3d1e9c 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/scan.py +++ b/backend/src/xfd_django/xfd_api/schema_models/scan.py @@ -121,7 +121,7 @@ class GetScanResponseModel(BaseModel): class GenericMessageResponseModel(BaseModel): - """Get Scans response model.""" + """Generic response model.""" status: str message: str diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 58cc6bd7..60626b26 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -24,16 +24,16 @@ from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import UUID4 -from .api_methods import scan, scan_tasks +from .api_methods import organization, scan, scan_tasks from .api_methods.api_keys import get_api_keys from .api_methods.cpe import get_cpes_by_id from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import get_domain_by_id -from .api_methods.organization import get_organizations, read_orgs from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user from .models import Assessment, User +from .schema_models import organization as OrganizationSchema from .schema_models import scan as scanSchema from .schema_models import scan_tasks as scanTaskSchema from .schema_models.assessment import Assessment as AssessmentSchema @@ -41,7 +41,6 @@ from .schema_models.cve import Cve as CveSchema from .schema_models.domain import Domain as DomainSchema from .schema_models.domain import DomainSearch -from .schema_models.organization import Organization as OrganizationSchema from .schema_models.user import User as UserSchema from .schema_models.vulnerability import Vulnerability as VulnerabilitySchema @@ -72,28 +71,6 @@ async def call_get_api_keys(): return get_api_keys() -@api_router.post( - "/test-orgs", - # dependencies=[Depends(get_current_active_user)], - response_model=List[OrganizationSchema], - tags=["Organizations", "Testing"], -) -async def call_read_orgs(): - """ - List all organizations with query parameters. - Args: - state (Optional[List[str]]): List of states to filter organizations by. - regionId (Optional[List[str]]): List of region IDs to filter organizations by. - - Raises: - HTTPException: If the user is not authorized or no organizations are found. - - Returns: - List[Organizations]: A list of organizations matching the filter criteria. - """ - return read_orgs() - - # TODO: Uncomment checks for current_user once authentication is implemented @api_router.get( "/assessments", @@ -293,31 +270,6 @@ async def call_get_users( return get_users(state, regionId, invitePending) -@api_router.get( - "/organizations", - # response_model=List[OrganizationSchema], - # dependencies=[Depends(get_current_active_user)], - tags=["Organizations"], -) -async def call_get_organizations( - state: Optional[List[str]] = Query(None), - regionId: Optional[List[str]] = Query(None), -): - """ - List all organizations with query parameters. - Args: - state (Optional[List[str]]): List of states to filter organizations by. - regionId (Optional[List[str]]): List of region IDs to filter organizations by. - - Raises: - HTTPException: If the user is not authorized or no organizations are found. - - Returns: - List[Organizations]: A list of organizations matching the filter criteria. - """ - return get_organizations(state, regionId) - - # ======================================== # Scan Endpoints # ======================================== @@ -459,3 +411,214 @@ async def get_scan_task_logs( ): """Get logs from a particular scan task.""" return scan_tasks.get_scan_task_logs(scan_task_id, current_user) + + +# ======================================== +# Organization Endpoints +# ======================================== + + +@api_router.get( + "/organizations", + dependencies=[Depends(get_current_active_user)], + response_model=List[OrganizationSchema.GetOrganizationSchema], + tags=["Organizations"], +) +async def list_organizations(current_user: User = Depends(get_current_active_user)): + """Retrieve a list of all organizations.""" + return organization.list_organizations(current_user) + + +@api_router.get( + "/organizations/tags", + dependencies=[Depends(get_current_active_user)], + response_model=List[OrganizationSchema.GetTagSchema], + tags=["Organizations"], +) +async def get_organization_tags(current_user: User = Depends(get_current_active_user)): + """Retrieve a list of organization tags.""" + return organization.get_tags(current_user) + + +@api_router.get( + "/organizations/{organization_id}", + dependencies=[Depends(get_current_active_user)], + response_model=OrganizationSchema.GetSingleOrganizationSchema, + tags=["Organizations"], +) +async def get_organization( + organization_id: str, current_user: User = Depends(get_current_active_user) +): + """Retrieve an organization by its ID.""" + return organization.get_organization(organization_id, current_user) + + +@api_router.get( + "/organizations/state/{state}", + dependencies=[Depends(get_current_active_user)], + response_model=List[OrganizationSchema.GetOrganizationSchema], + tags=["Organizations"], +) +async def get_organizations_by_state( + state: str, current_user: User = Depends(get_current_active_user) +): + """Retrieve organizations by state.""" + return organization.get_by_state(state, current_user) + + +@api_router.get( + "/organizations/regionId/{region_id}", + dependencies=[Depends(get_current_active_user)], + response_model=List[OrganizationSchema.GetOrganizationSchema], + tags=["Organizations"], +) +async def get_organizations_by_region( + region_id: str, current_user: User = Depends(get_current_active_user) +): + """Retrieve organizations by region ID.""" + return organization.get_by_region(region_id, current_user) + + +@api_router.get( + "/regions", + dependencies=[Depends(get_current_active_user)], + response_model=List[OrganizationSchema.RegionSchema], + tags=["Regions"], +) +async def list_regions(current_user: User = Depends(get_current_active_user)): + """Retrieve a list of all regions.""" + return organization.get_all_regions(current_user) + + +@api_router.post( + "/organizations", + dependencies=[Depends(get_current_active_user)], + response_model=OrganizationSchema.GetSingleOrganizationSchema, + tags=["Organizations"], +) +async def create_organization( + organization_data: OrganizationSchema.NewOrganization, + current_user: User = Depends(get_current_active_user), +): + """Create a new organization.""" + return organization.create_organization(organization_data, current_user) + + +@api_router.post( + "/organizations_upsert", + dependencies=[Depends(get_current_active_user)], + response_model=OrganizationSchema.GetSingleOrganizationSchema, + tags=["Organizations"], +) +async def upsert_organization( + organization_data: OrganizationSchema.NewOrganization, + current_user: User = Depends(get_current_active_user), +): + """Upsert an organization.""" + return organization.upsert_organization(organization_data, current_user) + + +@api_router.put( + "/organizations/{organization_id}", + dependencies=[Depends(get_current_active_user)], + response_model=OrganizationSchema.GetSingleOrganizationSchema, + tags=["Organizations"], +) +async def update_organization( + organization_id: str, + org_data: OrganizationSchema.NewOrganization, + current_user: User = Depends(get_current_active_user), +): + """Update an organization by its ID.""" + return organization.update_organization(organization_id, org_data, current_user) + + +@api_router.delete( + "/organizations/{organization_id}", + dependencies=[Depends(get_current_active_user)], + response_model=OrganizationSchema.GenericMessageResponseModel, + tags=["Organizations"], +) +async def delete_organization( + organization_id: str, current_user: User = Depends(get_current_active_user) +): + """Delete an organization by its ID.""" + return organization.delete_organization(organization_id, current_user) + + +@api_router.post( + "/v2/organizations/{organization_id}/users", + dependencies=[Depends(get_current_active_user)], + response_model=OrganizationSchema.GenericPostResponseModel, + tags=["Organizations"], +) +async def add_user_to_organization_v2( + organization_id: str, + user_data: OrganizationSchema.NewOrgUser, + current_user: User = Depends(get_current_active_user), +): + """Add a user to an organization.""" + return organization.add_user_to_org_v2(organization_id, user_data, current_user) + + +@api_router.post( + "/organizations/{organization_id}/roles/{role_id}/approve", + dependencies=[Depends(get_current_active_user)], + response_model=OrganizationSchema.GenericMessageResponseModel, + tags=["Organizations"], +) +async def approve_role( + organization_id: str, + role_id: str, + current_user: User = Depends(get_current_active_user), +): + """Approve a role within an organization.""" + return organization.approve_role(organization_id, role_id, current_user) + + +@api_router.post( + "/organizations/{organization_id}/roles/{role_id}/remove", + dependencies=[Depends(get_current_active_user)], + response_model=OrganizationSchema.GenericMessageResponseModel, + tags=["Organizations"], +) +async def remove_role( + organization_id: str, + role_id: str, + current_user: User = Depends(get_current_active_user), +): + """Remove a role from an organization.""" + return organization.remove_role(organization_id, role_id, current_user) + + +@api_router.post( + "/organizations/{organization_id}/granularScans/{scan_id}/update", + dependencies=[Depends(get_current_active_user)], + response_model=OrganizationSchema.UpdateOrgScanSchema, + tags=["Organizations"], +) +async def update_granular_scan( + organization_id: str, + scan_id: str, + scan_data: OrganizationSchema.NewOrgScan, + current_user: User = Depends(get_current_active_user), +): + """Update a granular scan for an organization.""" + return organization.update_org_scan( + organization_id, scan_id, scan_data, current_user + ) + + +@api_router.get( + "/v2/organizations", + dependencies=[Depends(get_current_active_user)], + response_model=List[OrganizationSchema.GetOrganizationSchema], + tags=["Organizations"], +) +async def list_organizations_v2( + state: Optional[List[str]] = Query(None), + regionId: Optional[List[str]] = Query(None), + current_user: User = Depends(get_current_active_user), +): + """Retrieve a list of all organizations (version 2).""" + return organization.list_organizations_v2(state, regionId, current_user) From fd8aadce1375d21d8e3a452849a323d5586cbd95 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Wed, 23 Oct 2024 10:28:15 -0400 Subject: [PATCH 069/314] Fix merge and pre-commit issues --- backend/src/xfd_django/xfd_api/auth.py | 12 ++++++++++-- backend/src/xfd_django/xfd_api/views.py | 26 +------------------------ 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index b9372517..8684b9be 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -19,7 +19,7 @@ import requests # from .helpers import user_to_dict -from .models import ApiKey, Organization, OrganizationTag, User +from .models import ApiKey, Organization, OrganizationTag, Role, User # JWT_ALGORITHM = "RS256" JWT_SECRET = os.getenv("JWT_SECRET") @@ -480,4 +480,12 @@ def get_tag_organizations(current_user, tag_id: str) -> list[str]: return [org.id for org in tag.organizations.all()] # Return an empty list if tag is not found - return [] \ No newline at end of file + return [] + + +def get_org_memberships(current_user) -> list[str]: + """Returns the organization IDs that a user is a member of.""" + roles = Role.objects.filter(userId=current_user) + if not roles: + return [] + return [role.organizationId.id for role in roles if role.organizationId] diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 25fd25b4..c2dcff4f 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -30,12 +30,11 @@ from .api_methods import api_key as api_key_methods from .api_methods import auth as auth_methods from .api_methods import notification as notification_methods -from .api_methods import scan, organization +from .api_methods import organization, scan from .api_methods.api_keys import get_api_keys from .api_methods.cpe import get_cpes_by_id from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import export_domains, get_domain_by_id, search_domains -from .api_methods.organization import get_organizations, read_orgs from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user @@ -50,7 +49,6 @@ from .schema_models.domain import Domain as DomainSchema from .schema_models.domain import DomainFilters, DomainSearch from .schema_models.notification import Notification as NotificationSchema -from .schema_models.organization import Organization as OrganizationSchema from .schema_models.role import Role as RoleSchema from .schema_models.user import User as UserSchema from .schema_models.vulnerability import Vulnerability as VulnerabilitySchema @@ -345,27 +343,6 @@ async def delete_api_key( return api_key_methods.delete(id, current_user) -@api_router.get( - "/organizations/{regionId}", - response_model=List[OrganizationSchema], - # dependencies=[Depends(get_current_active_user)], - tags=["Organization"], -) -async def call_get_organizations(regionId): - """ - List all organizations with query parameters. - Args: - regionId : Region IDs to filter organizations by. - - Raises: - HTTPException: If the user is not authorized or no organizations are found. - - Returns: - List[Organizations]: A list of organizations matching the filter criteria. - """ - return get_organizations(regionId) - - # GET ALL @api_router.get("/api-keys", response_model=List[ApiKeySchema], tags=["api-keys"]) async def get_all_api_keys(current_user: User = Depends(get_current_active_user)): @@ -541,7 +518,6 @@ async def invoke_scheduler(current_user: User = Depends(get_current_active_user) return response - # ======================================== # Organization Endpoints # ======================================== From 508929467709798091ef33ff3b48abeed7182a57 Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Wed, 23 Oct 2024 14:05:36 -0400 Subject: [PATCH 070/314] Added POST request logic for saved search endpoint; Added user authentication to POST request; Modified response model variables to match database --- .../xfd_api/api_methods/saved_search.py | 88 +++++++++++-------- .../xfd_api/schema_models/saved_search.py | 14 +-- backend/src/xfd_django/xfd_api/views.py | 15 +++- 3 files changed, 72 insertions(+), 45 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py index 82c09537..45a4d289 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -2,7 +2,7 @@ # Standard Python Libraries -from datetime import datetime +from datetime import datetime, timezone import json import uuid @@ -10,24 +10,42 @@ from django.http import JsonResponse from fastapi import HTTPException -from ..models import SavedSearch +from ..models import SavedSearch, User from ..schema_models.saved_search import SavedSearchFilters +def empty_string_check(name): + if name == "": + raise HTTPException(status_code=400, detail="Name cannot be empty") + + def create_saved_search(request): - data = json.loads(request.body) + try: + user = User.objects.get(id=request.get("createdById")) + except User.DoesNotExist: + raise HTTPException(status_code=404, detail="User not found") + search = SavedSearch.objects.create( - name=data["name"], - count=data["count"], - sort_direction="asc", - sort_field="name", - search_term=data["searchTerm"], - search_path=data["searchPath"], - filters=data["filters"], - create_vulnerabilities=data["createVulnerabilities"], - vulnerability_template=data.get("vulnerabilityTemplate"), - created_by=request.user, + id=uuid.uuid4(), + createdAt=datetime.now(timezone.utc), + updatedAt=datetime.now(timezone.utc), + name=request.get("name"), + count=request.get("count", 0), # Default to 0 if count does not exist + sortDirection=request.get("sortDirection", ""), + sortField=request.get("sortField", ""), + searchTerm=request.get("searchTerm", ""), + searchPath=request.get("searchPath", ""), + filters=[ + { + "type": "any", + "field": request.get("field", "organization.regionId"), + "values": [request.get("regionId")], + } + ], + createdById=user, ) + + search.save() return JsonResponse({"status": "Created", "search": search.id}, status=201) @@ -48,16 +66,16 @@ def list_saved_searches(): response = { "id": str(search.id), - "created_at": search.createdAt, - "updated_at": search.updatedAt, + "createdAt": search.createdAt, + "updatedAt": search.updatedAt, "name": search.name, - "search_term": search.searchTerm, - "sort_direction": search.sortDirection, - "sort_field": search.sortField, + "searchTerm": search.searchTerm, + "sortDirection": search.sortDirection, + "sortField": search.sortField, "count": search.count, "filters": filters_without_type, - "search_path": search.searchPath, - "createdBy_id": search.createdById.id, + "searchPath": search.searchPath, + "createdById": search.createdById.id, } saved_search_list.append(response) @@ -74,16 +92,16 @@ def get_saved_search(search_id): saved_search = SavedSearch.objects.get(id=search_id) response = { "id": str(saved_search.id), - "created_at": saved_search.createdAt, - "updated_at": saved_search.updatedAt, + "createdAt": saved_search.createdAt, + "updatedAt": saved_search.updatedAt, "name": saved_search.name, - "search_term": saved_search.searchTerm, - "sort_direction": saved_search.sortDirection, - "sort_field": saved_search.sortField, + "searchTerm": saved_search.searchTerm, + "sortDirection": saved_search.sortDirection, + "sortField": saved_search.sortField, "count": saved_search.count, "filters": saved_search.filters, - "search_path": saved_search.searchPath, - "createdBy_id": saved_search.createdById.id, + "searchPath": saved_search.searchPath, + "createdById": saved_search.createdById.id, } return response except SavedSearch.DoesNotExist as e: @@ -103,21 +121,21 @@ def update_saved_search(request): saved_search.name = request["name"] saved_search.updatedAt = datetime.now() - saved_search.searchTerm = request["search_term"] + saved_search.searchTerm = request["searchTerm"] saved_search.save() response = { "id": saved_search.id, - "created_at": saved_search.createdAt, - "updated_at": saved_search.updatedAt, + "createdAt": saved_search.createdAt, + "updatedAt": saved_search.updatedAt, "name": saved_search.name, - "search_term": saved_search.searchTerm, - "sort_direction": saved_search.sortDirection, - "sort_field": saved_search.sortField, + "searchTerm": saved_search.searchTerm, + "sortDirection": saved_search.sortDirection, + "sortField": saved_search.sortField, "count": saved_search.count, "filters": saved_search.filters, - "search_path": saved_search.searchPath, - "createdBy_id": saved_search.createdById.id, + "searchPath": saved_search.searchPath, + "createdById": saved_search.createdById.id, } return response diff --git a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py index db7ab162..42be3df2 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/saved_search.py +++ b/backend/src/xfd_django/xfd_api/schema_models/saved_search.py @@ -19,13 +19,13 @@ class SavedSearch(BaseModel): """SavedSearch schema.""" id: UUID - created_at: datetime - updated_at: datetime + createdAt: datetime + updatedAt: datetime name: str - search_term: str = "" - sort_direction: str - sort_field: str + searchTerm: str = "" + sortDirection: str + sortField: str count: int filters: List[SavedSearchFilters] - search_path: str - createdBy_id: UUID + searchPath: str + createdById: UUID diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 3c51136a..2769aae7 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -411,15 +411,24 @@ async def call_get_organizations(regionId): # TODO: Implement the following functions @api_router.post( "/saved-searches", + dependencies=[Depends(get_current_active_user)], + # response_model=SavedSearchSchema, tags=["Testing"], ) async def call_create_saved_search( name: str, search_term: str, - region_id: int, + region_id: str, + current_user: User = Depends(get_current_active_user), ): + request = { + "name": name, + "searchTerm": search_term, + "regionId": region_id, + "createdById": current_user.id, + } """Create a new saved search.""" - return {"status": "ok"} + return create_saved_search(request) # Get all existing saved searches is implemented in the following function @@ -461,7 +470,7 @@ async def call_update_saved_search( request = { "name": name, "saved_search_id": saved_search_id, - "search_term": search_term, + "searchTerm": search_term, } return update_saved_search(request) From 9d305d4203cbc49627ad394ccf177b5e69e8fcd6 Mon Sep 17 00:00:00 2001 From: Chrtorres Date: Wed, 23 Oct 2024 14:39:20 -0400 Subject: [PATCH 071/314] Added name validation for saved search endpoints --- .../xfd_api/api_methods/saved_search.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py index 45a4d289..0fcb55c2 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/saved_search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/saved_search.py @@ -14,10 +14,15 @@ from ..schema_models.saved_search import SavedSearchFilters -def empty_string_check(name): - if name == "": +def validate_name(name: str): + if name.strip() == "": raise HTTPException(status_code=400, detail="Name cannot be empty") + all_saved_searches = SavedSearch.objects.all() + for search in all_saved_searches: + if search.name.strip() == name.strip(): + raise HTTPException(status_code=400, detail="Name already exists") + def create_saved_search(request): try: @@ -25,6 +30,9 @@ def create_saved_search(request): except User.DoesNotExist: raise HTTPException(status_code=404, detail="User not found") + all_saved_searches = SavedSearch.objects.all() + validate_name(request.get("name")) + search = SavedSearch.objects.create( id=uuid.uuid4(), createdAt=datetime.now(timezone.utc), @@ -120,8 +128,9 @@ def update_saved_search(request): return HTTPException(status_code=404, detail=str(e)) saved_search.name = request["name"] - saved_search.updatedAt = datetime.now() + saved_search.updatedAt = datetime.now(timezone.utc) saved_search.searchTerm = request["searchTerm"] + validate_name(request.get("name")) saved_search.save() response = { From 5723996d0552110303a54e01a259914ccd574bc4 Mon Sep 17 00:00:00 2001 From: nickviola Date: Wed, 23 Oct 2024 13:48:54 -0500 Subject: [PATCH 072/314] Add tests and update search logic for export --- .../xfd_django/xfd_api/api_methods/search.py | 31 +++++------ .../xfd_django/xfd_api/tests/test_search.py | 52 +++++++++++++++++++ 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/search.py b/backend/src/xfd_django/xfd_api/api_methods/search.py index 305b9dd1..1ef8a4db 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/search.py @@ -38,17 +38,17 @@ class SearchBody(BaseModel): def get_options(search_body: SearchBody, event) -> Dict[str, Any]: """Determine options for filtering based on organization ID or tag ID.""" - if search_body.organizationId and ( - search_body.organizationId in get_org_memberships(event) + if search_body.organization_id and ( + search_body.organization_id in get_org_memberships(event) or is_global_view_admin(event) ): options = { - "organizationIds": [search_body.organizationId], + "organizationIds": [search_body.organization_id], "matchAllOrganizations": False, } - elif search_body.tagId: + elif search_body.tag_id: options = { - "organizationIds": get_tag_organizations(event, search_body.tagId), + "organizationIds": get_tag_organizations(event, str(search_body.tag_id)), "matchAllOrganizations": False, } else: @@ -119,18 +119,19 @@ def export(search_body: SearchBody, event) -> Dict[str, Any]: writer.writeheader() writer.writerows(results) - # Save to S3 # TODO: Replace with heler logic - s3 = boto3.client("s3") - bucket_name = "your-bucket-name" - csv_key = "domains.csv" - s3.put_object(Bucket=bucket_name, Key=csv_key, Body=csv_buffer.getvalue()) + # Save to S3 # TODO: Replace with helper logic + # s3 = boto3.client("s3") + # bucket_name = "your-bucket-name" + # csv_key = "domains.csv" + # s3.put_object(Bucket=bucket_name, Key=csv_key, Body=csv_buffer.getvalue()) - # Generate a presigned URL to access the CSV - url = s3.generate_presigned_url( - "get_object", Params={"Bucket": bucket_name, "Key": csv_key}, ExpiresIn=3600 - ) + # # Generate a presigned URL to access the CSV + # url = s3.generate_presigned_url( + # "get_object", Params={"Bucket": bucket_name, "Key": csv_key}, ExpiresIn=3600 + # ) - return {"url": url} + # return {"url": url} + return {"data": results} def search(search_body: SearchBody, event) -> Dict[str, Any]: diff --git a/backend/src/xfd_django/xfd_api/tests/test_search.py b/backend/src/xfd_django/xfd_api/tests/test_search.py index e69de29b..7005d31c 100644 --- a/backend/src/xfd_django/xfd_api/tests/test_search.py +++ b/backend/src/xfd_django/xfd_api/tests/test_search.py @@ -0,0 +1,52 @@ +# test_search.py +# Third-Party Libraries +from fastapi.testclient import TestClient + +# from xfd_api.views import api_router +from xfd_django.asgi import app + +client = TestClient(app) + +# Example search body to use for testing +search_body = { + "current": 1, + "results_per_page": 10, + "search_term": "example search term", + "sort_direction": "asc", + "sort_field": "createdAt", + "filters": [{"field": "organization.id", "values": ["1", "2", "3"], "type": "any"}], + "organization_ids": ["f834b2f8-3c10-4d3a-8ec7-9fc57b88c9e5"], + "organization_id": "f834b2f8-3c10-4d3a-8ec7-9fc57b88c9e5", + "tag_id": "a93a5d1b-1234-4567-890a-bc1234567890", +} + + +# Test for /search endpoint +def test_search_endpoint(): + response = client.post("/search", json=search_body) + assert response.status_code == 200 # Expecting HTTP 200 OK + data = response.json() + assert "current" in data # Check if the expected field is in the response + assert data["current"] == 1 # Ensure the current value matches the request + + +# Test for /search/export endpoint +def test_search_export_endpoint(): + response = client.post("/search/export", json=search_body) + assert response.status_code == 200 # Expecting HTTP 200 OK + data = response.json() + assert "url" in data # Check if the response contains the CSV URL + + +# Test for invalid request to /search endpoint (missing required fields) +def test_search_invalid_request(): + invalid_body = {"current": 1} # Missing many required fields + response = client.post("/search", json=invalid_body) + assert response.status_code == 422 # Expecting validation error + + +# Test for invalid request to /search/export endpoint (missing required fields) +def test_search_export_invalid_request(): + invalid_body = {"current": 1} # Missing many required fields + response = client.post("/search/export", json=invalid_body) + assert response.status_code == 422 # Expecting validation error From 51a17743e494364293ba3940daa00853e100e2d1 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 24 Oct 2024 08:29:37 -0400 Subject: [PATCH 073/314] Add organization tests --- .../xfd_api/api_methods/organization.py | 76 +- backend/src/xfd_django/xfd_api/auth.py | 19 +- backend/src/xfd_django/xfd_api/models.py | 2 +- .../xfd_api/schema_models/organization.py | 9 +- .../xfd_api/tests/test_organization.py | 1182 +++++++++++++++++ backend/src/xfd_django/xfd_api/views.py | 2 +- 6 files changed, 1264 insertions(+), 26 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/tests/test_organization.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/organization.py b/backend/src/xfd_django/xfd_api/api_methods/organization.py index 1e14213f..bfb18a74 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/organization.py +++ b/backend/src/xfd_django/xfd_api/api_methods/organization.py @@ -161,6 +161,7 @@ def get_organization(organization_id, current_user): "county": organization.county, "countyFips": organization.countyFips, "type": organization.type, + "createdBy": organization.createdBy, "userRoles": [ { "id": str(role.id), @@ -224,9 +225,12 @@ def get_organization(organization_id, current_user): return org_data + except HTTPException as http_exc: + raise http_exc + except Exception as e: print(f"An error occurred: {e}") - raise HTTPException(status_code=500, detail="An unexpected error occurred") + raise HTTPException(status_code=500, detail=str(e)) def get_by_state(state, current_user): @@ -318,6 +322,8 @@ def get_all_regions(current_user): # Convert to a list and return the regions return list(regions) + except HTTPException as http_exc: + raise http_exc except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -349,7 +355,7 @@ def create_organization(organization_data, current_user): # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): raise HTTPException( - status_code=403, detail="Unauthorized access. View logs for details." + status_code=403, detail="Unauthorized access." ) # Prepare the organization data for creation @@ -395,6 +401,9 @@ def create_organization(organization_data, current_user): "county": organization.county, "countyFips": organization.countyFips, "type": organization.type, + "createdBy": { + "id": str(current_user.id), # Simplify to just the user ID + }, "tags": [ { "id": str(tag.id), @@ -412,6 +421,8 @@ def create_organization(organization_data, current_user): else {}, } + except HTTPException as http_exc: + raise http_exc except Organization.DoesNotExist: raise HTTPException(status_code=404, detail="Parent organization not found") except Exception as e: @@ -474,6 +485,7 @@ def upsert_organization(organization_data, current_user): "county": organization.county, "countyFips": organization.countyFips, "type": organization.type, + "createdBy": organization.createdBy, "tags": [ { "id": str(tag.id), @@ -491,6 +503,8 @@ def upsert_organization(organization_data, current_user): else {}, } + except HTTPException as http_exc: + raise http_exc except Organization.DoesNotExist: raise HTTPException(status_code=404, detail="Parent organization not found") except Exception as e: @@ -557,6 +571,7 @@ def update_organization(organization_id: str, organization_data, current_user): "county": organization.county, "countyFips": organization.countyFips, "type": organization.type, + "createdBy": organization.createdBy, "tags": [ { "id": str(tag.id), @@ -598,6 +613,9 @@ def update_organization(organization_id: str, organization_data, current_user): ], } + except HTTPException as http_exc: + raise http_exc + except Organization.DoesNotExist: raise HTTPException(status_code=404, detail="Organization not found") except Exception as e: @@ -631,6 +649,9 @@ def delete_organization(id: str, current_user): "message": f"Organization {id} has been deleted successfully.", } + except HTTPException as http_exc: + raise http_exc + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -704,6 +725,9 @@ def add_user_to_org_v2(organization_id: str, user_data, current_user): }, } + except HTTPException as http_exc: + raise http_exc + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) @@ -738,6 +762,8 @@ def approve_role(organization_id: str, role_id, current_user): raise HTTPException(status_code=404, detail="Role not found") + except HTTPException as http_exc: + raise http_exc except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -769,6 +795,9 @@ def remove_role(organization_id: str, role_id, current_user): return {"status": "success", "message": "Role removed successfully"} + except HTTPException as http_exc: + raise http_exc + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -830,12 +859,44 @@ def update_org_scan(organization_id: str, scan_id, scan_data, current_user): # Return a success response return { - "status": "success", - "organization_id": organization_id, - "scan_id": scan_id, - "enabled": enabled, + "id": str(organization.id), + "createdAt": organization.createdAt.isoformat(), + "updatedAt": organization.updatedAt.isoformat(), + "acronym": organization.acronym, + "name": organization.name, + "rootDomains": organization.rootDomains, + "ipBlocks": organization.ipBlocks, + "isPassive": organization.isPassive, + "pendingDomains": organization.pendingDomains, + "country": organization.country, + "state": organization.state, + "regionId": organization.regionId, + "stateFips": organization.stateFips, + "stateName": organization.stateName, + "county": organization.county, + "countyFips": organization.countyFips, + "type": organization.type, + "granularScans": [ + { + "id": str(scan.id), + "createdAt": scan.createdAt.isoformat(), + "updatedAt": scan.updatedAt.isoformat(), + "name": scan.name, + "arguments": scan.arguments, + "frequency": scan.frequency, + "lastRun": scan.lastRun.isoformat() if scan.lastRun else None, + "isGranular": scan.isGranular, + "isUserModifiable": scan.isUserModifiable, + "isSingleScan": scan.isSingleScan, + "manualRunPending": scan.manualRunPending, + } + for scan in organization.granularScans.all() + ], } + except HTTPException as http_exc: + raise http_exc + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -905,6 +966,9 @@ def list_organizations_v2(state, regionId, current_user): return organization_list + except HTTPException as http_exc: + raise http_exc + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 8684b9be..312cc767 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -1,17 +1,17 @@ """Authentication utilities for the FastAPI application.""" # Standard Python Libraries -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import hashlib from hashlib import sha256 import os from urllib.parse import urlencode import uuid +from typing import Optional # Third-Party Libraries from django.conf import settings from django.forms.models import model_to_dict -from django.utils import timezone from fastapi import Depends, HTTPException, Security, status from fastapi.security import APIKeyHeader, OAuth2PasswordBearer import jwt @@ -64,7 +64,7 @@ def create_jwt_token(user): payload = { "id": str(user.id), "email": user.email, - "exp": datetime.now(datetime.timezone.utc) + timedelta(hours=JWT_TIMEOUT_HOURS), + "exp": datetime.now(timezone.utc) + timedelta(hours=JWT_TIMEOUT_HOURS), } return jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) @@ -200,7 +200,7 @@ def get_user_by_api_key(api_key: str): hashed_key = sha256(api_key.encode()).hexdigest() try: api_key_instance = ApiKey.objects.get(hashedKey=hashed_key) - api_key_instance.lastUsed = timezone.now() + api_key_instance.lastUsed = datetime.now(timezone.utc) api_key_instance.save(update_fields=["lastUsed"]) return api_key_instance.userId except ApiKey.DoesNotExist: @@ -209,8 +209,8 @@ def get_user_by_api_key(api_key: str): def get_current_active_user( - api_key: str = Security(api_key_header), - token: str = Depends(oauth2_scheme), + api_key: Optional[str] = Security(api_key_header), + token: Optional[str] = Depends(oauth2_scheme) ): """Ensure the current user is authenticated and active.""" user = None @@ -231,7 +231,6 @@ def get_current_active_user( ) # Fetch the user by ID from the database user = User.objects.get(id=user_id) - print(f"User found: {user_to_dict(user)}") except jwt.ExpiredSignatureError: print("Token has expired") raise HTTPException( @@ -431,7 +430,7 @@ def is_org_admin(current_user, organization_id) -> bool: # Check if the user has an admin role in the given organization for role in current_user.roles.all(): - if role.organization.id == organization_id and role.role == "admin": + if str(role.organization.id) == str(organization_id) and role.role == "admin": return True # If the user is a global write admin, they are considered an org admin @@ -485,7 +484,7 @@ def get_tag_organizations(current_user, tag_id: str) -> list[str]: def get_org_memberships(current_user) -> list[str]: """Returns the organization IDs that a user is a member of.""" - roles = Role.objects.filter(userId=current_user) + roles = Role.objects.filter(user=current_user) if not roles: return [] - return [role.organizationId.id for role in roles if role.organizationId] + return [role.organization.id for role in roles if role.organization] diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index 89e73055..b9f044c5 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -685,7 +685,7 @@ class User(models.Model): state = models.CharField(blank=True, null=True, max_length=255) def save(self, *args, **kwargs): - self.full_name = f"{self.first_name} {self.last_name}" + self.fullName = f"{self.firstName} {self.lastName}" super().save(*args, **kwargs) class Meta: diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization.py b/backend/src/xfd_django/xfd_api/schema_models/organization.py index 625d0ce5..b35b7212 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/organization.py +++ b/backend/src/xfd_django/xfd_api/schema_models/organization.py @@ -125,6 +125,7 @@ class GetSingleOrganizationSchema(BaseModel): rootDomains: List[str] ipBlocks: List[str] isPassive: bool + createdBy: Optional[Any] = {} pendingDomains: Optional[Any] = [] country: Optional[str] = None state: Optional[str] = None @@ -201,11 +202,3 @@ class GenericPostResponseModel(BaseModel): statusCode: int body: Any - -class UpdateOrgScanSchema(BaseModel): - """Update an org's scan model.""" - - status: str - organization_id: UUID - scan_id: UUID - enabled: bool diff --git a/backend/src/xfd_django/xfd_api/tests/test_organization.py b/backend/src/xfd_django/xfd_api/tests/test_organization.py new file mode 100644 index 00000000..60230871 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests/test_organization.py @@ -0,0 +1,1182 @@ +import secrets +from fastapi.testclient import TestClient +import pytest +from datetime import datetime +from xfd_api.auth import create_jwt_token +from xfd_api.models import User, Organization, UserType, Role, Scan, ScanTask, OrganizationTag +from xfd_django.asgi import app + +client = TestClient(app) + +# Test: Creating an organization by global admin should succeed +@pytest.mark.django_db(transaction=True) +def test_create_org_by_global_admin(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + name = f"test-{secrets.token_hex(4)}" + acronym = secrets.token_hex(2) + + response = client.post( + "/organizations/", + json={ + "ipBlocks": [], + "acronym": acronym, + "name": name, + "rootDomains": ["cisa.gov"], + "isPassive": False, + "tags": [{"name": "test"}] + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["createdBy"]["id"] == str(user.id) + assert data["name"] == name + assert data["tags"][0]["name"] == "test" + +# Test: Cannot add organization with the same acronym +@pytest.mark.django_db(transaction=True) +def test_create_duplicate_org_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + name = f"test-{secrets.token_hex(4)}" + acronym = secrets.token_hex(2) + + client.post( + "/organizations/", + json={ + "ipBlocks": [], + "acronym": acronym, + "name": name, + "rootDomains": ["cisa.gov"], + "isPassive": False, + "tags": [] + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + # Attempt to create another organization with the same acronym + response = client.post( + "/organizations/", + json={ + "ipBlocks": [], + "acronym": acronym, + "name": name, + "rootDomains": ["cisa.gov"], + "isPassive": False, + "tags": [] + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 500 + +# Test: Creating an organization by global view user should fail +@pytest.mark.django_db(transaction=True) +def test_create_org_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + print(user) + + name = f"test-{secrets.token_hex(4)}" + acronym = secrets.token_hex(2) + + response = client.post( + "/organizations/", + json={ + "ipBlocks": [], + "acronym": acronym, + "name": name, + "rootDomains": ["cisa.gov"], + "isPassive": False, + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + +# Test: Update organization by global admin +@pytest.mark.django_db(transaction=True) +def test_update_org_by_global_admin(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + acronym=secrets.token_hex(2), + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test.com"], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + new_name = f"test-{secrets.token_hex(4)}" + new_acronym = secrets.token_hex(2) + new_root_domains = ["newdomain.com"] + new_ip_blocks = ["1.1.1.1"] + is_passive = True + tags = [{"name": "updated"}] + + response = client.put( + f"/organizations/{organization.id}", + json={ + "name": new_name, + "acronym": new_acronym, + "rootDomains": new_root_domains, + "ipBlocks": new_ip_blocks, + "isPassive": is_passive, + "tags": tags, + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["name"] == new_name + assert data["rootDomains"] == new_root_domains + assert data["ipBlocks"] == new_ip_blocks + assert data["isPassive"] == is_passive + assert data["tags"][0]["name"] == tags[0]["name"] + +# Test: Update organization by global view should fail +@pytest.mark.django_db(transaction=True) +def test_update_org_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + acronym=secrets.token_hex(2), + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test.com"], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + new_name = f"test-{secrets.token_hex(4)}" + new_acronym = secrets.token_hex(2) + new_root_domains = ["newdomain.com"] + new_ip_blocks = ["1.1.1.1"] + is_passive = True + tags = [{"name": "updated"}] + + response = client.put( + f"/organizations/{organization.id}", + json={ + "name": new_name, + "acronym": new_acronym, + "rootDomains": new_root_domains, + "ipBlocks": new_ip_blocks, + "isPassive": is_passive, + "tags": tags, + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + +# Test: Deleting an organization by global admin should succeed +@pytest.mark.django_db(transaction=True) +def test_delete_org_by_global_admin(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test.com"], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.delete( + f"/organizations/{organization.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + +# Test: Deleting an organization by org admin should fail +@pytest.mark.django_db(transaction=True) +def test_delete_org_by_org_admin_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test.com"], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Assign admin role to the user for the organization + Role.objects.create( + user=user, + organization=organization, + role="admin", + ) + + response = client.delete( + f"/organizations/{organization.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + +# Test: Deleting an organization by global view should fail +@pytest.mark.django_db(transaction=True) +def test_delete_org_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + acronym=secrets.token_hex(2), + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test.com"], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.delete( + f"/organizations/{organization.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + +# Test: List organizations by global view should succeed +@pytest.mark.django_db(transaction=True) +def test_list_orgs_by_global_view_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Create an organization + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.get( + "/organizations", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data) >= 1 + +# Test: List organizations by org member should only return their org +@pytest.mark.django_db(transaction=True) +def test_list_orgs_by_org_member_only_gets_their_org(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Create organizations + organization1 = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization2 = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Assign user a role in organization1 + Role.objects.create( + user=user, + organization=organization1, + role="user", + ) + + # Fetch organizations + response = client.get( + "/organizations", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data) == 1 + assert data[0]["id"] == str(organization1.id) + + +# Test: Get organization by global view should fail +@pytest.mark.django_db(transaction=True) +def test_get_org_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.get( + f"/organizations/{organization.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized'} + +# Test: Get organization by org admin user should pass +@pytest.mark.django_db(transaction=True) +def test_get_org_by_org_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Assign admin role to the user for the organization + Role.objects.create( + user=user, + organization=organization, + role="admin", + ) + + response = client.get( + f"/organizations/{organization.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["name"] == organization.name + +# Test: Get organization by org admin of different org should fail +@pytest.mark.django_db(transaction=True) +def test_get_org_by_org_admin_of_different_org_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization1 = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization2 = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Assign admin role to the user for organization1 + Role.objects.create( + user=user, + organization=organization1, + role="admin", + ) + + response = client.get( + f"/organizations/{organization2.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized'} + +# Test: Get organization by org regular user should fail +@pytest.mark.django_db(transaction=True) +def test_get_org_by_org_regular_user_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Assign regular user role to the user for the organization + Role.objects.create( + user=user, + organization=organization, + role="user", + ) + + response = client.get( + f"/organizations/{organization.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized'} + +# Test: Get organization by org admin should return associated scantasks +@pytest.mark.django_db(transaction=True) +def test_get_org_with_scan_tasks_by_org_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Assign admin role to the user for the organization + Role.objects.create( + user=user, + organization=organization, + role="admin", + ) + + # Create a scan and scantask associated with the organization + scan = Scan.objects.create( + name="censys", + arguments={}, + frequency=999999, + ) + + scan_task = ScanTask.objects.create( + scan=scan, + status="created", + type="fargate" + ) + + scan_task.organizations.add(organization) + + response = client.get( + f"/organizations/{organization.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["name"] == organization.name + assert len(data["scanTasks"]) == 1 + assert data["scanTasks"][0]["id"] == str(scan_task.id) + assert data["scanTasks"][0]["scan"]["id"] == str(scan.id) + +# Test: Enabling a user-modifiable scan by org admin should succeed +@pytest.mark.django_db(transaction=True) +def test_enable_user_modifiable_scan_by_org_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + Role.objects.create( + user=user, + organization=organization, + role="admin", + ) + + scan = Scan.objects.create( + name="censys", + arguments={}, + frequency=999999, + isGranular=True, + isUserModifiable=True, + ) + + response = client.post( + f"/organizations/{organization.id}/granularScans/{scan.id}/update", + json={"enabled": True}, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data["granularScans"]) == 1 + assert data["granularScans"][0]["id"] == str(scan.id) + + +# Test: Disabling a user-modifiable scan by org admin should succeed +@pytest.mark.django_db(transaction=True) +def test_disable_user_modifiable_scan_by_org_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + Role.objects.create( + user=user, + organization=organization, + role="admin", + ) + + scan = Scan.objects.create( + name="censys", + arguments={}, + frequency=999999, + isGranular=True, + isUserModifiable=True, + ) + + scan_task = ScanTask.objects.create( + scan=scan, + status="created", + type="fargate", + ) + scan_task.organizations.add(organization) + + response = client.post( + f"/organizations/{organization.id}/granularScans/{scan.id}/update", + json={"enabled": False}, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data["granularScans"]) == 0 + + +# Test: Enabling a user-modifiable scan by org user should fail +@pytest.mark.django_db(transaction=True) +def test_enable_user_modifiable_scan_by_org_user_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + Role.objects.create( + user=user, + organization=organization, + role="user", + ) + + scan = Scan.objects.create( + name="censys", + arguments={}, + frequency=999999, + isGranular=True, + isUserModifiable=True, + ) + + response = client.post( + f"/organizations/{organization.id}/granularScans/{scan.id}/update", + json={"enabled": True}, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + + +# Test: Enabling a user-modifiable scan by global admin should succeed +@pytest.mark.django_db(transaction=True) +def test_enable_user_modifiable_scan_by_global_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create( + name="censys", + arguments={}, + frequency=999999, + isGranular=True, + isUserModifiable=True, + ) + + response = client.post( + f"/organizations/{organization.id}/granularScans/{scan.id}/update", + json={"enabled": True}, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data["granularScans"]) == 1 + assert data["granularScans"][0]["id"] == str(scan.id) + + +# Test: Enabling a non-user-modifiable scan by org admin should fail +@pytest.mark.django_db(transaction=True) +def test_enable_non_user_modifiable_scan_by_org_admin_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + Role.objects.create( + user=user, + organization=organization, + role="admin", + ) + + scan = Scan.objects.create( + name="censys", + arguments={}, + frequency=999999, + isGranular=True, + isUserModifiable=False, # Not user-modifiable + ) + + response = client.post( + f"/organizations/{organization.id}/granularScans/{scan.id}/update", + json={"enabled": True}, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 404 + + +# Test: Approving a role by global admin should succeed +@pytest.mark.django_db(transaction=True) +def test_approve_role_by_global_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + role = Role.objects.create( + role="user", + approved=False, + organization=organization, + ) + + response = client.post( + f"/organizations/{organization.id}/roles/{role.id}/approve", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + role.refresh_from_db() + assert role.approved is True + + +# Test: Approving a role by global view should fail +@pytest.mark.django_db(transaction=True) +def test_approve_role_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + role = Role.objects.create( + role="user", + approved=False, + organization=organization, + ) + + response = client.post( + f"/organizations/{organization.id}/roles/{role.id}/approve", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + role.refresh_from_db() + assert role.approved is False + + +# Test: Approving a role by org admin should succeed +@pytest.mark.django_db(transaction=True) +def test_approve_role_by_org_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + Role.objects.create( + user=user, + organization=organization, + role="admin", + ) + + role = Role.objects.create( + role="user", + approved=False, + organization=organization, + ) + + response = client.post( + f"/organizations/{organization.id}/roles/{role.id}/approve", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + role.refresh_from_db() + assert role.approved is True + + +# Test: Approving a role by org user should fail +@pytest.mark.django_db(transaction=True) +def test_approve_role_by_org_user_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + Role.objects.create( + user=user, + organization=organization, + role="user", + ) + + role = Role.objects.create( + role="user", + approved=False, + organization=organization, + ) + + response = client.post( + f"/organizations/{organization.id}/roles/{role.id}/approve", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + role.refresh_from_db() + assert role.approved is False + + +# Test: removeRole by globalAdmin should work +@pytest.mark.django_db(transaction=True) +def test_remove_role_by_global_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + role = Role.objects.create( + role="user", + approved=False, + organization=organization, + ) + + response = client.post( + f"/organizations/{organization.id}/roles/{role.id}/remove", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + + +# Test: removeRole by globalView should fail +@pytest.mark.django_db(transaction=True) +def test_remove_role_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + role = Role.objects.create( + role="user", + approved=False, + organization=organization, + ) + + response = client.post( + f"/organizations/{organization.id}/roles/{role.id}/remove", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + + +# Test: removeRole by org admin should succeed +@pytest.mark.django_db(transaction=True) +def test_remove_role_by_org_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + Role.objects.create( + user=user, + organization=organization, + role="admin", + ) + + role = Role.objects.create( + role="user", + approved=False, + organization=organization, + ) + + response = client.post( + f"/organizations/{organization.id}/roles/{role.id}/remove", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + + +# Test: removeRole by org user should fail +@pytest.mark.django_db(transaction=True) +def test_remove_role_by_org_user_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + Role.objects.create( + user=user, + organization=organization, + role="user", + ) + + role = Role.objects.create( + role="user", + approved=False, + organization=organization, + ) + + response = client.post( + f"/organizations/{organization.id}/roles/{role.id}/remove", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + + +# Test: getTags by globalAdmin should work +@pytest.mark.django_db(transaction=True) +def test_get_tags_by_global_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + OrganizationTag.objects.create( + name=f"test-{secrets.token_hex(4)}", + ) + + response = client.get( + "/organizations/tags", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + assert len(response.json()) >= 1 + + +# Test: getTags by standard user should return no tags +@pytest.mark.django_db(transaction=True) +def test_get_tags_by_standard_user_returns_no_tags(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + OrganizationTag.objects.create( + name=f"test-{secrets.token_hex(4)}", + ) + + response = client.get( + "/organizations/tags", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + assert len(response.json()) == 0 diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index c2dcff4f..2e8d5183 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -699,7 +699,7 @@ async def remove_role( @api_router.post( "/organizations/{organization_id}/granularScans/{scan_id}/update", dependencies=[Depends(get_current_active_user)], - response_model=OrganizationSchema.UpdateOrgScanSchema, + response_model=OrganizationSchema.GetSingleOrganizationSchema, tags=["Organizations"], ) async def update_granular_scan( From f690c01e8a9eb76e4ec2f9f1d7dd7a15472d977b Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 24 Oct 2024 08:30:42 -0400 Subject: [PATCH 074/314] Run pre-commits --- .../xfd_api/api_methods/organization.py | 18 ++--- backend/src/xfd_django/xfd_api/auth.py | 4 +- .../xfd_api/schema_models/organization.py | 1 - .../xfd_api/tests/test_organization.py | 78 ++++++++++++------- 4 files changed, 60 insertions(+), 41 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/organization.py b/backend/src/xfd_django/xfd_api/api_methods/organization.py index bfb18a74..9177e10c 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/organization.py +++ b/backend/src/xfd_django/xfd_api/api_methods/organization.py @@ -227,7 +227,7 @@ def get_organization(organization_id, current_user): except HTTPException as http_exc: raise http_exc - + except Exception as e: print(f"An error occurred: {e}") raise HTTPException(status_code=500, detail=str(e)) @@ -354,9 +354,7 @@ def create_organization(organization_data, current_user): try: # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): - raise HTTPException( - status_code=403, detail="Unauthorized access." - ) + raise HTTPException(status_code=403, detail="Unauthorized access.") # Prepare the organization data for creation organization_data_dict = organization_data.dict( @@ -615,7 +613,7 @@ def update_organization(organization_id: str, organization_data, current_user): except HTTPException as http_exc: raise http_exc - + except Organization.DoesNotExist: raise HTTPException(status_code=404, detail="Organization not found") except Exception as e: @@ -651,7 +649,7 @@ def delete_organization(id: str, current_user): except HTTPException as http_exc: raise http_exc - + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -727,7 +725,7 @@ def add_user_to_org_v2(organization_id: str, user_data, current_user): except HTTPException as http_exc: raise http_exc - + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) @@ -797,7 +795,7 @@ def remove_role(organization_id: str, role_id, current_user): except HTTPException as http_exc: raise http_exc - + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -896,7 +894,7 @@ def update_org_scan(organization_id: str, scan_id, scan_data, current_user): except HTTPException as http_exc: raise http_exc - + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -968,7 +966,7 @@ def list_organizations_v2(state, regionId, current_user): except HTTPException as http_exc: raise http_exc - + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 312cc767..3f676e0a 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -5,9 +5,9 @@ import hashlib from hashlib import sha256 import os +from typing import Optional from urllib.parse import urlencode import uuid -from typing import Optional # Third-Party Libraries from django.conf import settings @@ -210,7 +210,7 @@ def get_user_by_api_key(api_key: str): def get_current_active_user( api_key: Optional[str] = Security(api_key_header), - token: Optional[str] = Depends(oauth2_scheme) + token: Optional[str] = Depends(oauth2_scheme), ): """Ensure the current user is authenticated and active.""" user = None diff --git a/backend/src/xfd_django/xfd_api/schema_models/organization.py b/backend/src/xfd_django/xfd_api/schema_models/organization.py index b35b7212..1fdacd9a 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/organization.py +++ b/backend/src/xfd_django/xfd_api/schema_models/organization.py @@ -201,4 +201,3 @@ class GenericPostResponseModel(BaseModel): statusCode: int body: Any - diff --git a/backend/src/xfd_django/xfd_api/tests/test_organization.py b/backend/src/xfd_django/xfd_api/tests/test_organization.py index 60230871..440c1569 100644 --- a/backend/src/xfd_django/xfd_api/tests/test_organization.py +++ b/backend/src/xfd_django/xfd_api/tests/test_organization.py @@ -1,13 +1,25 @@ +# Standard Python Libraries +from datetime import datetime import secrets + +# Third-Party Libraries from fastapi.testclient import TestClient import pytest -from datetime import datetime from xfd_api.auth import create_jwt_token -from xfd_api.models import User, Organization, UserType, Role, Scan, ScanTask, OrganizationTag +from xfd_api.models import ( + Organization, + OrganizationTag, + Role, + Scan, + ScanTask, + User, + UserType, +) from xfd_django.asgi import app client = TestClient(app) + # Test: Creating an organization by global admin should succeed @pytest.mark.django_db(transaction=True) def test_create_org_by_global_admin(): @@ -19,10 +31,10 @@ def test_create_org_by_global_admin(): createdAt=datetime.now(), updatedAt=datetime.now(), ) - + name = f"test-{secrets.token_hex(4)}" acronym = secrets.token_hex(2) - + response = client.post( "/organizations/", json={ @@ -31,7 +43,7 @@ def test_create_org_by_global_admin(): "name": name, "rootDomains": ["cisa.gov"], "isPassive": False, - "tags": [{"name": "test"}] + "tags": [{"name": "test"}], }, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) @@ -42,6 +54,7 @@ def test_create_org_by_global_admin(): assert data["name"] == name assert data["tags"][0]["name"] == "test" + # Test: Cannot add organization with the same acronym @pytest.mark.django_db(transaction=True) def test_create_duplicate_org_fails(): @@ -53,10 +66,10 @@ def test_create_duplicate_org_fails(): createdAt=datetime.now(), updatedAt=datetime.now(), ) - + name = f"test-{secrets.token_hex(4)}" acronym = secrets.token_hex(2) - + client.post( "/organizations/", json={ @@ -65,7 +78,7 @@ def test_create_duplicate_org_fails(): "name": name, "rootDomains": ["cisa.gov"], "isPassive": False, - "tags": [] + "tags": [], }, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) @@ -79,13 +92,14 @@ def test_create_duplicate_org_fails(): "name": name, "rootDomains": ["cisa.gov"], "isPassive": False, - "tags": [] + "tags": [], }, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) assert response.status_code == 500 + # Test: Creating an organization by global view user should fail @pytest.mark.django_db(transaction=True) def test_create_org_by_global_view_fails(): @@ -98,7 +112,7 @@ def test_create_org_by_global_view_fails(): updatedAt=datetime.now(), ) print(user) - + name = f"test-{secrets.token_hex(4)}" acronym = secrets.token_hex(2) @@ -113,9 +127,10 @@ def test_create_org_by_global_view_fails(): }, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) - + assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} + # Test: Update organization by global admin @pytest.mark.django_db(transaction=True) @@ -167,6 +182,7 @@ def test_update_org_by_global_admin(): assert data["isPassive"] == is_passive assert data["tags"][0]["name"] == tags[0]["name"] + # Test: Update organization by global view should fail @pytest.mark.django_db(transaction=True) def test_update_org_by_global_view_fails(): @@ -210,7 +226,8 @@ def test_update_org_by_global_view_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} + # Test: Deleting an organization by global admin should succeed @pytest.mark.django_db(transaction=True) @@ -240,6 +257,7 @@ def test_delete_org_by_global_admin(): assert response.status_code == 200 + # Test: Deleting an organization by org admin should fail @pytest.mark.django_db(transaction=True) def test_delete_org_by_org_admin_fails(): @@ -274,7 +292,8 @@ def test_delete_org_by_org_admin_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} + # Test: Deleting an organization by global view should fail @pytest.mark.django_db(transaction=True) @@ -304,7 +323,8 @@ def test_delete_org_by_global_view_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} + # Test: List organizations by global view should succeed @pytest.mark.django_db(transaction=True) @@ -317,7 +337,7 @@ def test_list_orgs_by_global_view_succeeds(): createdAt=datetime.now(), updatedAt=datetime.now(), ) - + # Create an organization organization = Organization.objects.create( name=f"test-{secrets.token_hex(4)}", @@ -327,7 +347,7 @@ def test_list_orgs_by_global_view_succeeds(): createdAt=datetime.now(), updatedAt=datetime.now(), ) - + response = client.get( "/organizations", headers={"Authorization": "Bearer " + create_jwt_token(user)}, @@ -337,6 +357,7 @@ def test_list_orgs_by_global_view_succeeds(): data = response.json() assert len(data) >= 1 + # Test: List organizations by org member should only return their org @pytest.mark.django_db(transaction=True) def test_list_orgs_by_org_member_only_gets_their_org(): @@ -398,7 +419,7 @@ def test_get_org_by_global_view_fails(): createdAt=datetime.now(), updatedAt=datetime.now(), ) - + organization = Organization.objects.create( name=f"test-{secrets.token_hex(4)}", rootDomains=["test-" + secrets.token_hex(4)], @@ -414,7 +435,8 @@ def test_get_org_by_global_view_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized'} + assert response.json() == {"detail": "Unauthorized"} + # Test: Get organization by org admin user should pass @pytest.mark.django_db(transaction=True) @@ -453,6 +475,7 @@ def test_get_org_by_org_admin_succeeds(): data = response.json() assert data["name"] == organization.name + # Test: Get organization by org admin of different org should fail @pytest.mark.django_db(transaction=True) def test_get_org_by_org_admin_of_different_org_fails(): @@ -496,7 +519,8 @@ def test_get_org_by_org_admin_of_different_org_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized'} + assert response.json() == {"detail": "Unauthorized"} + # Test: Get organization by org regular user should fail @pytest.mark.django_db(transaction=True) @@ -532,7 +556,8 @@ def test_get_org_by_org_regular_user_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized'} + assert response.json() == {"detail": "Unauthorized"} + # Test: Get organization by org admin should return associated scantasks @pytest.mark.django_db(transaction=True) @@ -569,11 +594,7 @@ def test_get_org_with_scan_tasks_by_org_admin_succeeds(): frequency=999999, ) - scan_task = ScanTask.objects.create( - scan=scan, - status="created", - type="fargate" - ) + scan_task = ScanTask.objects.create(scan=scan, status="created", type="fargate") scan_task.organizations.add(organization) @@ -589,6 +610,7 @@ def test_get_org_with_scan_tasks_by_org_admin_succeeds(): assert data["scanTasks"][0]["id"] == str(scan_task.id) assert data["scanTasks"][0]["scan"]["id"] == str(scan.id) + # Test: Enabling a user-modifiable scan by org admin should succeed @pytest.mark.django_db(transaction=True) def test_enable_user_modifiable_scan_by_org_admin_succeeds(): @@ -1046,7 +1068,7 @@ def test_remove_role_by_global_view_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} # Test: removeRole by org admin should succeed @@ -1129,7 +1151,7 @@ def test_remove_role_by_org_user_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} # Test: getTags by globalAdmin should work From 6edae934ce944601e4c18b0d058d654a94408e97 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 24 Oct 2024 10:26:45 -0400 Subject: [PATCH 075/314] Add scan tasks tests --- .../xfd_api/api_methods/scan_tasks.py | 53 +-- .../xfd_api/schema_models/scan_tasks.py | 18 +- .../xfd_api/tests/test_scan_task.py | 318 ++++++++++++++++++ backend/src/xfd_django/xfd_api/views.py | 49 ++- 4 files changed, 405 insertions(+), 33 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/tests/test_scan_task.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py b/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py index a224fe7b..1f1b0f8e 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py +++ b/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py @@ -23,9 +23,13 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): status_code=403, detail="Unauthorized access. View logs for details." ) + # Ensure that search_data is not None, and set default values if it is + if search_data is None: + search_data = ScanTaskSearch(pageSize=PAGE_SIZE, page=1, sort="createdAt", order="DESC", filters={}) + # Validate and parse the request body pageSize = search_data.pageSize or PAGE_SIZE - + # Determine the correct ordering based on the 'order' field ordering_field = ( f"-{search_data.sort}" @@ -35,7 +39,7 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): # Construct query based on filters qs = ( - ScanTask.objects.select_related("scanId") + ScanTask.objects.select_related("scan") .prefetch_related("organizations") .order_by(ordering_field) ) @@ -44,17 +48,14 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): filters = search_data.filters if filters: if filters.get("name"): - qs = qs.filter(scanId__name__icontains=filters["name"]) + qs = qs.filter(scan__name__icontains=filters["name"]) if filters.get("status"): qs = qs.filter(status__icontains=filters["status"]) if filters.get("organization"): qs = qs.filter(organizations__id=filters["organization"]) if filters.get("tag"): - print("We are in tags") orgs = get_tag_organizations(current_user, filters["tag"]) - print(orgs) qs = qs.filter(organizations__id__in=orgs) - print(qs) # Paginate results if pageSize != -1: @@ -63,25 +64,25 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): # Convert queryset into a serialized response results = [] for task in qs: - # Ensure scanId is not None before accessing its properties - if task.scanId is None: - print(f"Warning: ScanTask {task.id} has no scanId associated.") + # Ensure scan is not None before accessing its properties + if task.scan is None: + print(f"Warning: ScanTask {task.id} has no scan associated.") scan_data = None # or some default values, depending on how you want to handle this case else: scan_data = { - "id": str(task.scanId.id), - "createdAt": task.scanId.createdAt.isoformat() + "Z", - "updatedAt": task.scanId.updatedAt.isoformat() + "Z", - "name": task.scanId.name, - "arguments": task.scanId.arguments, - "frequency": task.scanId.frequency, - "lastRun": task.scanId.lastRun.isoformat() + "Z" - if task.scanId.lastRun + "id": str(task.scan.id), + "createdAt": task.scan.createdAt.isoformat() + "Z", + "updatedAt": task.scan.updatedAt.isoformat() + "Z", + "name": task.scan.name, + "arguments": task.scan.arguments, + "frequency": task.scan.frequency, + "lastRun": task.scan.lastRun.isoformat() + "Z" + if task.scan.lastRun else None, - "isGranular": task.scanId.isGranular, - "isUserModifiable": task.scanId.isUserModifiable, - "isSingleScan": task.scanId.isSingleScan, - "manualRunPending": task.scanId.manualRunPending, + "isGranular": task.scan.isGranular, + "isUserModifiable": task.scan.isUserModifiable, + "isSingleScan": task.scan.isSingleScan, + "manualRunPending": task.scan.manualRunPending, } results.append( { @@ -133,8 +134,10 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): count = qs.count() response = {"result": results, "count": count} - return response + + except HTTPException as http_exc: + raise http_exc except Exception as e: print(e) @@ -170,6 +173,9 @@ def kill_scan_task(scan_task_id, current_user): return {"statusCode": 200, "message": "ScanTask successfully marked as failed."} + except HTTPException as http_exc: + raise http_exc + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) @@ -202,6 +208,9 @@ def get_scan_task_logs(scan_task_id, current_user): return Response(content=logs or "", status_code=status.HTTP_200_OK) + except HTTPException as http_exc: + raise http_exc + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py b/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py index 3f4cfd16..f40eda34 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py +++ b/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py @@ -2,7 +2,7 @@ # Standard Python Libraries from datetime import datetime -from typing import Any, List, Optional +from typing import Any, List, Optional, Dict from uuid import UUID # Third-Party Libraries @@ -15,11 +15,11 @@ class ScanTaskSearch(BaseModel): """Scan-task search schema.""" - page: int - pageSize: int - sort: str - order: str - filters: Any + page: Optional[int] = 1 + pageSize: Optional[int] = 10 + sort: Optional[str] = "createdAt" + order: Optional[str] = "DESC" + filters: Optional[Dict[str, Optional[str]]] = {} class ScanTaskList(BaseModel): @@ -30,8 +30,8 @@ class ScanTaskList(BaseModel): updatedAt: datetime status: str type: str - fargateTaskArn: str - input: str + fargateTaskArn: Optional[str] + input: Optional[str] output: Optional[str] requestedAt: Optional[datetime] startedAt: Optional[datetime] @@ -44,7 +44,7 @@ class ScanTaskList(BaseModel): class ScanTaskListResponse(BaseModel): """Scan-task list schema.""" - result: List[ScanTaskList] + result: List[ScanTaskList] = [] count: int diff --git a/backend/src/xfd_django/xfd_api/tests/test_scan_task.py b/backend/src/xfd_django/xfd_api/tests/test_scan_task.py new file mode 100644 index 00000000..0a31a1aa --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests/test_scan_task.py @@ -0,0 +1,318 @@ +from unittest.mock import patch +import secrets +from fastapi.testclient import TestClient +import pytest +from datetime import datetime +from xfd_api.auth import create_jwt_token +from xfd_api.models import User, Organization, UserType, ScanTask, Scan, Role +from xfd_django.asgi import app + +client = TestClient(app) + + +# Test: list by globalView should return scan tasks +@pytest.mark.django_db(transaction=True) +def test_list_scan_tasks_by_global_view(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) + scan_task = ScanTask.objects.create( + scan=scan, type="fargate", status="failed" + ) + scan_task.organizations.add(organization) + + response = client.post( + "/scan-tasks/search", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["count"] >= 1 + assert any(task["id"] == str(scan_task.id) for task in data["result"]) + + +# Test: list by globalView with filter should return filtered scan tasks +@pytest.mark.django_db(transaction=True) +def test_list_filtered_scan_tasks_by_global_view(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) + scan_task = ScanTask.objects.create( + scan=scan, type="fargate", status="failed" + ) + scan_task.organizations.add(organization) + + scan2 = Scan.objects.create(name="censys", arguments={}, frequency=100) + scan_task2 = ScanTask.objects.create( + scan=scan2, type="fargate", status="failed" + ) + scan_task2.organizations.add(organization) + + response = client.post( + "/scan-tasks/search", + json={"filters": {"name": "findomain"}}, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + print(response.json()) + + assert response.status_code == 200 + data = response.json() + assert data["count"] >= 1 + assert any(task["id"] == str(scan_task.id) for task in data["result"]) + assert all(task["scan"]["name"] == "findomain" for task in data["result"]) + + +# Test: list by regular user should fail +@pytest.mark.django_db(transaction=True) +def test_list_scan_tasks_by_regular_user_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + Role.objects.create(user=user, organization=organization, role="user") + + response = client.post( + "/scan-tasks/search", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {"detail": "Unauthorized access. View logs for details."} + + +# Test: kill by globalAdmin should kill the scan task +@pytest.mark.django_db(transaction=True) +def test_kill_scan_task_by_global_admin(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) + scan_task = ScanTask.objects.create( + scan=scan, type="fargate", status="created" + ) + scan_task.organizations.add(organization) + + response = client.post( + f"/scan-tasks/{scan_task.id}/kill", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + print(response.json) + assert response.status_code == 200 + + +# Test: kill by globalAdmin should not work on a finished scan task +@pytest.mark.django_db(transaction=True) +def test_kill_finished_scan_task_by_global_admin_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) + scan_task = ScanTask.objects.create( + scan=scan, type="fargate", status="finished" + ) + scan_task.organizations.add(organization) + + response = client.post( + f"/scan-tasks/{scan_task.id}/kill", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 400 + assert "already finished" in response.text + + +# Test: kill by globalView should fail +@pytest.mark.django_db(transaction=True) +def test_kill_scan_task_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) + scan_task = ScanTask.objects.create( + scan=scan, type="fargate", status="created" + ) + scan_task.organizations.add(organization) + + response = client.post( + f"/scan-tasks/{scan_task.id}/kill", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {"detail": "Unauthorized access. View logs for details."} + + +# Test: logs by globalView user should get logs +@pytest.mark.django_db(transaction=True) +@patch("xfd_api.tasks.ecs_client.ECSClient.get_logs") +def test_get_logs_by_global_view(mock_get_logs): + + mock_get_logs.return_value = "logs" + + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) + scan_task = ScanTask.objects.create( + scan=scan, fargateTaskArn="fargateTaskArn", type="fargate", status="started" + ) + scan_task.organizations.add(organization) + + response = client.get( + f"/scan-tasks/{scan_task.id}/logs", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + assert response.text == "logs" + # Mock assertion to ensure logs fetching is called with the correct ARN + mock_get_logs.assert_called_with("fargateTaskArn") + + +# Test: logs by regular user should fail +@pytest.mark.django_db(transaction=True) +@patch("xfd_api.tasks.ecs_client.ECSClient.get_logs") +def test_get_logs_by_regular_user_fails(mock_get_logs): + + mock_get_logs.return_value = "logs" + + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) + scan_task = ScanTask.objects.create( + scan=scan, fargateTaskArn="fargateTaskArn", type="fargate", status="started" + ) + scan_task.organizations.add(organization) + + response = client.get( + f"/scan-tasks/{scan_task.id}/logs", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {"detail": "Unauthorized access. View logs for details."} + mock_get_logs.assert_not_called() diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 2e8d5183..012d39ba 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -22,15 +22,16 @@ # Third-Party Libraries from django.shortcuts import render -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Body from fastapi.security import APIKeyHeader, OAuth2PasswordBearer +from pydantic import UUID4 # from .schemas import Cpe from . import schema_models from .api_methods import api_key as api_key_methods from .api_methods import auth as auth_methods from .api_methods import notification as notification_methods -from .api_methods import organization, scan +from .api_methods import organization, scan, scan_tasks from .api_methods.api_keys import get_api_keys from .api_methods.cpe import get_cpes_by_id from .api_methods.cve import get_cves_by_id, get_cves_by_name @@ -40,6 +41,7 @@ from .auth import get_current_active_user from .login_gov import callback, login from .models import Assessment, User +from .schema_models import scan_tasks as scanTaskSchema from .schema_models import organization as OrganizationSchema from .schema_models import scan as scanSchema from .schema_models.api_key import ApiKey as ApiKeySchema @@ -518,6 +520,49 @@ async def invoke_scheduler(current_user: User = Depends(get_current_active_user) return response + +# ======================================== +# Scan Task Endpoints +# ======================================== + +@api_router.post( + "/scan-tasks/search", + dependencies=[Depends(get_current_active_user)], + response_model=scanTaskSchema.ScanTaskListResponse, + tags=["Scan Tasks"], +) +async def list_scan_tasks( + search_data: Optional[scanTaskSchema.ScanTaskSearch] = Body(None), + current_user: User = Depends(get_current_active_user), +): + """List scan tasks based on filters.""" + return scan_tasks.list_scan_tasks(search_data, current_user) + + +@api_router.post( + "/scan-tasks/{scan_task_id}/kill", + dependencies=[Depends(get_current_active_user)], + tags=["Scan Tasks"], +) +async def kill_scan_tasks( + scan_task_id: UUID4, current_user: User = Depends(get_current_active_user) +): + """Kill a scan task.""" + return scan_tasks.kill_scan_task(scan_task_id, current_user) + + +@api_router.get( + "/scan-tasks/{scan_task_id}/logs", + dependencies=[Depends(get_current_active_user)], + # response_model=scanTaskSchema.GenericResponse, + tags=["Scan Tasks"], +) +async def get_scan_task_logs( + scan_task_id: UUID4, current_user: User = Depends(get_current_active_user) +): + """Get logs from a particular scan task.""" + return scan_tasks.get_scan_task_logs(scan_task_id, current_user) + # ======================================== # Organization Endpoints # ======================================== From a80fe184df7382a9141989d1eafc0f33c79468d5 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Thu, 24 Oct 2024 10:29:50 -0500 Subject: [PATCH 076/314] Merging search_domains(), search_vulnerabilities(), stub out export methods. --- .../xfd_django/xfd_api/api_methods/domain.py | 41 ++---- .../xfd_api/api_methods/vulnerability.py | 43 +++--- .../xfd_api/helpers/filter_helpers.py | 136 +++++++++++------- .../xfd_api/schema_models/domain.py | 8 +- .../xfd_api/schema_models/vulnerability.py | 40 +++--- backend/src/xfd_django/xfd_api/views.py | 7 +- 6 files changed, 155 insertions(+), 120 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/domain.py b/backend/src/xfd_django/xfd_api/api_methods/domain.py index 8e919409..d692ad48 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/domain.py +++ b/backend/src/xfd_django/xfd_api/api_methods/domain.py @@ -12,7 +12,7 @@ from ..helpers.filter_helpers import filter_domains, sort_direction from ..models import Domain -from ..schema_models.domain import DomainSearch +from ..schema_models.domain import DomainFilters, DomainSearch def get_domain_by_id(domain_id: str): @@ -39,40 +39,27 @@ def search_domains(domain_search: DomainSearch): object: A paginated list of Domain objects """ try: - # Fetch all domains in list - if domain_search.order is not None: - domains = Domain.objects.all().order_by( - sort_direction(domain_search.sort, domain_search.order) - ) - else: - # Default sort order behavior - domains = Domain.objects.all() + domains = Domain.objects.all().order_by( + sort_direction(domain_search.sort, domain_search.order) + ) - if domain_search.filters is not None: - results = filter_domains(domains, domain_search.filters) - paginator = Paginator(results, domain_search.pageSize) + if domain_search.filters: + domains = filter_domains(domains, domain_search.filters) + paginator = Paginator(domains, domain_search.pageSize) - return paginator.get_page(domain_search.page) - else: - raise ValueError("DomainFilters cannot be NoneType") + return paginator.get_page(domain_search.page) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) -def export_domains(domain_search: DomainSearch): +def export_domains(domain_filters: DomainFilters): try: - domains = Domain.objects.all().order_by( - sort_direction(domain_search.sort, domain_search.order) - ) + domains = Domain.objects.all() - if domain_search.filters is not None: - results = filter_domains(domains, domain_search.filters) - paginator = Paginator(results, domain_search.pageSize) + if domain_filters: + domains = filter_domains(domains, domain_filters) - return paginator.get_page(domain_search.page) - # TODO: Implement S3 client methods after collab with entire team. - # return export_to_csv(paginator, domains, "testing", True) - else: - raise ValueError("DomainFilters cannot be NoneType") + # TODO: Integrate methods to generate CSV from queryset and save to S3 bucket + return domains except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py b/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py index 12ed0ff9..9b8b9a49 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py +++ b/backend/src/xfd_django/xfd_api/api_methods/vulnerability.py @@ -52,25 +52,36 @@ def search_vulnerabilities(vulnerability_search: VulnerabilitySearch): """ try: # Fetch all domains in list - if vulnerability_search.order is not None: - vulnerabilities = Vulnerability.objects.all().order_by( - sort_direction(vulnerability_search.sort, vulnerability_search.order) - ) - else: - # Default sort order behavior - vulnerabilities = Vulnerability.objects.all() + vulnerabilities = Vulnerability.objects.all().order_by( + sort_direction(vulnerability_search.sort, vulnerability_search.order) + ) - if vulnerability_search.filters is not None: - print(f"filters: {vulnerability_search.filters}") - results = filter_vulnerabilities( + if vulnerability_search.filters: + vulnerabilities = filter_vulnerabilities( vulnerabilities, vulnerability_search.filters ) - if vulnerability_search.groupBy is not None: - results = results.values(vulnerability_search.groupBy).order_by() - paginator = Paginator(results, vulnerability_search.pageSize) - return paginator.get_page(vulnerability_search.page) - else: - raise ValueError("DomainFilters cannot be NoneType") + if vulnerability_search.groupBy: + vulnerabilities = vulnerabilities.values( + vulnerability_search.groupBy + ).order_by() + + paginator = Paginator(vulnerabilities, vulnerability_search.pageSize) + return paginator.get_page(vulnerability_search.page) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +def export_vulnerabilities(vulnerability_filters: VulnerabilityFilters): + try: + vulnerabilities = Vulnerability.objects.all() + + if vulnerability_filters: + vulnerabilities = filter_vulnerabilities( + vulnerabilities, vulnerability_filters + ) + + # TODO: Integrate methods to generate CSV from queryset and save to S3 bucket + return vulnerabilities except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py index ffc0315b..f750d83e 100644 --- a/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py +++ b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py @@ -1,9 +1,12 @@ # Third-Party Libraries +from django.core.exceptions import ObjectDoesNotExist +from django.db.models.query import QuerySet +from django.http import Http404 from fastapi import HTTPException from ..models import Domain, Organization, Service, Vulnerability -from ..schema_models.domain import Domain, DomainFilters -from ..schema_models.vulnerability import Vulnerability, VulnerabilityFilters +from ..schema_models.domain import DomainFilters +from ..schema_models.vulnerability import VulnerabilityFilters def sort_direction(sort, order): @@ -25,7 +28,7 @@ def sort_direction(sort, order): raise HTTPException(status_code=500, detail="Invalid sort direction supplied") -def filter_domains(domains, domain_filters: DomainFilters): +def filter_domains(domains: QuerySet, domain_filters: DomainFilters): """ Filter domains Arguments: @@ -35,62 +38,69 @@ def filter_domains(domains, domain_filters: DomainFilters): object: a list of Domain objects """ try: - print("DEBUG") + print(f"domains: {domains}") if domain_filters.port: - print(f"port: {domain_filters.port}") - services_by_port = Service.objects.values("domainId").filter( - port=domain_filters.port + services_by_port = Service.objects.filter(port=domain_filters.port).values( + "domainId" ) - if services_by_port.exists(): - domains = domains.filter(id__in=services_by_port) + if not services_by_port.exists(): + raise Http404("No Domains found with the provided port") + domains = domains.filter(id__in=services_by_port) + if domain_filters.service: - print(f"service: {domain_filters.service}") - service_by_id = Service.objects.values("domainId").filter( - id__in=domain_filters.service + service_by_id = Service.objects.filter(id=domain_filters.service).values( + "domainId" ) - if service_by_id.exists(): - domains = domains.filter(id=service_by_id) + if not service_by_id.exists(): + raise Http404("No Domains found with the provided service") + domains = domains.filter(id__in=service_by_id) + if domain_filters.reverseName: - print(f"reverseName: {domain_filters.reverseName}") - domains_by_reverse_name = Domain.objects.values("id").filter( + domains_by_reverse_name = Domain.objects.filter( reverseName=domain_filters.reverseName - ) - if domains_by_reverse_name.exists(): - domains = domains.filter(id__in=domains_by_reverse_name) + ).values("id") + if not domains_by_reverse_name.exists(): + raise Http404("No Domains found with the provided reverse name") + domains = domains.filter(id__in=domains_by_reverse_name) + if domain_filters.ip: - print(f"ip: {domain_filters.ip}") - domains_by_ip = Domain.objects.values("id").filter(ip=domain_filters.ip) - if domains_by_ip.exists(): - domains = domains.filter(id__in=domains_by_ip) + domains_by_ip = Domain.objects.filter(ip=domain_filters.ip).values("id") + if not domains_by_ip.exists(): + raise Http404("No Domains found with the provided ip") + domains = domains.filter(id__in=domains_by_ip) + if domain_filters.organization: - print(f"organization: {domain_filters.organization}") - domains_by_org = Domain.objects.values("id").filter( + domains_by_org = Domain.objects.filter( organizationId_id=domain_filters.organization - ) - if domains_by_org.exists(): - domains = domains.filter(id__in=domains_by_org) + ).values("id") + if not domains_by_org.exists(): + raise Http404("No Domains found with the provided organization") + domains = domains.filter(id__in=domains_by_org) + if domain_filters.organizationName: - print(f"organizationName: {domain_filters.organizationName}") - organization_by_name = Organization.objects.values("id").filter( + organization_by_name = Organization.objects.filter( name=domain_filters.organizationName - ) - if organization_by_name.exists(): - domains = domains.filter(organizationId_id__in=organization_by_name) + ).values("id") + if not organization_by_name.exists(): + raise Http404("No Domains found with the provided organization name") + domains = domains.filter(organizationId_id__in=organization_by_name) + if domain_filters.vulnerabilities: - print(f"vulnerabilities: {domain_filters.vulnerabilities}") - vulnerabilities_by_id = Vulnerability.objects.values("domainId").filter( + vulnerabilities_by_id = Vulnerability.objects.filter( id=domain_filters.vulnerabilities - ) - if vulnerabilities_by_id.exists(): - domains = domains.filter(id__in=vulnerabilities_by_id) - + ).values("domainId") + if not vulnerabilities_by_id.exists(): + raise Http404("No Domains found with the provided vulnerability") + domains = domains.filter(id__in=vulnerabilities_by_id) return domains + except ObjectDoesNotExist: + print("No vulnerability found with that ID.") except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + print(f"Error: {e}") def filter_vulnerabilities( - vulnerabilities, vulnerability_filters: VulnerabilityFilters + vulnerabilities: QuerySet, vulnerability_filters: VulnerabilityFilters ): """ Filter vulnerabilitie @@ -102,59 +112,81 @@ def filter_vulnerabilities( """ try: if vulnerability_filters.id: - print(f"id: {vulnerability_filters.id}") vulnerability_by_id = Vulnerability.objects.values("id").get( id=vulnerability_filters.id ) - vulnerabilities = vulnerabilities.filter(id=vulnerability_by_id("id")) + if not vulnerability_by_id: + raise Http404("No Vulnerabilities found with the provided id") + vulnerabilities = vulnerabilities.filter(id=vulnerability_by_id) + if vulnerability_filters.title: - print(f"title: {vulnerability_filters.title}") vulnerabilities_by_title = Vulnerability.objects.values("id").filter( title=vulnerability_filters.title ) + if not vulnerabilities_by_title.exists(): + raise Http404("No Vulnerabilities found with the provided title") vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_title) + if vulnerability_filters.domain: - print(f"domain: {vulnerability_filters.domain}") - vulnerabilities_by_domain = Vulnerability.objects.values("id").filters( + vulnerabilities_by_domain = Vulnerability.objects.values("id").filter( domainId=vulnerability_filters.domain ) + if not vulnerabilities_by_domain.exists(): + raise Http404("No Vulnerabilities found with the provided domain") vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_domain) + if vulnerability_filters.severity: - print(f"severity: {vulnerability_filters.severity}") vulnerabilities_by_severity = Vulnerability.objects.values("id").filter( severity=vulnerability_filters.severity ) + if not vulnerabilities_by_severity.exists(): + raise Http404( + "No Vulnerabilities found with the provided severity level" + ) vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_severity) + if vulnerability_filters.cpe: - print(f"cpe: {vulnerability_filters.cpe}") vulnerabilities_by_cpe = Vulnerability.objects.values("id").filter( cpe=vulnerability_filters.cpe ) + if not vulnerabilities_by_cpe.exists(): + raise Http404("No Vulnerabilities found with the provided Cpe") vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_cpe) + if vulnerability_filters.state: - print(f"state: {vulnerability_filters.state}") vulnerabilities_by_state = Vulnerability.objects.values("id").filter( state=vulnerability_filters.state ) + if not vulnerabilities_by_state.exists(): + raise Http404("No Vulnerabilities found with the provided state") vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_state) + if vulnerability_filters.organization: - print(f"organization: {vulnerability_filters.organization}") domains = Domain.objects.all() domains_by_organization = Domain.objects.values("id").filter( organizationId_id=vulnerability_filters.organization ) + if not domains_by_organization.exists(): + raise Http404( + "No Organization-Domain found with the provided organization ID" + ) domains = domains.filter(id__in=domains_by_organization) vulnerabilities_by_domain = Vulnerability.objects.values("id").filter( id__in=domains ) + if not vulnerabilities_by_domain.exists(): + raise Http404( + "No Vulnerabilities found with the provided organization ID" + ) vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_domain) + if vulnerability_filters.isKev: - print(f"isKev: {vulnerability_filters.isKev}") vulnerabilities_by_is_kev = Vulnerability.objects.values("id").filter( isKev=vulnerability_filters.isKev ) + if not vulnerabilities_by_is_kev.exists(): + raise Http404("No Vulnerabilities found with the provided isKev value") vulnerabilities = vulnerabilities.filter(id__in=vulnerabilities_by_is_kev) - print(f"vulnerabilities: {vulnerabilities}") return vulnerabilities except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/schema_models/domain.py b/backend/src/xfd_django/xfd_api/schema_models/domain.py index 5c7d1367..67aff311 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/domain.py +++ b/backend/src/xfd_django/xfd_api/schema_models/domain.py @@ -46,7 +46,7 @@ class Config: class DomainFilters(BaseModel): """DomainFilters schema.""" - port: Optional[str] = None + port: Optional[int] = None service: Optional[str] = None reverseName: Optional[str] = None ip: Optional[str] = None @@ -64,9 +64,9 @@ class DomainSearch(BaseModel): page: int = 1 sort: Optional[str] = "ASC" - order: Optional[str] = None - filters: Optional[DomainFilters] - pageSize: Optional[int] = None + order: Optional[str] = "id" + filters: Optional[DomainFilters] = None + pageSize: Optional[int] = 25 class Config: from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py b/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py index dfd954ae..b0ee5381 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py +++ b/backend/src/xfd_django/xfd_api/schema_models/vulnerability.py @@ -16,25 +16,25 @@ class Vulnerability(BaseModel): id: UUID createdAt: datetime updatedAt: datetime - lastSeen: datetime + lastSeen: Optional[datetime] title: Optional[str] cve: Optional[str] cwe: Optional[str] cpe: Optional[str] description: Optional[str] - references: Json[Any] - cvss: float + references: Optional[Any] + cvss: Optional[float] severity: Optional[str] needsPopulation: bool state: Optional[str] substate: Optional[str] source: Optional[str] notes: Optional[str] - actions: Json[Any] - structuredData: Json[Any] + actions: Optional[Any] + structuredData: Optional[Any] isKev: bool - domainId: UUID - serviceId: UUID + domainId_id: UUID + serviceId_id: UUID class Config: from_attributes = True @@ -44,14 +44,14 @@ class VulnerabilityFilters(BaseModel): """VulnerabilityFilters schema.""" id: Optional[str] = None - title: Optional[str] - domain: Optional[str] - severity: Optional[str] - cpe: Optional[str] - state: Optional[str] - substate: Optional[str] + title: Optional[str] = None + domain: Optional[str] = None + severity: Optional[str] = None + cpe: Optional[str] = None + state: Optional[str] = None + substate: Optional[str] = None organization: Optional[str] = None - tag: Optional[str] + tag: Optional[str] = None isKev: Optional[bool] = None class Config: @@ -61,12 +61,12 @@ class Config: class VulnerabilitySearch(BaseModel): """VulnerabilitySearch schema.""" - page: int - sort: Optional[str] - order: str - filters: Optional[VulnerabilityFilters] - pageSize: Optional[int] - groupBy: Optional[str] + page: int = 1 + sort: Optional[str] = "ASC" + order: Optional[str] = "id" + filters: Optional[VulnerabilityFilters] = None + pageSize: Optional[int] = 25 + groupBy: Optional[str] = None class Config: from_attributes = True diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index f6b47162..5aa5fac1 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -239,7 +239,12 @@ async def call_get_domain_by_id(domain_id: str): return get_domain_by_id(domain_id) -@api_router.post("/vulnerabilities/search") +@api_router.post( + "/vulnerabilities/search", + # dependencies=[Depends(get_current_active_user)], + response_model=List[VulnerabilitySchema], + tags=["Vulnerabilities"], +) async def call_search_vulnerabilities(vulnerability_search: VulnerabilitySearch): try: return search_vulnerabilities(vulnerability_search) From f1502c77e03360f6307ad3f124b33f1558221186 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Thu, 24 Oct 2024 14:37:52 -0400 Subject: [PATCH 077/314] Add scan tests and fix schemas and pre-commits --- .../xfd_api/api_methods/organization.py | 23 +- .../xfd_django/xfd_api/api_methods/scan.py | 73 ++- .../xfd_api/api_methods/scan_tasks.py | 20 +- backend/src/xfd_django/xfd_api/auth.py | 2 +- backend/src/xfd_django/xfd_api/models.py | 6 +- .../xfd_django/xfd_api/schema_models/scan.py | 12 +- .../xfd_api/schema_models/scan_tasks.py | 2 +- .../src/xfd_django/xfd_api/tasks/scheduler.py | 17 +- .../src/xfd_django/xfd_api/tests/test_scan.py | 491 ++++++++++++++++++ .../xfd_api/tests/test_scan_task.py | 39 +- backend/src/xfd_django/xfd_api/views.py | 7 +- 11 files changed, 606 insertions(+), 86 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/tests/test_scan.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/organization.py b/backend/src/xfd_django/xfd_api/api_methods/organization.py index 9177e10c..7e2d9b71 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/organization.py +++ b/backend/src/xfd_django/xfd_api/api_methods/organization.py @@ -529,14 +529,21 @@ def update_organization(organization_id: str, organization_data, current_user): except Organization.DoesNotExist: raise HTTPException(status_code=404, detail="Organization not found") - # Manually update each field - organization.name = organization_data.name - organization.acronym = organization_data.acronym - organization.rootDomains = organization_data.rootDomains - organization.ipBlocks = organization_data.ipBlocks - organization.stateName = organization_data.stateName - organization.state = organization_data.state - organization.isPassive = organization_data.isPassive + # Update only the fields that are provided + if organization_data.name is not None: + organization.name = organization_data.name + if organization_data.acronym is not None: + organization.acronym = organization_data.acronym + if organization_data.rootDomains is not None: + organization.rootDomains = organization_data.rootDomains + if organization_data.ipBlocks is not None: + organization.ipBlocks = organization_data.ipBlocks + if organization_data.stateName is not None: + organization.stateName = organization_data.stateName + if organization_data.state is not None: + organization.state = organization_data.state + if organization_data.isPassive is not None: + organization.isPassive = organization_data.isPassive # Handle parent organization if provided if organization_data.parent: diff --git a/backend/src/xfd_django/xfd_api/api_methods/scan.py b/backend/src/xfd_django/xfd_api/api_methods/scan.py index 112d2722..dbcae237 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/scan.py +++ b/backend/src/xfd_django/xfd_api/api_methods/scan.py @@ -17,9 +17,7 @@ def list_scans(current_user): try: # Check if the user is a GlobalViewAdmin if not is_global_view_admin(current_user): - raise HTTPException( - status_code=403, detail="Unauthorized access. View logs for details." - ) + raise HTTPException(status_code=403, detail="Unauthorized access.") # Fetch scans and prefetch related tags scans = Scan.objects.prefetch_related("tags").all() @@ -62,6 +60,10 @@ def list_scans(current_user): } return response + + except HTTPException as http_exc: + raise http_exc + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -71,9 +73,7 @@ def list_granular_scans(current_user): try: # Check if the user is a GlobalViewAdmin if not is_global_view_admin(current_user): - raise HTTPException( - status_code=403, detail="Unauthorized access. View logs for details." - ) + raise HTTPException(status_code=403, detail="Unauthorized access.") # Fetch scans that match the criteria (isGranular, isUserModifiable, isSingleScan) scans = Scan.objects.filter( @@ -84,6 +84,9 @@ def list_granular_scans(current_user): return response + except HTTPException as http_exc: + raise http_exc + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -93,9 +96,7 @@ def create_scan(scan_data: NewScan, current_user): try: # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): - raise HTTPException( - status_code=403, detail="Unauthorized access. View logs for details." - ) + raise HTTPException(status_code=403, detail="Unauthorized access.") # Check if scan name is valid if scan_data.name not in SCAN_SCHEMA: @@ -131,6 +132,9 @@ def create_scan(scan_data: NewScan, current_user): "organizations": list(scan.organizations.values("id")), } + except HTTPException as http_exc: + raise http_exc + except Organization.DoesNotExist: raise HTTPException(status_code=404, detail="Organization not found") except OrganizationTag.DoesNotExist: @@ -145,9 +149,7 @@ def get_scan(scan_id: str, current_user): # Check if the user is a GlobalViewAdmin if not is_global_view_admin(current_user): - raise HTTPException( - status_code=403, detail="Unauthorized access. View logs for details." - ) + raise HTTPException(status_code=403, detail="Unauthorized access.") try: # Fetch the scan with its related organizations and tags @@ -194,9 +196,7 @@ def update_scan(scan_id: str, scan_data: NewScan, current_user): try: # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): - raise HTTPException( - status_code=403, detail="Unauthorized access. View logs for details." - ) + raise HTTPException(status_code=403, detail="Unauthorized access.") # Validate scan ID try: @@ -204,13 +204,19 @@ def update_scan(scan_id: str, scan_data: NewScan, current_user): except Scan.DoesNotExist: raise HTTPException(status_code=404, detail="Scan not found") - # Update the scan's fields with the new data - scan.name = scan_data.name - scan.arguments = scan_data.arguments - scan.frequency = scan_data.frequency - scan.isGranular = scan_data.isGranular - scan.isUserModifiable = scan_data.isUserModifiable - scan.isSingleScan = scan_data.isSingleScan + # Only update the fields that are provided in the request (non-null) + if scan_data.name is not None: + scan.name = scan_data.name + if scan_data.arguments is not None: + scan.arguments = scan_data.arguments + if scan_data.frequency is not None: + scan.frequency = scan_data.frequency + if scan_data.isGranular is not None: + scan.isGranular = scan_data.isGranular + if scan_data.isUserModifiable is not None: + scan.isUserModifiable = scan_data.isUserModifiable + if scan_data.isSingleScan is not None: + scan.isSingleScan = scan_data.isSingleScan # Update ManyToMany relationships if scan_data.organizations: @@ -235,6 +241,9 @@ def update_scan(scan_id: str, scan_data: NewScan, current_user): "organizations": list(scan.organizations.values("id")), } + except HTTPException as http_exc: + raise http_exc + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -244,9 +253,7 @@ def delete_scan(scan_id: str, current_user): try: # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): - raise HTTPException( - status_code=403, detail="Unauthorized access. View logs for details." - ) + raise HTTPException(status_code=403, detail="Unauthorized access.") # Validate scan ID try: @@ -257,6 +264,10 @@ def delete_scan(scan_id: str, current_user): scan.delete() return {"status": "success", "message": f"Scan {scan_id} deleted successfully."} + + except HTTPException as http_exc: + raise http_exc + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -266,9 +277,7 @@ def run_scan(scan_id: str, current_user): try: # Check if the user is a GlobalWriteAdmin if not is_global_write_admin(current_user): - raise HTTPException( - status_code=403, detail="Unauthorized access. View logs for details." - ) + raise HTTPException(status_code=403, detail="Unauthorized access.") # Validate the scan ID and check if it exists try: @@ -279,6 +288,10 @@ def run_scan(scan_id: str, current_user): scan.manualRunPending = True scan.save() return {"status": "success", "message": f"Scan {scan_id} deleted successfully."} + + except HTTPException as http_exc: + raise http_exc + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -302,5 +315,9 @@ async def invoke_scheduler(current_user): response = await lambda_client.run_command(name=lambda_function_name) return response + + except HTTPException as http_exc: + raise http_exc + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py b/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py index 1f1b0f8e..b44c2cdf 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py +++ b/backend/src/xfd_django/xfd_api/api_methods/scan_tasks.py @@ -2,6 +2,7 @@ # Standard Python Libraries from datetime import datetime, timezone +from typing import Optional # Third-Party Libraries from fastapi import HTTPException, Response, status @@ -14,7 +15,7 @@ PAGE_SIZE = 15 -def list_scan_tasks(search_data: ScanTaskSearch, current_user): +def list_scan_tasks(search_data: Optional[ScanTaskSearch], current_user): """List scans tasks based on search filter.""" try: # Check if the user is a GlobalViewAdmin @@ -25,15 +26,18 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): # Ensure that search_data is not None, and set default values if it is if search_data is None: - search_data = ScanTaskSearch(pageSize=PAGE_SIZE, page=1, sort="createdAt", order="DESC", filters={}) + search_data = ScanTaskSearch( + pageSize=PAGE_SIZE, page=1, sort="createdAt", order="DESC", filters={} + ) # Validate and parse the request body pageSize = search_data.pageSize or PAGE_SIZE - + page = search_data.page or 1 + # Determine the correct ordering based on the 'order' field ordering_field = ( f"-{search_data.sort}" - if search_data.order.upper() == "DESC" + if search_data.order and search_data.order.upper() == "DESC" else search_data.sort ) @@ -59,7 +63,7 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): # Paginate results if pageSize != -1: - qs = qs[(search_data.page - 1) * pageSize : search_data.page * pageSize] + qs = qs[(page - 1) * pageSize : page * pageSize] # Convert queryset into a serialized response results = [] @@ -135,7 +139,7 @@ def list_scan_tasks(search_data: ScanTaskSearch, current_user): count = qs.count() response = {"result": results, "count": count} return response - + except HTTPException as http_exc: raise http_exc @@ -175,7 +179,7 @@ def kill_scan_task(scan_task_id, current_user): except HTTPException as http_exc: raise http_exc - + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) @@ -210,7 +214,7 @@ def get_scan_task_logs(scan_task_id, current_user): except HTTPException as http_exc: raise http_exc - + except Exception as e: print(e) raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/auth.py b/backend/src/xfd_django/xfd_api/auth.py index 3f676e0a..6e865330 100644 --- a/backend/src/xfd_django/xfd_api/auth.py +++ b/backend/src/xfd_django/xfd_api/auth.py @@ -462,7 +462,7 @@ def get_organization_region(organization_id: str) -> str: return organization.regionId -def get_tag_organizations(current_user, tag_id: str) -> list[str]: +def get_tag_organizations(current_user, tag_id) -> list[str]: """Returns the organizations belonging to a tag, if the user can access the tag.""" # Check if the user is a global view admin if not is_global_view_admin(current_user): diff --git a/backend/src/xfd_django/xfd_api/models.py b/backend/src/xfd_django/xfd_api/models.py index b9f044c5..7cea5f29 100644 --- a/backend/src/xfd_django/xfd_api/models.py +++ b/backend/src/xfd_django/xfd_api/models.py @@ -212,7 +212,11 @@ class Domain(models.Model): trustymailResults = models.JSONField(db_column="trustymailResults", default=dict) discoveredBy = models.ForeignKey( - "Scan", on_delete=models.SET_NULL, null=True, blank=True + "Scan", + on_delete=models.SET_NULL, + null=True, + blank=True, + db_column="discoveredById", ) organization = models.ForeignKey( "Organization", on_delete=models.CASCADE, db_column="organizationId" diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan.py b/backend/src/xfd_django/xfd_api/schema_models/scan.py index 1b3d1e9c..0ceece88 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/scan.py +++ b/backend/src/xfd_django/xfd_api/schema_models/scan.py @@ -90,12 +90,12 @@ class NewScan(BaseModel): name: str arguments: Any - organizations: Optional[List[UUID]] - tags: Optional[List[IdSchema]] - frequency: int - isGranular: bool - isUserModifiable: Optional[bool] - isSingleScan: bool + organizations: Optional[List[UUID]] = [] + tags: Optional[List[IdSchema]] = [] + frequency: Optional[int] = None + isGranular: Optional[bool] = None + isUserModifiable: Optional[bool] = None + isSingleScan: Optional[bool] = None class CreateScanResponseModel(BaseModel): diff --git a/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py b/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py index f40eda34..e187934e 100644 --- a/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py +++ b/backend/src/xfd_django/xfd_api/schema_models/scan_tasks.py @@ -2,7 +2,7 @@ # Standard Python Libraries from datetime import datetime -from typing import Any, List, Optional, Dict +from typing import Any, Dict, List, Optional from uuid import UUID # Third-Party Libraries diff --git a/backend/src/xfd_django/xfd_api/tasks/scheduler.py b/backend/src/xfd_django/xfd_api/tasks/scheduler.py index 8a820010..96130224 100644 --- a/backend/src/xfd_django/xfd_api/tasks/scheduler.py +++ b/backend/src/xfd_django/xfd_api/tasks/scheduler.py @@ -60,7 +60,7 @@ async def launch_single_scan_task( global_scan = getattr(scan_schema, "global_scan", None) scan_task = scan_task or ScanTask.objects.create( - scanId=scan, type=task_type, status="created" + scan=scan, type=task_type, status="created" ) # Set the many-to-many relationship with organizations @@ -208,9 +208,9 @@ def should_run_scan(self, scan, organization=None): # Function to filter the scan tasks based on whether it's global or organization-specific. def filter_scan_tasks(tasks): if global_scan: - return tasks.filter(scanId=scan) + return tasks.filter(scan=scan) else: - return tasks.filter(scanId=scan).filter( + return tasks.filter(scan=scan).filter( organizations=organization ) | tasks.filter(organizations__id=organization.id) @@ -239,6 +239,13 @@ def filter_scan_tasks(tasks): frequency_seconds = ( scan.frequency * 1000 ) # Assuming frequency is in seconds. + + # Convert finishedAt to an aware datetime if it is naive + if timezone.is_naive(last_finished_scan_task.finishedAt): + last_finished_scan_task.finishedAt = timezone.make_aware( + last_finished_scan_task.finishedAt, timezone.get_current_timezone() + ) + # Perform the subtraction and check the time difference if ( timezone.now() - last_finished_scan_task.finishedAt ).total_seconds() < frequency_seconds: @@ -281,9 +288,9 @@ async def handler(event): organizations = Organization.objects.all() queued_scan_tasks = ( - ScanTask.objects.filter(scanId__in=scan_ids, status="queued") + ScanTask.objects.filter(scan__in=scan_ids, status="queued") .order_by("queuedAt") - .select_related("scanId") + .select_related("scan") ) scheduler = Scheduler() diff --git a/backend/src/xfd_django/xfd_api/tests/test_scan.py b/backend/src/xfd_django/xfd_api/tests/test_scan.py new file mode 100644 index 00000000..2c575b60 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests/test_scan.py @@ -0,0 +1,491 @@ +from unittest.mock import patch +import secrets +from fastapi.testclient import TestClient +import pytest +from datetime import datetime +from xfd_api.auth import create_jwt_token +from xfd_api.models import User, Scan, Organization, UserType, OrganizationTag +from xfd_django.asgi import app + +client = TestClient(app) + +# Test: list by globalAdmin should return all scans +@pytest.mark.django_db(transaction=True) +def test_list_scans_by_global_admin(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + name = f"test-{secrets.token_hex(4)}" + + Scan.objects.create( + name=name, + arguments={}, + frequency=999999, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + Scan.objects.create( + name=f"{name}-2", + arguments={}, + frequency=999999, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.get( + "/scans", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data["scans"]) >= 2 + assert len(data["organizations"]) >= 1 + assert any(org["id"] == str(organization.id) for org in data["organizations"]) + + +# Test: create by globalAdmin should succeed +@pytest.mark.django_db(transaction=True) +def test_create_scan_by_global_admin(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + name = "censys" + arguments = '{"a": "b"}' + frequency = 999999 + + response = client.post( + "/scans", + json={ + "name": name, + "arguments": arguments, + "frequency": frequency, + "isGranular": False, + "organizations": [], + "isUserModifiable": False, + "isSingleScan": False, + "tags": [] + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["name"] == name + assert data["arguments"] == arguments + assert data["frequency"] == frequency + assert data["isGranular"] is False + assert data["organizations"] == [] + assert data["tags"] == [] + assert data["createdBy"]["id"] == str(user.id) + +# Test: create a granular scan by globalAdmin should succeed +@pytest.mark.django_db(transaction=True) +def test_create_granular_scan_by_global_admin(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + name = "censys" + arguments = '{"a": "b"}' + frequency = 999999 + + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.post( + "/scans", + json={ + "name": name, + "arguments": arguments, + "frequency": frequency, + "isGranular": True, + "organizations": [str(organization.id)], + "isUserModifiable": False, + "isSingleScan": False, + "tags": [] + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["name"] == name + assert data["arguments"] == arguments + assert data["frequency"] == frequency + assert data["isGranular"] is True + assert str(organization.id) in [org["id"] for org in data["organizations"]] + + +# Test: create by globalView should fail +@pytest.mark.django_db(transaction=True) +def test_create_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.post( + "/scans", + json={ + "name": "censys", + "arguments": "{}", + "frequency": 999999, + "isGranular": False, + "organizations": [], + "isUserModifiable": False, + "isSingleScan": False + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + + +# Test: update by globalAdmin should succeed +@pytest.mark.django_db(transaction=True) +def test_update_by_global_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="censys", arguments="{}", frequency=999999) + + response = client.put( + f"/scans/{scan.id}", + json={ + "name": "findomain", + "arguments": "{}", + "frequency": 999991, + "isGranular": False, + "organizations": [], + "isUserModifiable": False, + "isSingleScan": False, + "tags": [] + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["name"] == "findomain" + assert data["arguments"] == "{}" + assert data["frequency"] == 999991 + + +# Test: update a non-granular scan to a granular scan by globalAdmin +@pytest.mark.django_db(transaction=True) +def test_update_non_granular_to_granular_by_global_admin(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create( + name="censys", arguments="{}", frequency=999999, isGranular=False, isSingleScan=False + ) + + tag = OrganizationTag.objects.create(name=f"test-{secrets.token_hex(4)}") + organization = Organization.objects.create( + name=f"test-{secrets.token_hex(4)}", + rootDomains=["test-" + secrets.token_hex(4)], + ipBlocks=[], + isPassive=False, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + organization.tags.set([tag]) + + response = client.put( + f"/scans/{scan.id}", + json={ + "name": "findomain", + "arguments": "{}", + "frequency": 999991, + "isGranular": True, + "organizations": [str(organization.id)], + "isSingleScan": False, + "isUserModifiable": True, + "tags": [{"id": str(tag.id)}] + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + print(response.json()) + assert response.status_code == 200 + data = response.json() + assert data["name"] == "findomain" + assert data["frequency"] == 999991 + assert data["isGranular"] is True + assert data["isUserModifiable"] is True + assert str(organization.id) in [org["id"] for org in data["organizations"]] + assert str(tag.id) in [t["id"] for t in data["tags"]] + + +# Test: update by globalView should fail +@pytest.mark.django_db(transaction=True) +def test_update_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="censys", arguments="{}", frequency=999999) + + response = client.put( + f"/scans/{scan.id}", + json={ + "name": "findomain", + "arguments": "{}", + "frequency": 999991 + }, + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + print(response.json()) + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + + +# Test: delete by globalAdmin should succeed +@pytest.mark.django_db(transaction=True) +def test_delete_by_global_admin_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="censys", arguments="{}", frequency=999999) + + response = client.delete( + f"/scans/{scan.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + + +# Test: delete by globalView should fail +@pytest.mark.django_db(transaction=True) +def test_delete_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="censys", arguments="{}", frequency=999999) + + response = client.delete( + f"/scans/{scan.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + + +# Test: get by globalView should succeed +@pytest.mark.django_db(transaction=True) +def test_get_by_global_view_succeeds(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="censys", arguments="{}", frequency=999999) + + response = client.get( + f"/scans/{scan.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["scan"]["name"] == "censys" + + +# Test: get by regular user on a scan not from their org should fail +@pytest.mark.django_db(transaction=True) +def test_get_by_regular_user_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create(name="censys", arguments="{}", frequency=999999) + + response = client.get( + f"/scans/{scan.id}", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + + +# Test: scheduler invoke by globalAdmin should succeed +@pytest.mark.django_db(transaction=True) +@patch("xfd_api.tasks.lambda_client.LambdaClient.run_command") +def test_scheduler_invoke_by_global_admin(mock_scheduler): + mock_scheduler.return_value = {} + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.post( + "/scheduler/invoke", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + print(response.json()) + assert response.status_code == 200 + assert response.json() == {} + mock_scheduler.assert_called_once() + + +# Test: scheduler invoke by globalView should fail +@pytest.mark.django_db(transaction=True) +@patch("xfd_api.tasks.lambda_client.LambdaClient.run_command") +def test_scheduler_invoke_by_global_view_fails(mock_scheduler): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + response = client.post( + "/scheduler/invoke", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} + mock_scheduler.assert_not_called() + + +# Test: run scan should set manualRunPending to true +@pytest.mark.django_db(transaction=True) +def test_run_scan_should_set_manualRunPending_to_true(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create( + name="censys", + arguments="{}", + frequency=999999, + lastRun=datetime.now(), + ) + + response = client.post( + f"/scans/{scan.id}/run", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 200 + + +# Test: runScan by globalView should fail +@pytest.mark.django_db(transaction=True) +def test_run_scan_by_global_view_fails(): + user = User.objects.create( + firstName="", + lastName="", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + scan = Scan.objects.create( + name="censys", + arguments="{}", + frequency=999999, + lastRun=datetime.now(), + ) + + response = client.post( + f"/scans/{scan.id}/run", + headers={"Authorization": "Bearer " + create_jwt_token(user)}, + ) + + assert response.status_code == 403 + assert response.json() == {'detail': 'Unauthorized access.'} diff --git a/backend/src/xfd_django/xfd_api/tests/test_scan_task.py b/backend/src/xfd_django/xfd_api/tests/test_scan_task.py index 0a31a1aa..c764a968 100644 --- a/backend/src/xfd_django/xfd_api/tests/test_scan_task.py +++ b/backend/src/xfd_django/xfd_api/tests/test_scan_task.py @@ -1,10 +1,13 @@ -from unittest.mock import patch +# Standard Python Libraries +from datetime import datetime import secrets +from unittest.mock import patch + +# Third-Party Libraries from fastapi.testclient import TestClient import pytest -from datetime import datetime from xfd_api.auth import create_jwt_token -from xfd_api.models import User, Organization, UserType, ScanTask, Scan, Role +from xfd_api.models import Organization, Role, Scan, ScanTask, User, UserType from xfd_django.asgi import app client = TestClient(app) @@ -32,9 +35,7 @@ def test_list_scan_tasks_by_global_view(): ) scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) - scan_task = ScanTask.objects.create( - scan=scan, type="fargate", status="failed" - ) + scan_task = ScanTask.objects.create(scan=scan, type="fargate", status="failed") scan_task.organizations.add(organization) response = client.post( @@ -70,15 +71,11 @@ def test_list_filtered_scan_tasks_by_global_view(): ) scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) - scan_task = ScanTask.objects.create( - scan=scan, type="fargate", status="failed" - ) + scan_task = ScanTask.objects.create(scan=scan, type="fargate", status="failed") scan_task.organizations.add(organization) scan2 = Scan.objects.create(name="censys", arguments={}, frequency=100) - scan_task2 = ScanTask.objects.create( - scan=scan2, type="fargate", status="failed" - ) + scan_task2 = ScanTask.objects.create(scan=scan2, type="fargate", status="failed") scan_task2.organizations.add(organization) response = client.post( @@ -149,9 +146,7 @@ def test_kill_scan_task_by_global_admin(): ) scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) - scan_task = ScanTask.objects.create( - scan=scan, type="fargate", status="created" - ) + scan_task = ScanTask.objects.create(scan=scan, type="fargate", status="created") scan_task.organizations.add(organization) response = client.post( @@ -185,9 +180,7 @@ def test_kill_finished_scan_task_by_global_admin_fails(): ) scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) - scan_task = ScanTask.objects.create( - scan=scan, type="fargate", status="finished" - ) + scan_task = ScanTask.objects.create(scan=scan, type="fargate", status="finished") scan_task.organizations.add(organization) response = client.post( @@ -221,9 +214,7 @@ def test_kill_scan_task_by_global_view_fails(): ) scan = Scan.objects.create(name="findomain", arguments={}, frequency=100) - scan_task = ScanTask.objects.create( - scan=scan, type="fargate", status="created" - ) + scan_task = ScanTask.objects.create(scan=scan, type="fargate", status="created") scan_task.organizations.add(organization) response = client.post( @@ -239,8 +230,7 @@ def test_kill_scan_task_by_global_view_fails(): @pytest.mark.django_db(transaction=True) @patch("xfd_api.tasks.ecs_client.ECSClient.get_logs") def test_get_logs_by_global_view(mock_get_logs): - - mock_get_logs.return_value = "logs" + mock_get_logs.return_value = "logs" user = User.objects.create( firstName="", @@ -281,8 +271,7 @@ def test_get_logs_by_global_view(mock_get_logs): @pytest.mark.django_db(transaction=True) @patch("xfd_api.tasks.ecs_client.ECSClient.get_logs") def test_get_logs_by_regular_user_fails(mock_get_logs): - - mock_get_logs.return_value = "logs" + mock_get_logs.return_value = "logs" user = User.objects.create( firstName="", diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 012d39ba..1c261822 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -22,7 +22,7 @@ # Third-Party Libraries from django.shortcuts import render -from fastapi import APIRouter, Depends, HTTPException, Query, Request, Body +from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request from fastapi.security import APIKeyHeader, OAuth2PasswordBearer from pydantic import UUID4 @@ -41,9 +41,9 @@ from .auth import get_current_active_user from .login_gov import callback, login from .models import Assessment, User -from .schema_models import scan_tasks as scanTaskSchema from .schema_models import organization as OrganizationSchema from .schema_models import scan as scanSchema +from .schema_models import scan_tasks as scanTaskSchema from .schema_models.api_key import ApiKey as ApiKeySchema from .schema_models.assessment import Assessment as AssessmentSchema from .schema_models.cpe import Cpe as CpeSchema @@ -520,11 +520,11 @@ async def invoke_scheduler(current_user: User = Depends(get_current_active_user) return response - # ======================================== # Scan Task Endpoints # ======================================== + @api_router.post( "/scan-tasks/search", dependencies=[Depends(get_current_active_user)], @@ -563,6 +563,7 @@ async def get_scan_task_logs( """Get logs from a particular scan task.""" return scan_tasks.get_scan_task_logs(scan_task_id, current_user) + # ======================================== # Organization Endpoints # ======================================== From b88107946773628357ac174fad1f76b4599c9c23 Mon Sep 17 00:00:00 2001 From: JCantu248 Date: Thu, 24 Oct 2024 14:26:35 -0500 Subject: [PATCH 078/314] Add TODO for export functions, to integrate CSV and S3 methods when available. --- backend/src/xfd_django/xfd_api/helpers/filter_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py index f750d83e..22a65673 100644 --- a/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py +++ b/backend/src/xfd_django/xfd_api/helpers/filter_helpers.py @@ -38,7 +38,6 @@ def filter_domains(domains: QuerySet, domain_filters: DomainFilters): object: a list of Domain objects """ try: - print(f"domains: {domains}") if domain_filters.port: services_by_port = Service.objects.filter(port=domain_filters.port).values( "domainId" From a2fdd478b4e3b92d828ac7dd1532a5448b791d63 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Fri, 25 Oct 2024 08:17:15 -0400 Subject: [PATCH 079/314] Add proxy endpoints --- .../xfd_django/xfd_api/api_methods/proxy.py | 52 +++++++++++ .../xfd_django/xfd_api/tests/test_proxy.py | 88 +++++++++++++++++++ .../src/xfd_django/xfd_api/tests/test_scan.py | 57 ++++++------ backend/src/xfd_django/xfd_api/views.py | 75 ++++++++++++++-- dev.env.example | 2 +- 5 files changed, 238 insertions(+), 36 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/api_methods/proxy.py create mode 100644 backend/src/xfd_django/xfd_api/tests/test_proxy.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/proxy.py b/backend/src/xfd_django/xfd_api/api_methods/proxy.py new file mode 100644 index 00000000..12fcbe9b --- /dev/null +++ b/backend/src/xfd_django/xfd_api/api_methods/proxy.py @@ -0,0 +1,52 @@ +"""API methods to support Proxy endpoints.""" + +# Standard Python Libraries +import os +from typing import List, Optional +import httpx + +# Third-Party Libraries +from django.shortcuts import render +from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request +from fastapi.security import APIKeyHeader, OAuth2PasswordBearer +from pydantic import UUID4 +from fastapi.responses import RedirectResponse, Response + +# Helper function to handle cookie manipulation +def manipulate_cookie(request: Request, cookie_name: str): + cookies = request.cookies.get(cookie_name) + if cookies: + return {cookie_name: cookies} + return {} + + +# Helper function to proxy requests +async def proxy_request(request: Request, target_url: str, path: Optional[str] = None, cookie_name: Optional[str] = None): + """Proxy the request to the target URL.""" + headers = dict(request.headers) + + # Cookie manipulation for specific cookie names + if cookie_name: + cookies = manipulate_cookie(request, cookie_name) + if cookies: + headers['Cookie'] = f"{cookie_name}={cookies[cookie_name]}" + + # Make the request to the target URL + async with httpx.AsyncClient() as client: + proxy_response = await client.request( + method=request.method, + url=f"{target_url}/{path}", + headers=headers, + params=request.query_params, + content=await request.body() + ) + + # Remove chunked encoding for API Gateway compatibility + proxy_response_headers = dict(proxy_response.headers) + proxy_response_headers.pop("transfer-encoding", None) + + return Response( + content=proxy_response.content, + status_code=proxy_response.status_code, + headers=proxy_response_headers + ) \ No newline at end of file diff --git a/backend/src/xfd_django/xfd_api/tests/test_proxy.py b/backend/src/xfd_django/xfd_api/tests/test_proxy.py new file mode 100644 index 00000000..44c1bc76 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/tests/test_proxy.py @@ -0,0 +1,88 @@ + +import pytest +from fastapi.testclient import TestClient +from xfd_api.models import User, UserType +from datetime import datetime +import secrets + +from xfd_django.asgi import app + +from xfd_api.auth import create_jwt_token + +# Initialize the test client with the FastAPI app +client = TestClient(app) + +@pytest.mark.django_db(transaction=True) +def test_standard_user_not_authorized_to_access_pe_proxy(): + """Test that a standard user is not authorized to access P&E proxy.""" + # Create a standard user + user = User.objects.create( + firstName="Standard", + lastName="User", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.STANDARD, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Generate a JWT token for the user + token = create_jwt_token(user) + + # Make a GET request to the P&E proxy endpoint with the user's token + response = client.get( + "/pe", + headers={"Authorization": f"Bearer {token}"} + ) + + # Assert that the user receives a 403 Unauthorized response + assert response.status_code == 403 + assert response.json() == {"detail": "Unauthorized"} + + +@pytest.mark.django_db(transaction=True) +def test_global_admin_authorized_to_access_pe_proxy(): + """Test that a global admin is authorized to access P&E proxy.""" + # Create a global admin user + user = User.objects.create( + firstName="Admin", + lastName="User", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_ADMIN, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Generate a JWT token for the global admin + token = create_jwt_token(user) + + # Make a GET request to the P&E proxy endpoint with the global admin's token + response = client.get( + "/pe", + headers={"Authorization": f"Bearer {token}"} + ) + + # Assert that the global admin is authorized and receives either a 200 or 504 response + assert response.status_code in [200, 504] + + +@pytest.mark.django_db(transaction=True) +def test_global_view_user_authorized_to_access_pe_proxy(): + """Test that a global view user is authorized to access P&E proxy.""" + # Create a global view user + user = User.objects.create( + firstName="View", + lastName="User", + email=f"{secrets.token_hex(4)}@example.com", + userType=UserType.GLOBAL_VIEW, + createdAt=datetime.now(), + updatedAt=datetime.now(), + ) + + # Make a GET request to the P&E proxy endpoint with the global view user's token + response = client.get( + "/pe", + headers={"Authorization": "Bearer " + create_jwt_token(user)} + ) + print(response.json()) + # Assert that the global view user is authorized and receives either a 200 or 504 response + assert response.status_code in [200, 504] diff --git a/backend/src/xfd_django/xfd_api/tests/test_scan.py b/backend/src/xfd_django/xfd_api/tests/test_scan.py index 2c575b60..d5fc90d5 100644 --- a/backend/src/xfd_django/xfd_api/tests/test_scan.py +++ b/backend/src/xfd_django/xfd_api/tests/test_scan.py @@ -1,14 +1,18 @@ -from unittest.mock import patch +# Standard Python Libraries +from datetime import datetime import secrets +from unittest.mock import patch + +# Third-Party Libraries from fastapi.testclient import TestClient import pytest -from datetime import datetime from xfd_api.auth import create_jwt_token -from xfd_api.models import User, Scan, Organization, UserType, OrganizationTag +from xfd_api.models import Organization, OrganizationTag, Scan, User, UserType from xfd_django.asgi import app client = TestClient(app) + # Test: list by globalAdmin should return all scans @pytest.mark.django_db(transaction=True) def test_list_scans_by_global_admin(): @@ -22,7 +26,7 @@ def test_list_scans_by_global_admin(): ) name = f"test-{secrets.token_hex(4)}" - + Scan.objects.create( name=name, arguments={}, @@ -51,7 +55,7 @@ def test_list_scans_by_global_admin(): "/scans", headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) - + assert response.status_code == 200 data = response.json() assert len(data["scans"]) >= 2 @@ -74,7 +78,7 @@ def test_create_scan_by_global_admin(): name = "censys" arguments = '{"a": "b"}' frequency = 999999 - + response = client.post( "/scans", json={ @@ -85,7 +89,7 @@ def test_create_scan_by_global_admin(): "organizations": [], "isUserModifiable": False, "isSingleScan": False, - "tags": [] + "tags": [], }, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) @@ -100,6 +104,7 @@ def test_create_scan_by_global_admin(): assert data["tags"] == [] assert data["createdBy"]["id"] == str(user.id) + # Test: create a granular scan by globalAdmin should succeed @pytest.mark.django_db(transaction=True) def test_create_granular_scan_by_global_admin(): @@ -115,7 +120,7 @@ def test_create_granular_scan_by_global_admin(): name = "censys" arguments = '{"a": "b"}' frequency = 999999 - + organization = Organization.objects.create( name=f"test-{secrets.token_hex(4)}", rootDomains=["test-" + secrets.token_hex(4)], @@ -124,7 +129,7 @@ def test_create_granular_scan_by_global_admin(): createdAt=datetime.now(), updatedAt=datetime.now(), ) - + response = client.post( "/scans", json={ @@ -135,7 +140,7 @@ def test_create_granular_scan_by_global_admin(): "organizations": [str(organization.id)], "isUserModifiable": False, "isSingleScan": False, - "tags": [] + "tags": [], }, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) @@ -170,13 +175,13 @@ def test_create_by_global_view_fails(): "isGranular": False, "organizations": [], "isUserModifiable": False, - "isSingleScan": False + "isSingleScan": False, }, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} # Test: update by globalAdmin should succeed @@ -203,7 +208,7 @@ def test_update_by_global_admin_succeeds(): "organizations": [], "isUserModifiable": False, "isSingleScan": False, - "tags": [] + "tags": [], }, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) @@ -228,9 +233,13 @@ def test_update_non_granular_to_granular_by_global_admin(): ) scan = Scan.objects.create( - name="censys", arguments="{}", frequency=999999, isGranular=False, isSingleScan=False + name="censys", + arguments="{}", + frequency=999999, + isGranular=False, + isSingleScan=False, ) - + tag = OrganizationTag.objects.create(name=f"test-{secrets.token_hex(4)}") organization = Organization.objects.create( name=f"test-{secrets.token_hex(4)}", @@ -252,7 +261,7 @@ def test_update_non_granular_to_granular_by_global_admin(): "organizations": [str(organization.id)], "isSingleScan": False, "isUserModifiable": True, - "tags": [{"id": str(tag.id)}] + "tags": [{"id": str(tag.id)}], }, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) @@ -284,17 +293,13 @@ def test_update_by_global_view_fails(): response = client.put( f"/scans/{scan.id}", - json={ - "name": "findomain", - "arguments": "{}", - "frequency": 999991 - }, + json={"name": "findomain", "arguments": "{}", "frequency": 999991}, headers={"Authorization": "Bearer " + create_jwt_token(user)}, ) print(response.json()) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} # Test: delete by globalAdmin should succeed @@ -339,7 +344,7 @@ def test_delete_by_global_view_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} # Test: get by globalView should succeed @@ -386,7 +391,7 @@ def test_get_by_regular_user_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} # Test: scheduler invoke by globalAdmin should succeed @@ -432,7 +437,7 @@ def test_scheduler_invoke_by_global_view_fails(mock_scheduler): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} mock_scheduler.assert_not_called() @@ -488,4 +493,4 @@ def test_run_scan_by_global_view_fails(): ) assert response.status_code == 403 - assert response.json() == {'detail': 'Unauthorized access.'} + assert response.json() == {"detail": "Unauthorized access."} diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 1c261822..252cc3f7 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -18,12 +18,15 @@ """ # Standard Python Libraries +import os from typing import List, Optional # Third-Party Libraries from django.shortcuts import render from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request +from fastapi.responses import RedirectResponse, Response from fastapi.security import APIKeyHeader, OAuth2PasswordBearer +import httpx from pydantic import UUID4 # from .schemas import Cpe @@ -31,7 +34,7 @@ from .api_methods import api_key as api_key_methods from .api_methods import auth as auth_methods from .api_methods import notification as notification_methods -from .api_methods import organization, scan, scan_tasks +from .api_methods import organization, proxy, scan, scan_tasks from .api_methods.api_keys import get_api_keys from .api_methods.cpe import get_cpes_by_id from .api_methods.cve import get_cves_by_id, get_cves_by_name @@ -71,15 +74,69 @@ async def healthcheck(): return {"status": "ok2"} -@api_router.get("/test-apikeys", tags=["Testing"]) -async def call_get_api_keys(): - """ - Get all API keys. +###################### +# Proxy +###################### - Returns: - list: A list of all API keys. - """ - return get_api_keys() + +# Matomo Proxy +@api_router.api_route( + "/matomo/{path:path}", + dependencies=[Depends(get_current_active_user)], + tags=["Analytics"], +) +async def matomo_proxy( + path: str, request: Request, current_user: User = Depends(get_current_active_user) +): + """Proxy requests to the Matomo analytics instance.""" + # Public paths -- directly allowed + allowed_paths = ["/matomo.php", "/matomo.js"] + if any( + [request.url.path.startswith(allowed_path) for allowed_path in allowed_paths] + ): + return await proxy.proxy_request(path, request, os.getenv("MATOMO_URL")) + + # Redirects for specific font files + if request.url.path in [ + "/plugins/Morpheus/fonts/matomo.woff2", + "/plugins/Morpheus/fonts/matomo.woff", + "/plugins/Morpheus/fonts/matomo.ttf", + ]: + return RedirectResponse( + url=f"https://cdn.jsdelivr.net/gh/matomo-org/matomo@3.14.1{request.url.path}" + ) + + # Ensure only global admin can access other paths + if current_user.userType != "globalAdmin": + raise HTTPException(status_code=403, detail="Unauthorized") + + # Handle the proxy request to Matomo + return await proxy.proxy_request( + request, os.getenv("MATOMO_URL", ""), path, cookie_name="MATOMO_SESSID" + ) + + +# P&E Proxy +@api_router.api_route( + "/pe/{path:path}", + dependencies=[Depends(get_current_active_user)], + tags=["P&E Proxy"], +) +async def pe_proxy( + path: str, request: Request, current_user: User = Depends(get_current_active_user) +): + """Proxy requests to the P&E Django application.""" + # Ensure only Global Admin and Global View users can access + if current_user.userType not in ["globalView", "globalAdmin"]: + raise HTTPException(status_code=403, detail="Unauthorized") + + # Handle the proxy request to the P&E Django application + return await proxy.proxy_request(request, os.getenv("PE_API_URL", ""), path) + + +###################### +# Assessments +###################### # TODO: Uncomment checks for current_user once authentication is implemented diff --git a/dev.env.example b/dev.env.example index e453e7af..008d35c0 100644 --- a/dev.env.example +++ b/dev.env.example @@ -87,7 +87,7 @@ REACT_APP_TERMS_VERSION=1 REACT_APP_COOKIE_DOMAIN=localhost MATOMO_URL=http://matomo -PE_API_URL=http://localhost:5000 +PE_API_URL=http://localhost:8000/healthcheck EXPORT_BUCKET_NAME=crossfeed-local-exports From 19650a808a2eb0b61de86c1267e8b92ee7f962a8 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Fri, 25 Oct 2024 08:23:11 -0400 Subject: [PATCH 080/314] Run linter --- .../xfd_django/xfd_api/api_methods/proxy.py | 28 ++++++++----- .../xfd_django/xfd_api/tests/test_proxy.py | 40 ++++++++----------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/proxy.py b/backend/src/xfd_django/xfd_api/api_methods/proxy.py index 12fcbe9b..4e94ee4c 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/proxy.py +++ b/backend/src/xfd_django/xfd_api/api_methods/proxy.py @@ -3,14 +3,15 @@ # Standard Python Libraries import os from typing import List, Optional -import httpx # Third-Party Libraries from django.shortcuts import render from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request +from fastapi.responses import RedirectResponse, Response from fastapi.security import APIKeyHeader, OAuth2PasswordBearer +import httpx from pydantic import UUID4 -from fastapi.responses import RedirectResponse, Response + # Helper function to handle cookie manipulation def manipulate_cookie(request: Request, cookie_name: str): @@ -21,16 +22,21 @@ def manipulate_cookie(request: Request, cookie_name: str): # Helper function to proxy requests -async def proxy_request(request: Request, target_url: str, path: Optional[str] = None, cookie_name: Optional[str] = None): +async def proxy_request( + request: Request, + target_url: str, + path: Optional[str] = None, + cookie_name: Optional[str] = None, +): """Proxy the request to the target URL.""" headers = dict(request.headers) - + # Cookie manipulation for specific cookie names if cookie_name: cookies = manipulate_cookie(request, cookie_name) if cookies: - headers['Cookie'] = f"{cookie_name}={cookies[cookie_name]}" - + headers["Cookie"] = f"{cookie_name}={cookies[cookie_name]}" + # Make the request to the target URL async with httpx.AsyncClient() as client: proxy_response = await client.request( @@ -38,15 +44,15 @@ async def proxy_request(request: Request, target_url: str, path: Optional[str] url=f"{target_url}/{path}", headers=headers, params=request.query_params, - content=await request.body() + content=await request.body(), ) - + # Remove chunked encoding for API Gateway compatibility proxy_response_headers = dict(proxy_response.headers) proxy_response_headers.pop("transfer-encoding", None) - + return Response( content=proxy_response.content, status_code=proxy_response.status_code, - headers=proxy_response_headers - ) \ No newline at end of file + headers=proxy_response_headers, + ) diff --git a/backend/src/xfd_django/xfd_api/tests/test_proxy.py b/backend/src/xfd_django/xfd_api/tests/test_proxy.py index 44c1bc76..f06b4eee 100644 --- a/backend/src/xfd_django/xfd_api/tests/test_proxy.py +++ b/backend/src/xfd_django/xfd_api/tests/test_proxy.py @@ -1,17 +1,18 @@ - -import pytest -from fastapi.testclient import TestClient -from xfd_api.models import User, UserType +# Standard Python Libraries from datetime import datetime import secrets -from xfd_django.asgi import app - +# Third-Party Libraries +from fastapi.testclient import TestClient +import pytest from xfd_api.auth import create_jwt_token +from xfd_api.models import User, UserType +from xfd_django.asgi import app # Initialize the test client with the FastAPI app client = TestClient(app) + @pytest.mark.django_db(transaction=True) def test_standard_user_not_authorized_to_access_pe_proxy(): """Test that a standard user is not authorized to access P&E proxy.""" @@ -24,16 +25,13 @@ def test_standard_user_not_authorized_to_access_pe_proxy(): createdAt=datetime.now(), updatedAt=datetime.now(), ) - + # Generate a JWT token for the user token = create_jwt_token(user) - + # Make a GET request to the P&E proxy endpoint with the user's token - response = client.get( - "/pe", - headers={"Authorization": f"Bearer {token}"} - ) - + response = client.get("/pe", headers={"Authorization": f"Bearer {token}"}) + # Assert that the user receives a 403 Unauthorized response assert response.status_code == 403 assert response.json() == {"detail": "Unauthorized"} @@ -51,16 +49,13 @@ def test_global_admin_authorized_to_access_pe_proxy(): createdAt=datetime.now(), updatedAt=datetime.now(), ) - + # Generate a JWT token for the global admin token = create_jwt_token(user) - + # Make a GET request to the P&E proxy endpoint with the global admin's token - response = client.get( - "/pe", - headers={"Authorization": f"Bearer {token}"} - ) - + response = client.get("/pe", headers={"Authorization": f"Bearer {token}"}) + # Assert that the global admin is authorized and receives either a 200 or 504 response assert response.status_code in [200, 504] @@ -77,11 +72,10 @@ def test_global_view_user_authorized_to_access_pe_proxy(): createdAt=datetime.now(), updatedAt=datetime.now(), ) - + # Make a GET request to the P&E proxy endpoint with the global view user's token response = client.get( - "/pe", - headers={"Authorization": "Bearer " + create_jwt_token(user)} + "/pe", headers={"Authorization": "Bearer " + create_jwt_token(user)} ) print(response.json()) # Assert that the global view user is authorized and receives either a 200 or 504 response From 4565fcca5f05491f75b27725bd93b29f2463fcf5 Mon Sep 17 00:00:00 2001 From: nickviola Date: Fri, 25 Oct 2024 08:16:34 -0500 Subject: [PATCH 081/314] Update search logic and fix elastic search local container --- backend/requirements.txt | 3 +- .../xfd_django/xfd_api/api_methods/search.py | 102 +++++++++++------- .../xfd_api/helpers/elastic_search.py | 96 +++++++---------- .../xfd_api/schema_models/search.py | 43 ++++++++ backend/src/xfd_django/xfd_api/views.py | 39 ++++--- docker-compose.yml | 2 +- 6 files changed, 169 insertions(+), 116 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/schema_models/search.py diff --git a/backend/requirements.txt b/backend/requirements.txt index 0e1d1968..f8760e3e 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,8 +1,9 @@ +aiohttp boto3 cryptography==38.0.0 django docker -elasticsearch +elasticsearch==7.9.0 fastapi==0.111.0 mangum==0.17.0 minio diff --git a/backend/src/xfd_django/xfd_api/api_methods/search.py b/backend/src/xfd_django/xfd_api/api_methods/search.py index 1ef8a4db..f412e95b 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/search.py +++ b/backend/src/xfd_django/xfd_api/api_methods/search.py @@ -15,27 +15,13 @@ get_tag_organizations, is_global_view_admin, ) -from xfd_api.helpers.elastic_search import build_request +from xfd_api.helpers.elastic_search import build_elasticsearch_query, es - -class Filter(BaseModel): - field: str - values: List[str] - type: str - - -class SearchBody(BaseModel): - current: int - results_per_page: int - search_term: str - sort_direction: str - sort_field: str - filters: List[Filter] - organization_ids: Optional[List[UUID]] = None - organization_id: Optional[UUID] = None - tag_id: Optional[UUID] = None +from ..schema_models.search import SearchBody +# TODO: Determine if new search method works with indexes, if so: +# remove the commented out original search methods def get_options(search_body: SearchBody, event) -> Dict[str, Any]: """Determine options for filtering based on organization ID or tag ID.""" if search_body.organization_id and ( @@ -64,13 +50,11 @@ def fetch_all_results( ) -> List[Dict[str, Any]]: """Fetch all search results from Elasticsearch.""" client = Elasticsearch() - RESULTS_PER_PAGE = 1000 results = [] current = 1 while True: - request = build_request( - {**filters, "current": current, "resultsPerPage": RESULTS_PER_PAGE}, options - ) + # Define the request as an empty dictionary for now + request: Dict[str, Any] = {} current += 1 try: search_results = client.search(index="domains", body=request) @@ -83,10 +67,63 @@ def fetch_all_results( return results +def search(search_body: SearchBody, event) -> Dict[str, Any]: + """Perform a search on Elasticsearch and return results.""" + request: Dict[str, Any] = {} + + client = Elasticsearch() + try: + search_results = client.search(index="domains", body=request) + except Exception as e: + print(f"Elasticsearch search error: {e}") + raise HTTPException(status_code=500, detail="Elasticsearch query failed") + + return search_results["hits"] + + +def search_post(request_input): + """Handle Elastic Search request + + Args: + request_input (request object): Post request object + """ + es_query = build_elasticsearch_query(request_input) + + # Perform search in Elasticsearch TODO: Confirm index name and format + response = es.search(index="domains-5", body=es_query) + + # Format response to match the required structure + result = { + "took": response["took"], + "timed_out": response["timed_out"], + "_shards": response["_shards"], + "hits": { + "total": response["hits"]["total"], + "max_score": response["hits"].get("max_score", None), + "hits": [ + { + "_index": hit["_index"], + "_type": hit["_type"], + "_id": hit["_id"], + "_score": hit["_score"], + "_source": hit["_source"], + "sort": hit.get("sort", []), + "inner_hits": hit.get("inner_hits", {}), + } + for hit in response["hits"]["hits"] + ], + }, + } + + return result + + def export(search_body: SearchBody, event) -> Dict[str, Any]: """Export the search results into a CSV and upload to S3.""" options = get_options(search_body, event) + print(f"Export Options: {options}") results = fetch_all_results(search_body.dict(), options) + print(f"Export results: {results}") # Process results for CSV for res in results: @@ -119,9 +156,10 @@ def export(search_body: SearchBody, event) -> Dict[str, Any]: writer.writeheader() writer.writerows(results) - # Save to S3 # TODO: Replace with helper logic + # TODO: Replace with helper s3 logic + # Save to S3 # s3 = boto3.client("s3") - # bucket_name = "your-bucket-name" + # bucket_name = "export-bucket-name" # csv_key = "domains.csv" # s3.put_object(Bucket=bucket_name, Key=csv_key, Body=csv_buffer.getvalue()) @@ -131,19 +169,5 @@ def export(search_body: SearchBody, event) -> Dict[str, Any]: # ) # return {"url": url} + # TODO: Modify return once s3 logic is confirmed. return {"data": results} - - -def search(search_body: SearchBody, event) -> Dict[str, Any]: - """Perform a search on Elasticsearch and return results.""" - options = get_options(search_body, event) - request = build_request(search_body.dict(), options) - - client = Elasticsearch() - try: - search_results = client.search(index="domains", body=request) - except Exception as e: - print(f"Elasticsearch search error: {e}") - raise HTTPException(status_code=500, detail="Elasticsearch query failed") - - return search_results["hits"] diff --git a/backend/src/xfd_django/xfd_api/helpers/elastic_search.py b/backend/src/xfd_django/xfd_api/helpers/elastic_search.py index ccd836ac..85381292 100644 --- a/backend/src/xfd_django/xfd_api/helpers/elastic_search.py +++ b/backend/src/xfd_django/xfd_api/helpers/elastic_search.py @@ -1,58 +1,44 @@ # Standard Python Libraries -from typing import Any, Dict - - -def build_request( - searchBody: Dict[str, Any], options: Dict[str, Any] -) -> Dict[str, Any]: - """ - Build an Elasticsearch query from the search body and options. - - :param searchBody: The body containing search parameters such as current page, resultsPerPage, - searchTerm, sortField, sortDirection, filters, etc. - :param options: Additional options for filtering by organizations, etc. - :return: The Elasticsearch query. - """ - # Pagination - current_page = searchBody.get("current", 1) - results_per_page = searchBody.get("resultsPerPage", 20) - - # Search term - search_term = searchBody.get("searchTerm", "") - - # Sorting - sort_field = searchBody.get("sortField", "createdAt") - sort_direction = searchBody.get("sortDirection", "desc") - - # Filters - filters = searchBody.get("filters", []) - - # Organization filtering - organization_ids = options.get("organizationIds", []) - - # Elasticsearch query - query = { - "from": (current_page - 1) * results_per_page, - "size": results_per_page, - "sort": [{sort_field: {"order": sort_direction}}], - "query": {"bool": {"must": [{"match": {"_all": search_term}}], "filter": []}}, - } - - # Apply filters - for filter_item in filters: - field = filter_item.get("field") - values = filter_item.get("values", []) - filter_type = filter_item.get("type", "any") - - if filter_type == "any": - query["query"]["bool"]["filter"].append({"terms": {field: values}}) - elif filter_type == "range": - query["query"]["bool"]["filter"].append({"range": {field: values}}) - - # Apply organization filters - if organization_ids: - query["query"]["bool"]["filter"].append( - {"terms": {"organization.Id": organization_ids}} +import os +from typing import Any, Dict, List, Optional + +# Third-Party Libraries +from elasticsearch import AsyncElasticsearch + +from ..schema_models.search import SearchRequest + +# Elasticsearch client +es = AsyncElasticsearch( + hosts=[os.getenv("ELASTICSEARCH_ENDPOINT")], + headers={"Content-Type": "application/json"}, # Set correct Content-Type header +) + + +# Elasticsearch Query Builder +def build_elasticsearch_query(request: SearchRequest) -> dict: + # Define the query type explicitly + query: Dict[str, Any] = {"bool": {"must": [], "filter": []}} + + # Add search term + if request.searchTerm: + query["bool"]["must"].append( + { + "multi_match": { + "query": request.searchTerm, + "fields": ["name^3", "organization.name", "organization.acronym"], + } + } ) - return query + # Add filters + for filter in request.filters: + if filter.type == "any": + query["bool"]["filter"].append({"terms": {filter.field: filter.values}}) + + # Return the query with sorting + return { + "query": query, + "sort": [{request.sortField: {"order": request.sortDirection}}], + "from": (request.current - 1) * request.resultsPerPage, + "size": request.resultsPerPage, + } diff --git a/backend/src/xfd_django/xfd_api/schema_models/search.py b/backend/src/xfd_django/xfd_api/schema_models/search.py new file mode 100644 index 00000000..2ad22f69 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/schema_models/search.py @@ -0,0 +1,43 @@ +# Standard Python Libraries +from typing import Any, List, Optional +from uuid import UUID + +# Third-Party Libraries +from pydantic import BaseModel + + +# Input request schema +class Filter(BaseModel): + field: str + values: List[str] + type: str + + +# TODO this is based on current payload Ajust as needed +class SearchRequest(BaseModel): + current: int + filters: List[Filter] + resultsPerPage: int + searchTerm: Optional[str] = "" + sortDirection: str = "asc" + sortField: str = "name" + + +# Response schema (based on your example) +class SearchResponse(BaseModel): + took: int + timed_out: bool + _shards: Any + hits: Any + + +class SearchBody(BaseModel): + current: int + results_per_page: int + search_term: str + sort_direction: str + sort_field: str + filters: List[Filter] + organization_ids: Optional[List[UUID]] = None + organization_id: Optional[UUID] = None + tag_id: Optional[UUID] = None diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index 9d0980cc..22bc904f 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -18,11 +18,12 @@ """ # Standard Python Libraries -from typing import List, Optional +from re import A +from typing import Any, List, Optional # Third-Party Libraries from django.shortcuts import render -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from fastapi.security import APIKeyHeader, OAuth2PasswordBearer # from .schemas import Cpe @@ -36,7 +37,7 @@ from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import export_domains, get_domain_by_id, search_domains from .api_methods.organization import get_organizations, read_orgs -from .api_methods.search import SearchBody, export, search +from .api_methods.search import export, search_post from .api_methods.user import get_users from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user @@ -52,6 +53,7 @@ from .schema_models.notification import Notification as NotificationSchema from .schema_models.organization import Organization as OrganizationSchema from .schema_models.role import Role as RoleSchema +from .schema_models.search import SearchBody, SearchRequest, SearchResponse from .schema_models.user import User as UserSchema from .schema_models.vulnerability import Vulnerability as VulnerabilitySchema @@ -553,28 +555,25 @@ async def invoke_scheduler(current_user: User = Depends(get_current_active_user) return response -@api_router.post("/search") -async def search_endpoint(request: Request, body: SearchBody): +@api_router.post( + "/search", + dependencies=[Depends(get_current_active_user)], + response_model=SearchResponse, + tags=["Search"], +) +async def search(request: SearchRequest): try: - # Example of parsing UUIDs correctly - organization_id = body.organization_id - tag_id = body.tag_id - - # Search logic - # Using the parsed and validated UUIDs in the search - results = { - "current": body.current, - "organization_id": str(organization_id) if organization_id else None, - "tag_id": str(tag_id) if tag_id else None, - } - - return results + search_post(request) except Exception as e: - raise HTTPException(status_code=400, detail=str(e)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) -@api_router.post("/search/export") +@api_router.post( + "/search/export", dependencies=[Depends(get_current_active_user)], tags=["Search"] +) async def export_endpoint(request: Request): try: body = await request.json() diff --git a/docker-compose.yml b/docker-compose.yml index a6585a7f..d745f082 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,7 +106,7 @@ services: - 9200:9200 - 9300:9300 logging: - driver: none + driver: json-file # kib: # image: docker.elastic.co/kibana/kibana:7.9.0 From df742ac07b5e78287c586d2cf9a72612a59ef420 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Fri, 25 Oct 2024 09:40:48 -0500 Subject: [PATCH 082/314] Add Update Users Endpoint Add update users endpoint to views Add update users definition to api_methods/user.py Refactor user endpoints to use Request object for passing arguments --- .../xfd_django/xfd_api/api_methods/user.py | 144 +++++++++++++++--- backend/src/xfd_django/xfd_api/views.py | 42 +++-- 2 files changed, 150 insertions(+), 36 deletions(-) diff --git a/backend/src/xfd_django/xfd_api/api_methods/user.py b/backend/src/xfd_django/xfd_api/api_methods/user.py index ed9eb5c0..9359aae3 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/user.py +++ b/backend/src/xfd_django/xfd_api/api_methods/user.py @@ -7,41 +7,57 @@ from typing import List, Optional # Third-Party Libraries -from fastapi import HTTPException +from fastapi import HTTPException, Request from fastapi.responses import JSONResponse +from ..auth import ( + can_access_user, + is_global_view_admin, + is_global_write_admin, + is_regional_admin, +) from ..models import User +from ..schema_models.user import UpdateUser as UpdateUserSchema from ..schema_models.user import User as UserSchema -async def accept_terms(current_user: User, version: str): +async def accept_terms(request: Request): """ Accept the latest terms of service. - Args: - current_user (User): The current authenticated user. - version (str): The version of the terms of service. + request : The HTTP request containing the user and the terms version. Returns: User: The updated user. """ - if not current_user: - raise HTTPException(status_code=401, detail="User not authenticated.") - - current_user.dateAcceptedTerms = datetime.now() - current_user.acceptedTermsVersion = version - current_user.save() - - return UserSchema.from_orm(current_user) + try: + current_user = request.state.user + if not current_user: + raise HTTPException(status_code=401, detail="User not authenticated.") + + body = await request.json() + version = body.get("version") + if not version: + raise HTTPException( + status_code=400, detail="Missing version in request body." + ) + + current_user.dateAcceptedTerms = datetime.now() + current_user.acceptedTermsVersion = version + current_user.save() + + return UserSchema.model_validate(current_user) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) # TODO: Add user context and permissions -def delete_user(user_id: str): +def delete_user(request: Request): """ Delete a user by ID. - Args: - user_id : The ID of the user to delete. + request : The HTTP request containing authorization and target for deletion.. + Raises: HTTPException: If the user is not authorized or the user is not found. @@ -50,28 +66,110 @@ def delete_user(user_id: str): """ try: - user = User.objects.get(id=user_id) - result = user.delete() + # current_user = request.state.user + target_user_id = request.path_params["user_id"] + target_user = User.objects.get(id=target_user_id) + result = target_user.delete() return JSONResponse(status_code=200, content={"result": result}) + except Exception as e: raise HTTPException(status_code=500, detail=str(e)) -def get_users(regionId): +def get_users(request: Request): """ - Retrieve a list of users based on optional filter parameters. + Retrieve a list of all users. + Args: + request : The HTTP request containing authorization information. + + Raises: + HTTPException: If the user is not authorized. + + Returns: + List[User]: A list of all users. + """ + try: + current_user = request.state.user + if not (is_global_view_admin(current_user) or is_regional_admin(current_user)): + raise HTTPException(status_code=401, detail="Unauthorized") + + users = User.objects.all().prefetch_related("roles", "roles.organization") + return [UserSchema.model_validate(user) for user in users] + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +def get_users_v2(request: Request): + """ + Retrieve a list of users based on optional filter parameters. Args: - regionId : Region ID to filter users by. + request : The HTTP request containing query parameters. + Raises: HTTPException: If the user is not authorized or no users are found. Returns: List[User]: A list of users matching the filter criteria. """ + try: + query_params = request.query_params + filters = {} + + if "state" in query_params: + filters["state"] = query_params["state"] + if "regionId" in query_params: + filters["regionId"] = query_params["regionId"] + if "invitePending" in query_params: + filters["invitePending"] = query_params["invitePending"] + + users = User.objects.filter(**filters).prefetch_related("roles") + return [UserSchema.model_validate(user) for user in users] + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +async def update_user(request: Request): + """ + Update a particular user. + Args: + request: The HTTP request containing the update data. + + Raises: + HTTPException: If the user is not authorized or the user is not found. + + Returns: + User: The updated user. + """ try: - users = User.objects.filter(regionId=regionId).prefetch_related("roles") - return [UserSchema.from_orm(user) for user in users] + # Check if the current user can access the user to be updated + current_user = request.state.user + target_user_id = request.path_params["user_id"] + if not can_access_user(current_user, target_user_id): + raise HTTPException(status_code=401, detail="Unauthorized") + + # Validate the user ID + if not target_user_id or not User.objects.filter(id=target_user_id).exists(): + raise HTTPException(status_code=404, detail="User not found") + + # Parse and validate the request body + body = await request.json() + update_data = UpdateUserSchema(**body) + + # Check if the current user can set the userType + if not is_global_write_admin(current_user) and update_data.userType: + raise HTTPException(status_code=401, detail="Unauthorized to set userType") + + # Retrieve the user to be updated + user = User.objects.get(id=target_user_id) + user.firstName = update_data.firstName or user.firstName + user.lastName = update_data.lastName or user.lastName + user.fullName = f"{user.firstName} {user.lastName}" + user.userType = update_data.userType or user.userType + + # Save the updated user + user.save() + + return user except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/src/xfd_django/xfd_api/views.py b/backend/src/xfd_django/xfd_api/views.py index bf94746c..4aa965eb 100644 --- a/backend/src/xfd_django/xfd_api/views.py +++ b/backend/src/xfd_django/xfd_api/views.py @@ -36,7 +36,7 @@ from .api_methods.cve import get_cves_by_id, get_cves_by_name from .api_methods.domain import export_domains, get_domain_by_id, search_domains from .api_methods.organization import get_organizations, read_orgs -from .api_methods.user import accept_terms, delete_user, get_users +from .api_methods.user import accept_terms, delete_user, get_users, update_user from .api_methods.vulnerability import get_vulnerability_by_id, update_vulnerability from .auth import get_current_active_user from .login_gov import callback, login @@ -112,7 +112,6 @@ async def call_read_orgs(): async def list_assessments(): """ Lists all assessments for the logged-in user. - Args: current_user (User): The current authenticated user. @@ -320,28 +319,27 @@ async def callback_route(request: Request): # GET Current User @api_router.post("/users/acceptTerms", tags=["Users"]) async def call_accept_terms( - version: str, current_user: User = Depends(get_current_active_user) + request: Request, current_user: User = Depends(get_current_active_user) ): """ Accept the latest terms of service. - Args: version (str): The version of the terms of service. Returns: User: The updated user. """ - return accept_terms(current_user, version) + return accept_terms(request) # TODO: Add authentication and permissions @api_router.delete("/users/{userId}", tags=["Users"]) -async def call_delete_user(userId: str): +async def call_delete_user(userId, request: Request): """ Delete a user by ID. - Args: userId : The ID of the user to delete. + request : The HTTP request containing authorization. Raises: HTTPException: If the user is not authorized or the user is not found. @@ -349,7 +347,8 @@ async def call_delete_user(userId: str): Returns: JSONResponse: The result of the deletion. """ - return delete_user(userId) + request.path_params["user_id"] = userId + return delete_user(request) @api_router.get("/users/me", tags=["Users"]) @@ -358,17 +357,16 @@ async def read_users_me(current_user: User = Depends(get_current_active_user)): @api_router.get( - "/users/{regionId}", + "/users", response_model=List[UserSchema], # dependencies=[Depends(get_current_active_user)], tags=["Users"], ) -async def call_get_users(regionId): +async def call_get_users(request: Request): """ Call get_users() - Args: - regionId: Region IDs to filter users by. + request : The HTTP request containing query parameters. Raises: HTTPException: If the user is not authorized or no users are found. @@ -376,7 +374,25 @@ async def call_get_users(regionId): Returns: List[User]: A list of users matching the filter criteria. """ - return get_users(regionId) + return get_users(request) + + +@api_router.post("/users/{userId}", tags=["Users"]) +async def call_update_user(userId, request: Request): + """ + Update a user by ID. + Args: + userId : The ID of the user to update. + request : The HTTP request containing authorization and target for update. + + Raises: + HTTPException: If the user is not authorized or the user is not found. + + Returns: + JSONResponse: The result of the update. + """ + request.path_params["user_id"] = userId + return update_user(request) ###################### From 3f900f91a7834b118074afc701470743f5fb52d7 Mon Sep 17 00:00:00 2001 From: aloftus23 Date: Fri, 25 Oct 2024 10:47:24 -0400 Subject: [PATCH 083/314] Revert "Merge branch 'develop' into AL-python-serverless-proxy" This reverts commit 4b85bf2d3443738c8815ae085c8206b49772ea26, reversing changes made to 19650a808a2eb0b61de86c1267e8b92ee7f962a8. --- .github/workflows/playwright.yml | 1 + backend/src/api/app.ts | 165 +---- backend/src/api/auth.ts | 20 +- backend/src/api/domains.ts | 5 +- backend/src/api/logs.ts | 175 ----- backend/src/api/organizations.ts | 5 +- backend/src/api/saved-searches.ts | 7 + backend/src/api/scans.ts | 9 - backend/src/api/stats.ts | 4 - backend/src/api/users.ts | 9 +- backend/src/api/vulnerabilities.ts | 4 - backend/src/models/connection.ts | 4 +- backend/src/models/domain.ts | 10 - backend/src/models/index.ts | 1 - backend/src/models/log.ts | 19 - backend/src/models/saved-search.ts | 11 + backend/src/tasks/flagFloatingIps.ts | 41 -- backend/src/tasks/helpers/checkIpInCidr.ts | 30 - backend/src/tasks/helpers/checkOrgIsFceb.ts | 39 -- backend/src/tasks/saved-search.ts | 15 + backend/src/tasks/search-sync-domains.ts | 4 - backend/src/tasks/syncdb.ts | 1 - .../censysCertificates.test.ts.snap | 72 --- .../__snapshots__/censysIpv4.test.ts.snap | 72 --- .../__snapshots__/lookingGlass.test.ts.snap | 6 - .../__snapshots__/search-sync.test.ts.snap | 2 - .../test/__snapshots__/shodan.test.ts.snap | 6 - .../src/tasks/test/censysCertificates.test.ts | 12 - backend/src/tasks/test/censysIpv4.test.ts | 12 - backend/src/tasks/test/cve.test.ts | 22 - backend/src/tasks/test/dnstwist.test.ts | 6 - backend/src/tasks/test/es-client.test.ts | 1 - backend/src/tasks/test/findomain.test.ts | 1 - .../tasks/test/helpers/getAllDomains.test.ts | 4 - backend/src/tasks/test/intrigue-ident.test.ts | 1 - backend/src/tasks/test/lookingGlass.test.ts | 3 - backend/src/tasks/test/search-sync.test.ts | 9 - backend/src/tasks/test/shodan.test.ts | 7 +- backend/src/tasks/test/sslyze.test.ts | 2 - backend/src/tools/logger.ts | 128 ---- backend/src/worker.ts | 2 - .../__snapshots__/saved-searches.test.ts.snap | 2 + backend/test/__snapshots__/stats.test.ts.snap | 4 - backend/test/domains.test.ts | 27 - backend/test/saved-searches.test.ts | 49 +- backend/test/stats.test.ts | 4 - backend/test/vulnerabilities.test.ts | 21 - frontend/package-lock.json | 6 - frontend/package.json | 1 - frontend/public/index.html | 8 + frontend/scripts/api.js | 18 +- frontend/scripts/docs.js | 6 - frontend/src/App.tsx | 298 ++++----- frontend/src/assets/nvd.jpeg | Bin 5938 -> 0 bytes frontend/src/components/DomainDetails.tsx | 21 +- frontend/src/components/DrawerInterior.tsx | 299 +++++---- frontend/src/components/FilterDrawerV2.tsx | 11 +- frontend/src/components/Footer/Footer.tsx | 1 - frontend/src/components/Header.tsx | 274 +++++--- frontend/src/components/Layout.tsx | 8 +- frontend/src/components/Logs/Logs.tsx | 209 ------ .../OrganizationList/OrganizationList.tsx | 6 +- ...tionFilters.tsx => OrganizationSearch.tsx} | 189 ++---- .../ReadySetCyber/RSCAuthLoginCreate.tsx | 240 +++++++ .../components/ReadySetCyber/RSCFooter.tsx | 11 +- .../src/components/ReadySetCyber/RSCLogin.tsx | 224 ++++++- .../components/ReadySetCyber/RSCQuestion.tsx | 1 - .../ReadySetCyber/RSCRegisterForm.tsx | 233 +++++++ .../ReadySetCyber/RSCregisterFormStyle.ts | 48 ++ frontend/src/components/RouteGuard.tsx | 1 + .../SaveSearchModal/SaveSearchModal.tsx | 348 ---------- frontend/src/components/SearchBar.tsx | 2 +- .../SkipToMainContent/SkipToMainContent.tsx | 30 +- frontend/src/components/Subnav.tsx | 2 +- frontend/src/components/Table/Table.tsx | 4 +- .../__snapshots__/header.spec.tsx.snap | 201 +++--- .../__snapshots__/layout.spec.tsx.snap | 47 +- .../__snapshots__/searchBar.spec.tsx.snap | 2 +- frontend/src/context/SavedSearchContext.ts | 20 - .../context/SavedSearchContextProvider.tsx | 58 -- frontend/src/context/SearchProvider/types.ts | 8 +- .../src/context/StaticsContextProvider.tsx | 7 +- frontend/src/context/Theme.tsx | 15 - frontend/src/context/userStateUtils.ts | 2 - frontend/src/hooks/useDomainApi.ts | 25 +- frontend/src/hooks/useUserLevel.ts | 16 +- frontend/src/hooks/useUserTypeFilters.ts | 32 +- frontend/src/index.tsx | 4 - frontend/src/pages/AdminTools/AdminTools.tsx | 5 - frontend/src/pages/Domains/Domains.tsx | 29 +- .../src/pages/Organization/OrgMembers.tsx | 12 +- .../src/pages/Organization/OrgScanHistory.tsx | 200 ------ .../src/pages/Organization/OrgSettings.tsx | 486 -------------- .../src/pages/Organization/Organization.tsx | 605 +++++++++++++++++- frontend/src/pages/Organization/style.ts | 116 ++++ .../src/pages/RegionUsers/RegionUsers.tsx | 103 +-- frontend/src/pages/Risk/Risk.tsx | 38 +- .../src/pages/Risk/TopVulnerableDomains.tsx | 19 +- .../src/pages/Risk/TopVulnerablePorts.tsx | 118 ++-- .../src/pages/Risk/VulnerabilityBarChart.tsx | 81 +-- frontend/src/pages/Risk/VulnerabilityCard.tsx | 5 +- frontend/src/pages/Risk/style.ts | 2 +- frontend/src/pages/Risk/utils.ts | 3 +- frontend/src/pages/Scans/ScanTasksView.tsx | 6 +- frontend/src/pages/Scans/ScansView.tsx | 61 +- frontend/src/pages/Search/Dashboard.tsx | 458 +++++++++++++ frontend/src/pages/Search/DashboardPage.tsx | 2 +- frontend/src/pages/Search/FilterDrawer.tsx | 497 ++++++++++++++ frontend/src/pages/Search/FilterTags.tsx | 90 +-- frontend/src/pages/Search/Inventory.tsx | 280 -------- frontend/src/pages/Search/SortBar.tsx | 21 +- .../pages/Search/Styling/filterTagsStyle.ts | 3 +- .../src/pages/Search/Styling/sortBarStyle.ts | 8 + frontend/src/pages/TermsOfUse/TermsOfUse.tsx | 7 +- .../__snapshots__/termsOfUse.spec.tsx.snap | 4 +- frontend/src/pages/Users/UserForm.tsx | 16 +- frontend/src/pages/Users/Users.tsx | 2 +- .../pages/Vulnerabilities/Vulnerabilities.tsx | 241 +++---- .../src/pages/Vulnerability/CveSection.tsx | 5 +- .../src/pages/Vulnerability/Vulnerability.tsx | 70 +- frontend/src/types/domain.ts | 54 +- frontend/src/types/index.ts | 21 +- frontend/src/types/org-query.ts | 8 - frontend/src/types/query.ts | 11 - frontend/src/types/saved-search.ts | 3 + frontend/src/types/stats.ts | 2 +- frontend/src/types/vulnerability.ts | 38 ++ frontend/src/types/webpage.ts | 16 + 128 files changed, 3613 insertions(+), 4123 deletions(-) delete mode 100644 backend/src/api/logs.ts delete mode 100644 backend/src/models/log.ts delete mode 100644 backend/src/tasks/flagFloatingIps.ts delete mode 100644 backend/src/tasks/helpers/checkIpInCidr.ts delete mode 100644 backend/src/tasks/helpers/checkOrgIsFceb.ts delete mode 100644 backend/src/tools/logger.ts delete mode 100644 frontend/src/assets/nvd.jpeg delete mode 100644 frontend/src/components/Logs/Logs.tsx rename frontend/src/components/{RegionAndOrganizationFilters.tsx => OrganizationSearch.tsx} (64%) create mode 100644 frontend/src/components/ReadySetCyber/RSCAuthLoginCreate.tsx create mode 100644 frontend/src/components/ReadySetCyber/RSCRegisterForm.tsx create mode 100644 frontend/src/components/ReadySetCyber/RSCregisterFormStyle.ts delete mode 100644 frontend/src/components/SaveSearchModal/SaveSearchModal.tsx delete mode 100644 frontend/src/context/SavedSearchContext.ts delete mode 100644 frontend/src/context/SavedSearchContextProvider.tsx delete mode 100644 frontend/src/pages/Organization/OrgScanHistory.tsx delete mode 100644 frontend/src/pages/Organization/OrgSettings.tsx create mode 100644 frontend/src/pages/Organization/style.ts create mode 100644 frontend/src/pages/Search/Dashboard.tsx create mode 100644 frontend/src/pages/Search/FilterDrawer.tsx delete mode 100644 frontend/src/pages/Search/Inventory.tsx delete mode 100644 frontend/src/types/org-query.ts delete mode 100644 frontend/src/types/query.ts create mode 100644 frontend/src/types/vulnerability.ts create mode 100644 frontend/src/types/webpage.ts diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 7e8d4bed..ad27beba 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -2,6 +2,7 @@ name: UI Testing on: deployment_status: + push: defaults: run: working-directory: ./playwright diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 3c97f6fd..093f0f46 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -13,7 +13,6 @@ import * as search from './search'; import * as vulnerabilities from './vulnerabilities'; import * as organizations from './organizations'; import * as scans from './scans'; -import * as logs from './logs'; import * as users from './users'; import * as scanTasks from './scan-tasks'; import * as stats from './stats'; @@ -23,13 +22,12 @@ import * as reports from './reports'; import * as savedSearches from './saved-searches'; import rateLimit from 'express-rate-limit'; import { createProxyMiddleware } from 'http-proxy-middleware'; -import { Organization, User, UserType, connectToDatabase } from '../models'; +import { User, UserType, connectToDatabase } from '../models'; import * as assessments from './assessments'; import * as jwt from 'jsonwebtoken'; import { Request, Response, NextFunction } from 'express'; import fetch from 'node-fetch'; import * as searchOrganizations from './organizationSearch'; -import { Logger, RecordMessage } from '../tools/logger'; const sanitizer = require('sanitizer'); @@ -45,40 +43,27 @@ if ( setInterval(() => scheduler({}, {} as any, () => null), 30000); } -const handlerToExpress = - (handler, message?: RecordMessage, action?: string) => async (req, res) => { - const { statusCode, body } = await handler( - { - pathParameters: req.params, - query: req.query, - requestContext: req.requestContext, - body: JSON.stringify(req.body || '{}'), - headers: req.headers, - path: req.originalUrl - }, - {} - ); - // Add additional status codes that we may return for succesfull requests - - if (message && action) { - const logger = new Logger(req); - if (statusCode === 200) { - logger.record(action, 'success', message, body); - } else { - logger.record(action, 'fail', message, body); - } - } - - try { - const parsedBody = JSON.parse(sanitizer.sanitize(body)); - res.status(200).json(parsedBody); - } catch (e) { - // Not valid JSON - may be a string response. - console.log('Error?', e); - res.setHeader('content-type', 'text/plain'); - res.status(statusCode).send(sanitizer.sanitize(body)); - } - }; +const handlerToExpress = (handler) => async (req, res) => { + const { statusCode, body } = await handler( + { + pathParameters: req.params, + query: req.query, + requestContext: req.requestContext, + body: JSON.stringify(req.body || '{}'), + headers: req.headers, + path: req.originalUrl + }, + {} + ); + try { + const parsedBody = JSON.parse(sanitizer.sanitize(body)); + res.status(statusCode).json(parsedBody); + } catch (e) { + // Not a JSON body + res.setHeader('content-type', 'text/plain'); + res.status(statusCode).send(sanitizer.sanitize(body)); + } +}; const app = express(); @@ -141,12 +126,6 @@ app.use( }) ); -//Middleware to set Cache-Control headers -app.use((req, res, next) => { - res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate'); - next(); -}); - app.use((req, res, next) => { res.setHeader('X-XSS-Protection', '0'); // Okta header @@ -255,7 +234,10 @@ app.post('/auth/okta-callback', async (req, res) => { oktaId: oktaId, firstName: decodedToken.given_name, lastName: decodedToken.family_name, - invitePending: true + invitePending: true, + // TODO: Replace these default Region/State values with user selection + state: 'Virginia', + regionId: '3' }); await user.save(); } else { @@ -595,7 +577,6 @@ authenticatedRoute.delete( handlerToExpress(savedSearches.del) ); authenticatedRoute.get('/scans', handlerToExpress(scans.list)); -authenticatedRoute.post('/logs/search', handlerToExpress(logs.list)); authenticatedRoute.get('/granularScans', handlerToExpress(scans.listGranular)); authenticatedRoute.post('/scans', handlerToExpress(scans.create)); authenticatedRoute.get('/scans/:scanId', handlerToExpress(scans.get)); @@ -653,39 +634,12 @@ authenticatedRoute.delete( ); authenticatedRoute.post( '/v2/organizations/:organizationId/users', - handlerToExpress( - organizations.addUserV2, - async (req, token) => { - const orgId = req?.params?.organizationId; - const userId = req?.body?.userId; - const role = req?.body?.role; - if (orgId && userId) { - const orgRecord = await Organization.findOne({ where: { id: orgId } }); - const userRecord = await User.findOne({ where: { id: userId } }); - return { - timestamp: new Date(), - userPerformedAssignment: token?.id, - organization: orgRecord, - role: role, - user: userRecord - }; - } - return { - timestamp: new Date(), - userId: token?.id, - updatePayload: req.body - }; - }, - 'USER ASSIGNED' - ) + handlerToExpress(organizations.addUserV2) ); - authenticatedRoute.post( '/organizations/:organizationId/roles/:roleId/approve', handlerToExpress(organizations.approveRole) ); - -// TO-DO Add logging => /users => user has an org and you change them to a new organization authenticatedRoute.post( '/organizations/:organizationId/roles/:roleId/remove', handlerToExpress(organizations.removeRole) @@ -703,58 +657,9 @@ authenticatedRoute.post( handlerToExpress(organizations.checkDomainVerification) ); authenticatedRoute.post('/stats', handlerToExpress(stats.get)); -authenticatedRoute.post( - '/users', - handlerToExpress( - users.invite, - async (req, token, responseBody) => { - const userId = token?.id; - if (userId) { - const userRecord = await User.findOne({ where: { id: userId } }); - return { - timestamp: new Date(), - userPerformedInvite: userRecord, - invitePayload: req.body, - createdUserRecord: responseBody - }; - } - return { - timestamp: new Date(), - userId: token?.id, - invitePayload: req.body, - createdUserRecord: responseBody - }; - }, - 'USER INVITE' - ) -); +authenticatedRoute.post('/users', handlerToExpress(users.invite)); authenticatedRoute.get('/users', handlerToExpress(users.list)); -authenticatedRoute.delete( - '/users/:userId', - handlerToExpress( - users.del, - async (req, token, res) => { - const userId = req?.params?.userId; - const userPerformedRemovalId = token?.id; - if (userId && userPerformedRemovalId) { - const userPerformdRemovalRecord = await User.findOne({ - where: { id: userPerformedRemovalId } - }); - return { - timestamp: new Date(), - userPerformedRemoval: userPerformdRemovalRecord, - userRemoved: userId - }; - } - return { - timestamp: new Date(), - userPerformedRemoval: token?.id, - userRemoved: req.params.userId - }; - }, - 'USER DENY/REMOVE' - ) -); +authenticatedRoute.delete('/users/:userId', handlerToExpress(users.del)); authenticatedRoute.get( '/users/state/:state', handlerToExpress(users.getByState) @@ -779,17 +684,7 @@ authenticatedRoute.post( authenticatedRoute.put( '/users/:userId/register/approve', checkGlobalAdminOrRegionAdmin, - handlerToExpress( - users.registrationApproval, - async (req, token) => { - return { - timestamp: new Date(), - userId: token?.id, - userToApprove: req.params.userId - }; - }, - 'USER APPROVE' - ) + handlerToExpress(users.registrationApproval) ); authenticatedRoute.put( diff --git a/backend/src/api/auth.ts b/backend/src/api/auth.ts index 9a248d5c..a40356e8 100644 --- a/backend/src/api/auth.ts +++ b/backend/src/api/auth.ts @@ -21,7 +21,6 @@ export interface UserToken { org: string; role: 'user' | 'admin'; }[]; - regionId: string | null; dateAcceptedTerms: Date | undefined; acceptedTermsVersion: string | undefined; lastLoggedIn: Date | undefined; @@ -96,7 +95,6 @@ export const userTokenBody = (user): UserToken => ({ email: user.email, userType: user.userType, dateAcceptedTerms: user.dateAcceptedTerms, - regionId: user.regionId, acceptedTermsVersion: user.acceptedTermsVersion, lastLoggedIn: user.lastLoggedIn, loginBlockedByMaintenance: user.loginBlockedByMaintenance, @@ -312,25 +310,9 @@ export const matchesRegion = async ( return regionId === (await getRegion(organizationId)); }; -/** Checks if the authorizer's region is the same as the user's being modified */ -export const matchesUserRegion = ( - event: APIGatewayProxyEvent, - userRegionId?: string -) => { - // Global admins can match with any region - if (isGlobalWriteAdmin(event)) return true; - if (!event.requestContext.authorizer || !userRegionId) return false; - return userRegionId === event.requestContext.authorizer.regionId; -}; - /** Checks if the current user is allowed to access (modify) a user with id userId */ export const canAccessUser = (event: APIGatewayProxyEvent, userId?: string) => { - return ( - userId && - (userId === getUserId(event) || - isGlobalWriteAdmin(event) || - isRegionalAdmin(event)) - ); + return userId && (userId === getUserId(event) || isGlobalWriteAdmin(event)); }; /** Checks if a user is an admin of the given organization */ diff --git a/backend/src/api/domains.ts b/backend/src/api/domains.ts index 07b7fa13..651e5233 100644 --- a/backend/src/api/domains.ts +++ b/backend/src/api/domains.ts @@ -154,10 +154,6 @@ class DomainSearch { }); } - qs.andWhere( - '(domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))' - ); - await this.filterResultQueryset(qs, event); return qs.getManyAndCount(); } @@ -173,6 +169,7 @@ class DomainSearch { * - Domains */ export const list = wrapHandler(async (event) => { + console.log('Hello, list handler'); if (!isGlobalViewAdmin(event) && getOrgMemberships(event).length === 0) { console.log('returning no results'); return { diff --git a/backend/src/api/logs.ts b/backend/src/api/logs.ts deleted file mode 100644 index 75087058..00000000 --- a/backend/src/api/logs.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { SelectQueryBuilder } from 'typeorm'; -import { Log } from '../models'; -import { validateBody, wrapHandler } from './helpers'; -import { IsDate, IsOptional, IsString } from 'class-validator'; - -type ParsedQuery = { - [key: string]: string | ParsedQuery; -}; - -const parseQueryString = (query: string): ParsedQuery => { - // Parses a query string that is used to search the JSON payload of a record - // Example => createdUserPayload.userId: 123124121424 - const result: ParsedQuery = {}; - - const parts = query.match(/(\w+(\.\w+)*):\s*[^:]+/g); - - if (!parts) { - return result; - } - - parts.forEach((part) => { - const [key, value] = part.split(/:(.+)/); - - if (!key || value === undefined) return; - - const keyParts = key.trim().split('.'); - let current = result; - - keyParts.forEach((part, index) => { - if (index === keyParts.length - 1) { - current[part] = value.trim(); - } else { - if (!current[part]) { - current[part] = {}; - } - current = current[part] as ParsedQuery; - } - }); - }); - - return result; -}; - -const generateSqlConditions = ( - parsedQuery: ParsedQuery, - jsonPath: string[] = [] -): string[] => { - const conditions: string[] = []; - - for (const [key, value] of Object.entries(parsedQuery)) { - if (typeof value === 'object') { - const newPath = [...jsonPath, key]; - conditions.push(...generateSqlConditions(value, newPath)); - } else { - const jsonField = - jsonPath.length > 0 - ? `${jsonPath.map((path) => `'${path}'`).join('->')}->>'${key}'` - : `'${key}'`; - conditions.push( - `payload ${ - jsonPath.length > 0 ? '->' : '->>' - } ${jsonField} = '${value}'` - ); - } - } - - return conditions; -}; -class Filter { - @IsString() - value: string; - - @IsString() - operator?: string; -} - -class DateFilter { - @IsDate() - value: string; - - @IsString() - operator: - | 'is' - | 'not' - | 'after' - | 'onOrAfter' - | 'before' - | 'onOrBefore' - | 'empty' - | 'notEmpty'; -} -class LogSearch { - @IsOptional() - eventType?: Filter; - @IsOptional() - result?: Filter; - @IsOptional() - timestamp?: Filter; - @IsOptional() - payload?: Filter; -} - -const generateDateCondition = (filter: DateFilter): string => { - const { operator } = filter; - - switch (operator) { - case 'is': - return `log.createdAt = :timestamp`; - case 'not': - return `log.createdAt != :timestamp`; - case 'after': - return `log.createdAt > :timestamp`; - case 'onOrAfter': - return `log.createdAt >= :timestamp`; - case 'before': - return `log.createdAt < :timestamp`; - case 'onOrBefore': - return `log.createdAt <= :timestamp`; - case 'empty': - return `log.createdAt IS NULL`; - case 'notEmpty': - return `log.createdAt IS NOT NULL`; - default: - throw new Error('Invalid operator'); - } -}; - -const filterResultQueryset = async (qs: SelectQueryBuilder, filters) => { - if (filters?.eventType) { - qs.andWhere('log.eventType ILIKE :eventType', { - eventType: `%${filters?.eventType?.value}%` - }); - } - if (filters?.result) { - qs.andWhere('log.result ILIKE :result', { - result: `%${filters?.result?.value}%` - }); - } - if (filters?.payload) { - try { - const parsedQuery = parseQueryString(filters?.payload?.value); - const conditions = generateSqlConditions(parsedQuery); - qs.andWhere(conditions[0]); - } catch (error) {} - } - - if (filters?.timestamp) { - const timestampCondition = generateDateCondition(filters?.timestamp); - try { - } catch (error) {} - qs.andWhere(timestampCondition, { - timestamp: new Date(filters?.timestamp?.value) - }); - } - - return qs; -}; - -export const list = wrapHandler(async (event) => { - const search = await validateBody(LogSearch, event.body); - - const qs = Log.createQueryBuilder('log'); - - const filterQs = await filterResultQueryset(qs, search); - - const [results, resultsCount] = await filterQs.getManyAndCount(); - - return { - statusCode: 200, - body: JSON.stringify({ - result: results, - count: resultsCount - }) - }; -}); diff --git a/backend/src/api/organizations.ts b/backend/src/api/organizations.ts index 6a75afe7..c223ba66 100644 --- a/backend/src/api/organizations.ts +++ b/backend/src/api/organizations.ts @@ -32,8 +32,7 @@ import { isRegionalAdmin, isRegionalAdminForOrganization, getOrgMemberships, - isGlobalViewAdmin, - matchesUserRegion + isGlobalViewAdmin } from './auth'; import { In } from 'typeorm'; import { plainToClass } from 'class-transformer'; @@ -982,8 +981,6 @@ export const addUserV2 = wrapHandler(async (event) => { // Get User from the database const user = await User.findOneOrFail(userId); - if (!matchesUserRegion(event, user.regionId)) return Unauthorized; - const newRoleData = { user: user, organization: org, diff --git a/backend/src/api/saved-searches.ts b/backend/src/api/saved-searches.ts index fb0d30ba..1497e429 100644 --- a/backend/src/api/saved-searches.ts +++ b/backend/src/api/saved-searches.ts @@ -61,6 +61,13 @@ class NewSavedSearch { @IsArray() filters: { field: string; values: any[]; type: string }[]; + + @IsBoolean() + createVulnerabilities: boolean; + + @IsObject() + @IsOptional() + vulnerabilityTemplate: Partial; } const PAGE_SIZE = 20; diff --git a/backend/src/api/scans.ts b/backend/src/api/scans.ts index 5d7621b6..79d20483 100644 --- a/backend/src/api/scans.ts +++ b/backend/src/api/scans.ts @@ -128,15 +128,6 @@ export const SCAN_SCHEMA: ScanSchema = { description: 'Open source tool that integrates passive APIs in order to discover target subdomains' }, - flagFloatingIps: { - type: 'fargate', - isPassive: true, - global: false, - cpu: '2048', - memory: '16384', - description: - 'Loops through all domains and determines if their associated IP can be found in a report Cidr block.' - }, hibp: { type: 'fargate', isPassive: true, diff --git a/backend/src/api/stats.ts b/backend/src/api/stats.ts index 4794b5de..54238947 100644 --- a/backend/src/api/stats.ts +++ b/backend/src/api/stats.ts @@ -94,10 +94,6 @@ export const get = wrapHandler(async (event) => { }); } - qs.andWhere( - '(domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))' - ); - // Handles the case where no orgs and no regions are set, and we pull stats for a region that will never exist if ( search.filters?.organizations?.length === 0 && diff --git a/backend/src/api/users.ts b/backend/src/api/users.ts index 08cc4a2d..da35e748 100644 --- a/backend/src/api/users.ts +++ b/backend/src/api/users.ts @@ -33,8 +33,7 @@ import { isGlobalViewAdmin, isRegionalAdmin, isOrgAdmin, - isGlobalWriteAdmin, - matchesUserRegion + isGlobalWriteAdmin } from './auth'; import { fetchAssessmentsByUser } from '../tasks/rscSync'; @@ -635,9 +634,6 @@ export const registrationApproval = wrapHandler(async (event) => { return NotFound; } - // Check if authorizer's region matches the user's - if (!matchesUserRegion(event, user.regionId)) return Unauthorized; - // Send email notification await sendRegistrationApprovedEmail( user.email, @@ -923,9 +919,6 @@ export const updateV2 = wrapHandler(async (event) => { return NotFound; } - // Check if authorizer's region matches the user's - if (!matchesUserRegion(event, user.regionId)) return Unauthorized; - if (body.state) { body.regionId = REGION_STATE_MAP[body.state]; } diff --git a/backend/src/api/vulnerabilities.ts b/backend/src/api/vulnerabilities.ts index da2fb697..65b71ad0 100644 --- a/backend/src/api/vulnerabilities.ts +++ b/backend/src/api/vulnerabilities.ts @@ -173,10 +173,6 @@ class VulnerabilitySearch { .leftJoinAndSelect('domain.organization', 'organization') .leftJoinAndSelect('vulnerability.service', 'service'); - qs.andWhere( - '(domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))' - ); - if (groupBy) { qs = qs .groupBy('title, cve, "isKev", description, severity') diff --git a/backend/src/models/connection.ts b/backend/src/models/connection.ts index a32ba88d..8f2cfdbf 100644 --- a/backend/src/models/connection.ts +++ b/backend/src/models/connection.ts @@ -21,7 +21,6 @@ import { User, Vulnerability, Webpage, - Log, // Models for the Mini Data Lake database CertScan, @@ -195,8 +194,7 @@ const connectDb = async (logging?: boolean) => { Service, User, Vulnerability, - Webpage, - Log + Webpage ], synchronize: false, name: 'default', diff --git a/backend/src/models/domain.ts b/backend/src/models/domain.ts index c33d9d10..2a6be83c 100644 --- a/backend/src/models/domain.ts +++ b/backend/src/models/domain.ts @@ -111,16 +111,6 @@ export class Domain extends BaseEntity { }) cloudHosted: boolean; - @Column({ - default: false - }) - fromCidr: boolean; - - @Column({ - default: false - }) - isFceb: boolean; - /** SSL Certificate information */ @Column({ type: 'jsonb', diff --git a/backend/src/models/index.ts b/backend/src/models/index.ts index d0b3814f..85627560 100644 --- a/backend/src/models/index.ts +++ b/backend/src/models/index.ts @@ -19,7 +19,6 @@ export * from './service'; export * from './user'; export * from './vulnerability'; export * from './webpage'; -export * from './log'; // Mini data lake models export * from './mini_data_lake/cert_scans'; export * from './mini_data_lake/cidrs'; diff --git a/backend/src/models/log.ts b/backend/src/models/log.ts deleted file mode 100644 index 2d128b18..00000000 --- a/backend/src/models/log.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { BaseEntity, Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; - -@Entity() -export class Log extends BaseEntity { - @PrimaryGeneratedColumn('uuid') - id: string; - - @Column('json') - payload: Object; - - @Column({ nullable: false }) - createdAt: Date; - - @Column({ nullable: true }) - eventType: string; - - @Column({ nullable: false }) - result: string; -} diff --git a/backend/src/models/saved-search.ts b/backend/src/models/saved-search.ts index c8b42a07..91a9f63f 100644 --- a/backend/src/models/saved-search.ts +++ b/backend/src/models/saved-search.ts @@ -42,6 +42,17 @@ export class SavedSearch extends BaseEntity { @Column() searchPath: string; + @Column({ + default: false + }) + createVulnerabilities: boolean; + + // Content of vulnerability when search is configured to create vulnerabilities from results + @Column({ type: 'jsonb', default: '{}' }) + vulnerabilityTemplate: Partial & { + title: string; + }; + @ManyToOne((type) => User, { onDelete: 'SET NULL', onUpdate: 'CASCADE' diff --git a/backend/src/tasks/flagFloatingIps.ts b/backend/src/tasks/flagFloatingIps.ts deleted file mode 100644 index e081d941..00000000 --- a/backend/src/tasks/flagFloatingIps.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { CommandOptions } from './ecs-client'; -import checkIpInCidr from './helpers/checkIpInCidr'; -import checkOrgIsFceb from './helpers/checkOrgIsFceb'; -import { Organization, connectToDatabase } from '../models'; - -export const handler = async (commandOptions: CommandOptions) => { - const { organizationId, organizationName } = commandOptions; - const db_connection = await connectToDatabase(); - const organization_repo = db_connection.getRepository(Organization); - - const organizations = await organization_repo.find({ - where: { id: organizationId }, - relations: ['domains'] - }); - - for (const organization of organizations) { - console.log('Running on ', organizationName); - const isExecutive = await checkOrgIsFceb(organization.acronym); - - if (isExecutive) { - // If executive, mark all domains as isFceb = true - for (const domain of organization.domains) { - domain.isFceb = true; - await domain.save(); // Save each domain - } - } else { - for (const domain of organization.domains) { - if (domain.ip) { - // Set fromCidr field based on the check - domain.fromCidr = await checkIpInCidr( - domain.ip, - organization.acronym - ); - - // Optionally save domain if its fromCidr value has changed - await domain.save(); // Save the domain - } - } - } - } -}; diff --git a/backend/src/tasks/helpers/checkIpInCidr.ts b/backend/src/tasks/helpers/checkIpInCidr.ts deleted file mode 100644 index 368e5ab1..00000000 --- a/backend/src/tasks/helpers/checkIpInCidr.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { getRepository } from 'typeorm'; -import { Cidr, DL_Organization, connectToDatalake2 } from '../../models'; - -export default async (ip: string, acronym: string): Promise => { - // Connect to the database - const mdl_connection = await connectToDatalake2(); - const mdl_organization_repo = mdl_connection.getRepository(DL_Organization); - - // Find the organization by acronym - const organization = await mdl_organization_repo.findOne({ - where: { acronym }, - relations: ['cidrs'] - }); - - if (!organization || organization.cidrs.length === 0) { - return false; // Return false if the organization is not found or has no CIDRs - } - - // Check if the IP is in any of the organization's CIDRs - const mdl_cidr_repo = mdl_connection.getRepository(Cidr); - const result = await mdl_cidr_repo - .createQueryBuilder('cidr') - .where('cidr.network >>= :ip', { ip }) - .andWhere('cidr.id IN (:...cidrIds)', { - cidrIds: organization.cidrs.map((cidr) => cidr.id) - }) - .getCount(); - - return result > 0; // Return true if the IP is in any CIDR, otherwise false -}; diff --git a/backend/src/tasks/helpers/checkOrgIsFceb.ts b/backend/src/tasks/helpers/checkOrgIsFceb.ts deleted file mode 100644 index c979a4c1..00000000 --- a/backend/src/tasks/helpers/checkOrgIsFceb.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { getRepository } from 'typeorm'; -import { DL_Organization, connectToDatalake2 } from '../../models'; - -export default async (acronym: string): Promise => { - // Connect to the database - const mdl_connection = await connectToDatalake2(); - const mdl_organization_repo = mdl_connection.getRepository(DL_Organization); - - // Find the organization by acronym - const organization = await mdl_organization_repo.findOne({ - where: { acronym }, - relations: ['sectors', 'parent'] - }); - - if (!organization) { - return false; // Return false if the organization is not found - } - - const isOrganizationExecutive = async ( - org: DL_Organization - ): Promise => { - // Check if the current organization has the EXECUTIVE sector - if (org.sectors.some((sector) => sector.acronym === 'EXECUTIVE')) { - return true; - } - // If there is a parent organization, check it recursively - if (org.parent) { - const parentOrg = await mdl_organization_repo.findOne({ - where: { id: org.parent.id }, - relations: ['sectors'] - }); - return parentOrg ? await isOrganizationExecutive(parentOrg) : false; - } - return false; - }; - - // Check if the organization or its parents are executive - return await isOrganizationExecutive(organization); -}; diff --git a/backend/src/tasks/saved-search.ts b/backend/src/tasks/saved-search.ts index 8e8afca1..cc63911b 100644 --- a/backend/src/tasks/saved-search.ts +++ b/backend/src/tasks/saved-search.ts @@ -50,6 +50,21 @@ export const handler = async (commandOptions: CommandOptions) => { const hits: number = searchResults.body.hits.total.value; search.count = hits; search.save(); + + if (search.createVulnerabilities) { + const results = await fetchAllResults(filters, restrictions); + const vulnerabilities: Vulnerability[] = results.map((domain) => + plainToClass(Vulnerability, { + domain: domain, + lastSeen: new Date(Date.now()), + ...search.vulnerabilityTemplate, + state: 'open', + source: 'saved-search', + needsPopulation: false + }) + ); + await saveVulnerabilitiesToDb(vulnerabilities, false); + } } console.log(`Saved search finished for ${savedSearches.length} searches`); diff --git a/backend/src/tasks/search-sync-domains.ts b/backend/src/tasks/search-sync-domains.ts index 849ac4fa..ac9337a6 100644 --- a/backend/src/tasks/search-sync-domains.ts +++ b/backend/src/tasks/search-sync-domains.ts @@ -40,10 +40,6 @@ export const handler = async (commandOptions: CommandOptions) => { qs.where('organization.id=:org', { org: organizationId }); } - qs.andWhere( - '(domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))' - ); - const domainIds = (await qs.getMany()).map((e) => e.id); console.log(`Got ${domainIds.length} domains.`); if (domainIds.length) { diff --git a/backend/src/tasks/syncdb.ts b/backend/src/tasks/syncdb.ts index 1f966255..8419dd1b 100644 --- a/backend/src/tasks/syncdb.ts +++ b/backend/src/tasks/syncdb.ts @@ -98,7 +98,6 @@ export const handler: Handler = async (event) => { name: Sentencer.make('{{ adjective }}-{{ noun }}.crossfeed.local'), ip: ['127', randomNum(), randomNum(), randomNum()].join('.'), // Create random loopback addresses fromRootDomain: 'crossfeed.local', - isFceb: true, subdomainSource: 'findomain', organization }).save(); diff --git a/backend/src/tasks/test/__snapshots__/censysCertificates.test.ts.snap b/backend/src/tasks/test/__snapshots__/censysCertificates.test.ts.snap index 6ecb9999..b5930df0 100644 --- a/backend/src/tasks/test/__snapshots__/censysCertificates.test.ts.snap +++ b/backend/src/tasks/test/__snapshots__/censysCertificates.test.ts.snap @@ -8,12 +8,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.60", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain1", "organization": null, "reverseName": "first_file_testdomain1", @@ -31,12 +29,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "52.74.149.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain10", "organization": null, "reverseName": "first_file_testdomain10", @@ -54,12 +50,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain11", "organization": null, "reverseName": "first_file_testdomain11", @@ -77,12 +71,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "1.1.1.1", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain12", "organization": null, "reverseName": "first_file_testdomain12", @@ -555,12 +547,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.61", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain3.gov", "organization": null, "reverseName": "gov.first_file_testdomain3", @@ -591,12 +581,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "45.79.207.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain5", "organization": null, "reverseName": "first_file_testdomain5", @@ -614,12 +602,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "156.249.159.119", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain6", "organization": null, "reverseName": "first_file_testdomain6", @@ -637,12 +623,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "221.10.15.220", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain7", "organization": null, "reverseName": "first_file_testdomain7", @@ -660,12 +644,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "81.141.166.145", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain8", "organization": null, "reverseName": "first_file_testdomain8", @@ -683,12 +665,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "24.65.82.187", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain9", "organization": null, "reverseName": "first_file_testdomain9", @@ -1121,12 +1101,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "subdomain.first_file_testdomain2.gov", "organization": null, "reverseName": "gov.first_file_testdomain2.subdomain", @@ -1579,12 +1557,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "85.24.146.152", "ipOnly": false, - "isFceb": true, "name": "subdomain.first_file_testdomain4.gov", "organization": null, "reverseName": "gov.first_file_testdomain4.subdomain", @@ -1621,12 +1597,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.60", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain1", "organization": null, "reverseName": "first_file_testdomain1", @@ -1644,12 +1618,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "52.74.149.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain10", "organization": null, "reverseName": "first_file_testdomain10", @@ -1667,12 +1639,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain11", "organization": null, "reverseName": "first_file_testdomain11", @@ -1690,12 +1660,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "1.1.1.1", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain12", "organization": null, "reverseName": "first_file_testdomain12", @@ -1713,12 +1681,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.61", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain3.gov", "organization": null, "reverseName": "gov.first_file_testdomain3", @@ -1736,12 +1702,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "45.79.207.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain5", "organization": null, "reverseName": "first_file_testdomain5", @@ -1759,12 +1723,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "156.249.159.119", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain6", "organization": null, "reverseName": "first_file_testdomain6", @@ -1782,12 +1744,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "221.10.15.220", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain7", "organization": null, "reverseName": "first_file_testdomain7", @@ -1805,12 +1765,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "81.141.166.145", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain8", "organization": null, "reverseName": "first_file_testdomain8", @@ -1828,12 +1786,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "24.65.82.187", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain9", "organization": null, "reverseName": "first_file_testdomain9", @@ -1851,12 +1807,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "subdomain.first_file_testdomain2.gov", "organization": null, "reverseName": "gov.first_file_testdomain2.subdomain", @@ -1874,12 +1828,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "85.24.146.152", "ipOnly": false, - "isFceb": true, "name": "subdomain.first_file_testdomain4.gov", "organization": null, "reverseName": "gov.first_file_testdomain4.subdomain", @@ -1902,12 +1854,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.60", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain1", "organization": null, "reverseName": "first_file_testdomain1", @@ -1925,12 +1875,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "52.74.149.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain10", "organization": null, "reverseName": "first_file_testdomain10", @@ -1948,12 +1896,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain11", "organization": null, "reverseName": "first_file_testdomain11", @@ -1971,12 +1917,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "1.1.1.1", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain12", "organization": null, "reverseName": "first_file_testdomain12", @@ -1994,12 +1938,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.61", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain3.gov", "organization": null, "reverseName": "gov.first_file_testdomain3", @@ -2017,12 +1959,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "45.79.207.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain5", "organization": null, "reverseName": "first_file_testdomain5", @@ -2040,12 +1980,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "156.249.159.119", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain6", "organization": null, "reverseName": "first_file_testdomain6", @@ -2063,12 +2001,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "221.10.15.220", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain7", "organization": null, "reverseName": "first_file_testdomain7", @@ -2086,12 +2022,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "81.141.166.145", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain8", "organization": null, "reverseName": "first_file_testdomain8", @@ -2109,12 +2043,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "24.65.82.187", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain9", "organization": null, "reverseName": "first_file_testdomain9", @@ -2132,12 +2064,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "subdomain.first_file_testdomain2.gov", "organization": null, "reverseName": "gov.first_file_testdomain2.subdomain", @@ -2155,12 +2085,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "85.24.146.152", "ipOnly": false, - "isFceb": true, "name": "subdomain.first_file_testdomain4.gov", "organization": null, "reverseName": "gov.first_file_testdomain4.subdomain", diff --git a/backend/src/tasks/test/__snapshots__/censysIpv4.test.ts.snap b/backend/src/tasks/test/__snapshots__/censysIpv4.test.ts.snap index dd3d6f31..418b24d8 100644 --- a/backend/src/tasks/test/__snapshots__/censysIpv4.test.ts.snap +++ b/backend/src/tasks/test/__snapshots__/censysIpv4.test.ts.snap @@ -8,12 +8,10 @@ Array [ "cloudHosted": false, "country": "JP", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.60", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain1", "organization": null, "reverseName": "first_file_testdomain1", @@ -1263,12 +1261,10 @@ on this server.

"cloudHosted": false, "country": "SG", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "52.74.149.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain10", "organization": null, "reverseName": "first_file_testdomain10", @@ -3989,12 +3985,10 @@ on this server.

"cloudHosted": false, "country": "RU", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain11", "organization": null, "reverseName": "first_file_testdomain11", @@ -4089,12 +4083,10 @@ on this server.

"cloudHosted": false, "country": "TR", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "1.1.1.1", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain12", "organization": null, "reverseName": "first_file_testdomain12", @@ -4174,12 +4166,10 @@ on this server.

"cloudHosted": false, "country": "RU", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain2", "organization": null, "reverseName": "first_file_testdomain2", @@ -4274,12 +4264,10 @@ on this server.

"cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.61", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain3", "organization": null, "reverseName": "first_file_testdomain3", @@ -4297,12 +4285,10 @@ on this server.

"cloudHosted": false, "country": "SE", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "85.24.146.152", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain4", "organization": null, "reverseName": "first_file_testdomain4", @@ -4367,12 +4353,10 @@ on this server.

"cloudHosted": false, "country": "US", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "45.79.207.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain5", "organization": null, "reverseName": "first_file_testdomain5", @@ -4622,12 +4606,10 @@ on this server.

"cloudHosted": false, "country": "US", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "156.249.159.119", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain6", "organization": null, "reverseName": "first_file_testdomain6", @@ -4964,12 +4946,10 @@ a img { "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "221.10.15.220", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain7", "organization": null, "reverseName": "first_file_testdomain7", @@ -5032,12 +5012,10 @@ a img { "cloudHosted": false, "country": "GB", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "81.141.166.145", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain8", "organization": null, "reverseName": "first_file_testdomain8", @@ -5084,12 +5062,10 @@ a img { "cloudHosted": false, "country": "CA", "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "24.65.82.187", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain9", "organization": null, "reverseName": "first_file_testdomain9", @@ -5147,12 +5123,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.60", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain1", "organization": null, "reverseName": "first_file_testdomain1", @@ -5170,12 +5144,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "52.74.149.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain10", "organization": null, "reverseName": "first_file_testdomain10", @@ -5193,12 +5165,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain11", "organization": null, "reverseName": "first_file_testdomain11", @@ -5216,12 +5186,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "1.1.1.1", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain12", "organization": null, "reverseName": "first_file_testdomain12", @@ -5239,12 +5207,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain2", "organization": null, "reverseName": "first_file_testdomain2", @@ -5262,12 +5228,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.61", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain3", "organization": null, "reverseName": "first_file_testdomain3", @@ -5285,12 +5249,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "85.24.146.152", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain4", "organization": null, "reverseName": "first_file_testdomain4", @@ -5308,12 +5270,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "45.79.207.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain5", "organization": null, "reverseName": "first_file_testdomain5", @@ -5331,12 +5291,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "156.249.159.119", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain6", "organization": null, "reverseName": "first_file_testdomain6", @@ -5354,12 +5312,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "221.10.15.220", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain7", "organization": null, "reverseName": "first_file_testdomain7", @@ -5377,12 +5333,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "81.141.166.145", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain8", "organization": null, "reverseName": "first_file_testdomain8", @@ -5400,12 +5354,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "24.65.82.187", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain9", "organization": null, "reverseName": "first_file_testdomain9", @@ -5428,12 +5380,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.60", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain1", "organization": null, "reverseName": "first_file_testdomain1", @@ -5451,12 +5401,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "52.74.149.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain10", "organization": null, "reverseName": "first_file_testdomain10", @@ -5474,12 +5422,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain11", "organization": null, "reverseName": "first_file_testdomain11", @@ -5497,12 +5443,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "1.1.1.1", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain12", "organization": null, "reverseName": "first_file_testdomain12", @@ -5520,12 +5464,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain2", "organization": null, "reverseName": "first_file_testdomain2", @@ -5543,12 +5485,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "153.126.148.61", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain3", "organization": null, "reverseName": "first_file_testdomain3", @@ -5566,12 +5506,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "85.24.146.152", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain4", "organization": null, "reverseName": "first_file_testdomain4", @@ -5589,12 +5527,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "45.79.207.117", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain5", "organization": null, "reverseName": "first_file_testdomain5", @@ -5612,12 +5548,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "156.249.159.119", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain6", "organization": null, "reverseName": "first_file_testdomain6", @@ -5635,12 +5569,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "221.10.15.220", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain7", "organization": null, "reverseName": "first_file_testdomain7", @@ -5658,12 +5590,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "81.141.166.145", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain8", "organization": null, "reverseName": "first_file_testdomain8", @@ -5681,12 +5611,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "24.65.82.187", "ipOnly": false, - "isFceb": true, "name": "first_file_testdomain9", "organization": null, "reverseName": "first_file_testdomain9", diff --git a/backend/src/tasks/test/__snapshots__/lookingGlass.test.ts.snap b/backend/src/tasks/test/__snapshots__/lookingGlass.test.ts.snap index 54e021e9..1b47c381 100644 --- a/backend/src/tasks/test/__snapshots__/lookingGlass.test.ts.snap +++ b/backend/src/tasks/test/__snapshots__/lookingGlass.test.ts.snap @@ -8,12 +8,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "1.123.135.123", "ipOnly": true, - "isFceb": true, "name": null, "organization": null, "reverseName": "123.135.123.1", @@ -31,12 +29,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "100.123.100.123", "ipOnly": true, - "isFceb": true, "name": null, "organization": null, "reverseName": "123.100.123.100", @@ -54,12 +50,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": false, "fromRootDomain": null, "id": null, "ip": "123.123.123.123", "ipOnly": true, - "isFceb": true, "name": null, "organization": null, "reverseName": "123.123.123.123", diff --git a/backend/src/tasks/test/__snapshots__/search-sync.test.ts.snap b/backend/src/tasks/test/__snapshots__/search-sync.test.ts.snap index 8a12e364..50e7302f 100644 --- a/backend/src/tasks/test/__snapshots__/search-sync.test.ts.snap +++ b/backend/src/tasks/test/__snapshots__/search-sync.test.ts.snap @@ -16,8 +16,6 @@ Array [ "country", "asn", "cloudHosted", - "fromCidr", - "isFceb", "ssl", "censysCertificatesResults", "trustymailResults", diff --git a/backend/src/tasks/test/__snapshots__/shodan.test.ts.snap b/backend/src/tasks/test/__snapshots__/shodan.test.ts.snap index 89bdf564..54a71a2d 100644 --- a/backend/src/tasks/test/__snapshots__/shodan.test.ts.snap +++ b/backend/src/tasks/test/__snapshots__/shodan.test.ts.snap @@ -8,12 +8,10 @@ Array [ "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": null, "fromRootDomain": null, "id": null, "ip": "153.126.148.60", "ipOnly": false, - "isFceb": null, "name": null, "organization": null, "reverseName": "first_file_testdomain1", @@ -75,12 +73,10 @@ A003 BAD Error in IMAP command received by server. "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": null, "fromRootDomain": null, "id": null, "ip": "1.1.1.1", "ipOnly": false, - "isFceb": null, "name": null, "organization": null, "reverseName": "first_file_testdomain12", @@ -117,12 +113,10 @@ Resolver ID: AMS", "cloudHosted": false, "country": null, "createdAt": null, - "fromCidr": null, "fromRootDomain": null, "id": null, "ip": "31.134.10.156", "ipOnly": false, - "isFceb": null, "name": null, "organization": null, "reverseName": "first_file_testdomain2", diff --git a/backend/src/tasks/test/censysCertificates.test.ts b/backend/src/tasks/test/censysCertificates.test.ts index c1b3235e..d58f8f83 100644 --- a/backend/src/tasks/test/censysCertificates.test.ts +++ b/backend/src/tasks/test/censysCertificates.test.ts @@ -40,73 +40,61 @@ describe('censys certificates', () => { await Domain.create({ name: 'first_file_testdomain1', ip: '153.126.148.60', - isFceb: true, organization }).save(); await Domain.create({ name: 'subdomain.first_file_testdomain2.gov', ip: '31.134.10.156', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain3.gov', ip: '153.126.148.61', - isFceb: true, organization }).save(); await Domain.create({ name: 'subdomain.first_file_testdomain4.gov', ip: '85.24.146.152', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain5', ip: '45.79.207.117', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain6', ip: '156.249.159.119', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain7', ip: '221.10.15.220', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain8', ip: '81.141.166.145', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain9', ip: '24.65.82.187', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain10', ip: '52.74.149.117', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain11', ip: '31.134.10.156', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain12', ip: '1.1.1.1', - isFceb: true, organization }).save(); }); diff --git a/backend/src/tasks/test/censysIpv4.test.ts b/backend/src/tasks/test/censysIpv4.test.ts index 22d00f7d..4fc5ba44 100644 --- a/backend/src/tasks/test/censysIpv4.test.ts +++ b/backend/src/tasks/test/censysIpv4.test.ts @@ -40,73 +40,61 @@ describe('censys ipv4', () => { await Domain.create({ name: 'first_file_testdomain1', ip: '153.126.148.60', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain2', ip: '31.134.10.156', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain3', ip: '153.126.148.61', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain4', ip: '85.24.146.152', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain5', ip: '45.79.207.117', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain6', ip: '156.249.159.119', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain7', ip: '221.10.15.220', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain8', ip: '81.141.166.145', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain9', ip: '24.65.82.187', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain10', ip: '52.74.149.117', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain11', ip: '31.134.10.156', - isFceb: true, organization }).save(); await Domain.create({ name: 'first_file_testdomain12', ip: '1.1.1.1', - isFceb: true, organization }).save(); }); diff --git a/backend/src/tasks/test/cve.test.ts b/backend/src/tasks/test/cve.test.ts index b1f47aa1..5983dc88 100644 --- a/backend/src/tasks/test/cve.test.ts +++ b/backend/src/tasks/test/cve.test.ts @@ -91,7 +91,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const service = await Service.create({ @@ -140,7 +139,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const service = await Service.create({ @@ -181,7 +179,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const service = await Service.create({ @@ -226,7 +223,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const service = await Service.create({ @@ -271,7 +267,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const service = await Service.create({ @@ -310,7 +305,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const service = await Service.create({ @@ -376,7 +370,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const service = await Service.create({ @@ -404,7 +397,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); await cve({ @@ -429,7 +421,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -464,7 +455,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -497,7 +487,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -532,7 +521,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -567,7 +555,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -607,7 +594,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -640,7 +626,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization, ssl: { valid: false @@ -648,7 +633,6 @@ describe('cve', () => { }).save(); const domain2 = await Domain.create({ name: name + '-2', - isFceb: true, organization, ssl: { valid: true @@ -690,7 +674,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization, ssl: { validTo: new Date(Date.now()).toISOString() @@ -698,7 +681,6 @@ describe('cve', () => { }).save(); const domain2 = await Domain.create({ name: name + '-2', - isFceb: true, organization, ssl: { validTo: '9999-08-23T03:36:57.231Z' @@ -776,7 +758,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); let vulnerability = await Vulnerability.create({ @@ -832,7 +813,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); let vulnerability = await Vulnerability.create({ @@ -874,7 +854,6 @@ describe('cve', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); let vulnerability = await Vulnerability.create({ @@ -916,7 +895,6 @@ describe('cve', () => { // const name = 'test-' + Math.random(); // const domain = await Domain.create({ // name, - // isFceb: true, // organization // }).save(); // await Webpage.create({ diff --git a/backend/src/tasks/test/dnstwist.test.ts b/backend/src/tasks/test/dnstwist.test.ts index 7ffb88e3..58e4d220 100644 --- a/backend/src/tasks/test/dnstwist.test.ts +++ b/backend/src/tasks/test/dnstwist.test.ts @@ -83,7 +83,6 @@ describe('dnstwist', () => { const domain = await Domain.create({ name, ip: '0.0.0.0', - isFceb: true, organization }).save(); const vulns = await Vulnerability.find({ @@ -97,7 +96,6 @@ describe('dnstwist', () => { const domain = await Domain.create({ name, ip: '0.0.0.0', - isFceb: true, organization }).save(); @@ -147,7 +145,6 @@ describe('dnstwist', () => { const root_domain = await Domain.create({ name, ip: '0.0.0.0', - isFceb: true, organization }).save(); @@ -155,7 +152,6 @@ describe('dnstwist', () => { const sub_domain = await Domain.create({ name: sub_name, ip: '10.20.30.40', - isFceb: true, organization }).save(); @@ -202,7 +198,6 @@ describe('dnstwist', () => { const domain = await Domain.create({ name, ip: '0.0.0.0', - isFceb: true, organization }).save(); //represents the result of an older dnstwist run @@ -266,7 +261,6 @@ describe('dnstwist', () => { const domain = await Domain.create({ name, ip: '0.0.0.0', - isFceb: true, organization }).save(); await Vulnerability.create({ diff --git a/backend/src/tasks/test/es-client.test.ts b/backend/src/tasks/test/es-client.test.ts index daf3edb3..7fc9b79b 100644 --- a/backend/src/tasks/test/es-client.test.ts +++ b/backend/src/tasks/test/es-client.test.ts @@ -19,7 +19,6 @@ beforeAll(async () => { connection = await connectToDatabase(); domain = Domain.create({ name: 'first_file_testdomain5', - isFceb: true, ip: '45.79.207.117' }); webpage = Webpage.create({ diff --git a/backend/src/tasks/test/findomain.test.ts b/backend/src/tasks/test/findomain.test.ts index aa83b049..3c2757d0 100644 --- a/backend/src/tasks/test/findomain.test.ts +++ b/backend/src/tasks/test/findomain.test.ts @@ -75,7 +75,6 @@ describe('findomain', () => { const domain = await Domain.create({ organization, name: 'filedrop.cisa.gov', - isFceb: true, discoveredBy: scanOld, fromRootDomain: 'oldrootdomain.cisa.gov' }).save(); diff --git a/backend/src/tasks/test/helpers/getAllDomains.test.ts b/backend/src/tasks/test/helpers/getAllDomains.test.ts index 71a38a6f..b72a1807 100644 --- a/backend/src/tasks/test/helpers/getAllDomains.test.ts +++ b/backend/src/tasks/test/helpers/getAllDomains.test.ts @@ -21,7 +21,6 @@ describe('getAllDomains', () => { const domain = await Domain.create({ name, ip, - isFceb: true, organization }).save(); const domains = await getAllDomains(); @@ -58,7 +57,6 @@ describe('getAllDomains', () => { const domain = await Domain.create({ name, ip, - isFceb: true, organization: organization }).save(); const name2 = Math.random() + ''; @@ -66,7 +64,6 @@ describe('getAllDomains', () => { const domain2 = await Domain.create({ name: name2, ip: ip2, - isFceb: true, organization: organization2 }).save(); const name3 = Math.random() + ''; @@ -74,7 +71,6 @@ describe('getAllDomains', () => { const domain3 = await Domain.create({ name: name3, ip: ip3, - isFceb: true, organization: organization3 }).save(); const domains = await getAllDomains([organization.id, organization2.id]); diff --git a/backend/src/tasks/test/intrigue-ident.test.ts b/backend/src/tasks/test/intrigue-ident.test.ts index b159a5a3..db9dcbef 100644 --- a/backend/src/tasks/test/intrigue-ident.test.ts +++ b/backend/src/tasks/test/intrigue-ident.test.ts @@ -261,7 +261,6 @@ describe('intrigue ident', () => { const domain = await Domain.create({ organization, name: 'www.cisa.gov', - isFceb: true, ip: '0.0.0.0' }).save(); let service = await Service.create({ diff --git a/backend/src/tasks/test/lookingGlass.test.ts b/backend/src/tasks/test/lookingGlass.test.ts index 1404a970..95e69b15 100644 --- a/backend/src/tasks/test/lookingGlass.test.ts +++ b/backend/src/tasks/test/lookingGlass.test.ts @@ -422,19 +422,16 @@ describe('lookingGlass', () => { await Domain.create({ name: '1.123.135.123', ip: '1.123.135.123', - isFceb: true, organization }).save(), await Domain.create({ name: '100.123.100.123', ip: '100.123.100.123', - isFceb: true, organization }).save(), await Domain.create({ name: '123.123.123.123', ip: '123.123.123.123', - isFceb: true, organization }).save() ]; diff --git a/backend/src/tasks/test/search-sync.test.ts b/backend/src/tasks/test/search-sync.test.ts index 5e2a9793..47b4dbca 100644 --- a/backend/src/tasks/test/search-sync.test.ts +++ b/backend/src/tasks/test/search-sync.test.ts @@ -35,7 +35,6 @@ describe('search_sync', () => { const domain = await Domain.create({ name: 'cisa.gov', organization, - isFceb: true, syncedAt: new Date('9999-10-10') }).save(); @@ -60,7 +59,6 @@ describe('search_sync', () => { const domain = await Domain.create({ name: 'cisa.gov', organization, - isFceb: true, syncedAt: new Date('9999-10-10') }).save(); @@ -86,7 +84,6 @@ describe('search_sync', () => { const domain = await Domain.create({ name: 'cisa.gov', organization, - isFceb: true, syncedAt: new Date('9999-10-10') }).save(); @@ -112,7 +109,6 @@ describe('search_sync', () => { const domain = await Domain.create({ name: 'cisa.gov', organization, - isFceb: true, syncedAt: new Date('9999-10-10') }).save(); @@ -137,7 +133,6 @@ describe('search_sync', () => { const domain = await Domain.create({ name: 'cisa.gov', organization, - isFceb: true, syncedAt: new Date('9999-10-10') }).save(); @@ -170,7 +165,6 @@ describe('search_sync', () => { const domain = await Domain.create({ name: 'cisa.gov', organization, - isFceb: true, syncedAt: new Date('9999-10-10') }).save(); @@ -197,7 +191,6 @@ describe('search_sync', () => { const domain = await Domain.create({ name: 'cisa.gov', organization, - isFceb: true, syncedAt: new Date('9999-10-10') }).save(); @@ -216,7 +209,6 @@ describe('search_sync', () => { const domain = await Domain.create({ name: 'cisa.gov', organization, - isFceb: true, syncedAt: new Date('9999-09-19T19:57:32.346Z'), updatedAt: new Date('9999-09-20T19:57:32.346Z') }).save(); @@ -265,7 +257,6 @@ describe('search_sync', () => { Domain.create({ name: 'cisa-' + Math.random() + '.gov', organization, - isFceb: true, syncedAt: new Date('9999-09-19T19:57:32.346Z'), updatedAt: new Date('9999-09-20T19:57:32.346Z') }).save() diff --git a/backend/src/tasks/test/shodan.test.ts b/backend/src/tasks/test/shodan.test.ts index 9c6ab642..aca7d033 100644 --- a/backend/src/tasks/test/shodan.test.ts +++ b/backend/src/tasks/test/shodan.test.ts @@ -161,19 +161,16 @@ describe('shodan', () => { await Domain.create({ name: 'first_file_testdomain1', ip: '153.126.148.60', - isFceb: true, organization }).save(), await Domain.create({ name: 'first_file_testdomain2', ip: '31.134.10.156', - isFceb: true, organization }).save(), await Domain.create({ name: 'first_file_testdomain12', ip: '1.1.1.1', - isFceb: true, organization }).save() ]; @@ -210,9 +207,7 @@ describe('shodan', () => { updatedAt: null, createdAt: null, syncedAt: null, - name: null, - isFceb: null, - fromCidr: null + name: null })) ).toMatchSnapshot(); expect(domains.filter((e) => !e.organization).length).toEqual(0); diff --git a/backend/src/tasks/test/sslyze.test.ts b/backend/src/tasks/test/sslyze.test.ts index 4fccb2b8..5ef1ea65 100644 --- a/backend/src/tasks/test/sslyze.test.ts +++ b/backend/src/tasks/test/sslyze.test.ts @@ -37,7 +37,6 @@ describe('sslyze', () => { let domain = await Domain.create({ organization, name: 'www.cisa.gov', - isFceb: true, ip: '0.0.0.0' }).save(); const service = await Service.create({ @@ -75,7 +74,6 @@ describe('sslyze', () => { let domain = await Domain.create({ organization, name: 'www.cisa.gov', - isFceb: true, ip: '0.0.0.0' }).save(); const service = await Service.create({ diff --git a/backend/src/tools/logger.ts b/backend/src/tools/logger.ts deleted file mode 100644 index b8c58738..00000000 --- a/backend/src/tools/logger.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { Request } from 'express'; -import * as jwt from 'jsonwebtoken'; -import { ApiKey, User } from '../models'; -import { Log } from '../models/log'; -import { getRepository, Repository } from 'typeorm'; -import { UserToken } from 'src/api/auth'; -import { createHash } from 'crypto'; - -type AccessTokenPayload = { - id: string; -}; - -type LoggerUserState = { - data: User | undefined; - ready: boolean; - attempts: number; -}; - -type RecordPayload = object & { - timestamp: Date; -}; - -export type RecordMessage = - | (( - request: Request, - token: AccessTokenPayload | undefined, - responseBody?: object - ) => Promise) - | RecordPayload; - -export class Logger { - private request: Request; - private logId: string; - private token: AccessTokenPayload | undefined; - private user: LoggerUserState = { - data: undefined, - ready: false, - attempts: 0 - }; - private logRep: Repository; - - async record( - action: string, - result: 'success' | 'fail', - messageOrCB: RecordMessage | undefined, - responseBody?: object | string - ) { - try { - if (!this.logRep) { - const logRepository = getRepository(Log); - this.logRep = logRepository; - } - - if (!this.token) { - await this.parseToken(); - } - - const parsedResponseBody = - typeof responseBody === 'string' && - responseBody !== 'User registration approved.' - ? JSON.parse(responseBody) - : responseBody; - - const payload = - typeof messageOrCB === 'function' - ? await messageOrCB(this.request, this.token, parsedResponseBody) - : messageOrCB; - - const logRecord = await this.logRep.create({ - payload: payload as object, - createdAt: payload?.timestamp, - result: result, - eventType: action - }); - - logRecord.save(); - } catch (error) { - console.warn('Error occured in loggingMiddleware', error); - } - } - - async parseToken() { - const authorizationHeader = this.request.headers.authorization; - - if (!authorizationHeader) { - throw 'Missing token/api key'; - } - - if (/^[A-Fa-f0-9]{32}$/.test(authorizationHeader)) { - // API Key Logic - const hashedKey = createHash('sha256') - .update(authorizationHeader) - .digest('hex'); - const apiKey = await ApiKey.findOne( - { hashedKey }, - { relations: ['user'] } - ); - - if (!apiKey) { - throw 'Invalid API key'; - } - - // Update last used and assign token - apiKey.lastUsed = new Date(); - await apiKey.save(); - - this.token = { id: apiKey.user.id }; - } else { - // JWT Logic - try { - const parsedUserFromJwt = jwt.verify( - authorizationHeader, - process.env.JWT_SECRET! - ) as UserToken; - this.token = { id: parsedUserFromJwt.id }; - } catch (err) { - throw 'Invalid JWT token'; - } - } - } - - // Constructor takes a request and sets it to a class variable - constructor(req: Request) { - this.request = req; - } -} - -// Database Tables diff --git a/backend/src/worker.ts b/backend/src/worker.ts index 3651fefe..94f0ca6e 100644 --- a/backend/src/worker.ts +++ b/backend/src/worker.ts @@ -24,7 +24,6 @@ import { handler as trustymail } from './tasks/trustymail'; import { handler as vulnSync } from './tasks/vuln-sync'; import { handler as vulnScanningSync } from './tasks/vs_sync'; import { handler as xpanseSync } from './tasks/xpanse-sync'; -import { handler as flagFloatingIps } from './tasks/flagFloatingIps'; import { SCAN_SCHEMA } from './api/scans'; /** @@ -48,7 +47,6 @@ async function main() { dnstwist, dotgov, findomain, - flagFloatingIps, intrigueIdent, lookingGlass, portscanner, diff --git a/backend/test/__snapshots__/saved-searches.test.ts.snap b/backend/test/__snapshots__/saved-searches.test.ts.snap index 94c479ac..efd26fcd 100644 --- a/backend/test/__snapshots__/saved-searches.test.ts.snap +++ b/backend/test/__snapshots__/saved-searches.test.ts.snap @@ -3,6 +3,7 @@ exports[`saved-search create create by user should succeed 1`] = ` Object { "count": 3, + "createVulnerabilities": false, "createdAt": Any, "createdBy": Object { "id": Any, @@ -15,5 +16,6 @@ Object { "sortDirection": "", "sortField": "", "updatedAt": Any, + "vulnerabilityTemplate": Object {}, } `; diff --git a/backend/test/__snapshots__/stats.test.ts.snap b/backend/test/__snapshots__/stats.test.ts.snap index f1fe7196..e825ef0c 100644 --- a/backend/test/__snapshots__/stats.test.ts.snap +++ b/backend/test/__snapshots__/stats.test.ts.snap @@ -50,12 +50,10 @@ Object { "cloudHosted": false, "country": null, "createdAt": Any, - "fromCidr": false, "fromRootDomain": null, "id": Any, "ip": null, "ipOnly": false, - "isFceb": true, "name": Any, "reverseName": Any, "screenshot": null, @@ -150,12 +148,10 @@ Object { "cloudHosted": false, "country": null, "createdAt": Any, - "fromCidr": false, "fromRootDomain": null, "id": Any, "ip": null, "ipOnly": false, - "isFceb": true, "name": Any, "reverseName": Any, "screenshot": null, diff --git a/backend/test/domains.test.ts b/backend/test/domains.test.ts index 2c11ea6f..df029169 100644 --- a/backend/test/domains.test.ts +++ b/backend/test/domains.test.ts @@ -41,7 +41,6 @@ describe('domains', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); await Service.create({ @@ -63,7 +62,6 @@ describe('domains', () => { }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -92,12 +90,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); await Domain.create({ name, - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -121,12 +117,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); await Domain.create({ name, - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -153,12 +147,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -186,12 +178,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -223,12 +213,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -250,12 +238,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); await Domain.create({ name: name + '-1', - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization }).save(); const response = await request(app) @@ -278,12 +264,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); await Domain.create({ name: name + '-1', - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization }).save(); const response = await request(app) @@ -307,12 +291,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); await Domain.create({ name, - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -335,12 +317,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); await Domain.create({ name, - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -364,12 +344,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); await Domain.create({ name, - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -393,12 +371,10 @@ describe('domains', () => { const name = 'test-' + Math.random(); await Domain.create({ name, - isFceb: true, organization }).save(); await Domain.create({ name: name + '-2', - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -422,7 +398,6 @@ describe('domains', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); const webpage = await Webpage.create({ @@ -446,7 +421,6 @@ describe('domains', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -464,7 +438,6 @@ describe('domains', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization: organization2 }).save(); const response = await request(app) diff --git a/backend/test/saved-searches.test.ts b/backend/test/saved-searches.test.ts index fc87a624..2158122a 100644 --- a/backend/test/saved-searches.test.ts +++ b/backend/test/saved-searches.test.ts @@ -45,7 +45,9 @@ describe('saved-search', () => { sortField: '', searchTerm: '', searchPath: '', - filters: [] + filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {} }) .expect(200); expect(response.body).toMatchSnapshot({ @@ -69,11 +71,14 @@ describe('saved-search', () => { sortField: '', searchTerm: '', searchPath: '', - filters: [] + filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {} }; const search = await SavedSearch.create(body).save(); body.name = 'test-' + Math.random(); body.searchTerm = '123'; + body.createVulnerabilities = true; const response = await request(app) .put(`/saved-searches/${search.id}`) .set( @@ -99,7 +104,9 @@ describe('saved-search', () => { sortField: '', searchTerm: '', searchPath: '', - filters: [] + filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {} }; const search = await SavedSearch.create({ ...body, @@ -107,6 +114,7 @@ describe('saved-search', () => { }).save(); body.name = 'test-' + Math.random(); body.searchTerm = '123'; + body.createVulnerabilities = true; const response = await request(app) .put(`/saved-searches/${search.id}`) .set( @@ -120,6 +128,9 @@ describe('saved-search', () => { .expect(200); expect(response.body.name).toEqual(body.name); expect(response.body.searchTerm).toEqual(body.searchTerm); + expect(response.body.createVulnerabilities).toEqual( + body.createVulnerabilities + ); }); it('update by standard user without access should fail', async () => { const user = await User.create({ @@ -142,6 +153,8 @@ describe('saved-search', () => { searchTerm: '', searchPath: '', filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {}, createdBy: user }; const search = await SavedSearch.create(body).save(); @@ -166,7 +179,9 @@ describe('saved-search', () => { sortField: '', searchTerm: '', searchPath: '', - filters: [] + filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {} }; const search = await SavedSearch.create(body).save(); const response = await request(app) @@ -191,7 +206,9 @@ describe('saved-search', () => { sortField: '', searchTerm: '', searchPath: '', - filters: [] + filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {} }).save(); const response = await request(app) .delete(`/saved-searches/${search.id}`) @@ -218,6 +235,8 @@ describe('saved-search', () => { searchTerm: '', searchPath: '', filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {}, createdBy: user }).save(); const response = await request(app) @@ -253,6 +272,8 @@ describe('saved-search', () => { searchTerm: '', searchPath: '', filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {}, createdBy: user }).save(); const response = await request(app) @@ -288,6 +309,8 @@ describe('saved-search', () => { searchTerm: '', searchPath: '', filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {}, createdBy: user }).save(); const response = await request(app) @@ -312,7 +335,9 @@ describe('saved-search', () => { sortField: '', searchTerm: '', searchPath: '', - filters: [] + filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {} }).save(); const response = await request(app) .get(`/saved-searches`) @@ -346,6 +371,8 @@ describe('saved-search', () => { searchTerm: '', searchPath: '', filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {}, createdBy: user }).save(); // this org should not show up in the response @@ -357,6 +384,8 @@ describe('saved-search', () => { searchTerm: '', searchPath: '', filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {}, createdBy: user1 }).save(); const response = await request(app) @@ -382,7 +411,9 @@ describe('saved-search', () => { sortField: '', searchTerm: '', searchPath: '', - filters: [] + filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {} }).save(); const response = await request(app) .get(`/saved-searches/${search.id}`) @@ -409,6 +440,8 @@ describe('saved-search', () => { searchTerm: '', searchPath: '', filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {}, createdBy: user }).save(); const response = await request(app) @@ -444,6 +477,8 @@ describe('saved-search', () => { searchTerm: '', searchPath: '', filters: [], + createVulnerabilities: false, + vulnerabilityTemplate: {}, createdBy: user1 }).save(); const response = await request(app) diff --git a/backend/test/stats.test.ts b/backend/test/stats.test.ts index 41b7160b..5a5588f6 100644 --- a/backend/test/stats.test.ts +++ b/backend/test/stats.test.ts @@ -69,7 +69,6 @@ describe('stats', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); await Vulnerability.create({ @@ -85,7 +84,6 @@ describe('stats', () => { }).save(); await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization: organization2 }).save(); const response = await request(app) @@ -128,7 +126,6 @@ describe('stats', () => { const name = 'test-' + Math.random(); const domain = await Domain.create({ name, - isFceb: true, organization }).save(); await Vulnerability.create({ @@ -144,7 +141,6 @@ describe('stats', () => { }).save(); const domain2 = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization: organization2 }).save(); await Vulnerability.create({ diff --git a/backend/test/vulnerabilities.test.ts b/backend/test/vulnerabilities.test.ts index a5b5d324..e402a944 100644 --- a/backend/test/vulnerabilities.test.ts +++ b/backend/test/vulnerabilities.test.ts @@ -37,7 +37,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -46,7 +45,6 @@ describe('vulnerabilities', () => { }).save(); const domain2 = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization: organization2 }).save(); const vulnerability2 = await Vulnerability.create({ @@ -85,7 +83,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -94,7 +91,6 @@ describe('vulnerabilities', () => { }).save(); const domain2 = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization: organization2 }).save(); const vulnerability2 = await Vulnerability.create({ @@ -129,7 +125,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const title = 'test-' + Math.random(); @@ -139,7 +134,6 @@ describe('vulnerabilities', () => { }).save(); const domain2 = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization: organization2 }).save(); const vulnerability2 = await Vulnerability.create({ @@ -175,7 +169,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const title = 'test-' + Math.random(); @@ -185,7 +178,6 @@ describe('vulnerabilities', () => { }).save(); const domain2 = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization: organization2 }).save(); const vulnerability2 = await Vulnerability.create({ @@ -226,7 +218,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const title = 'test-' + Math.random(); @@ -236,7 +227,6 @@ describe('vulnerabilities', () => { }).save(); const domain2 = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization: organization2 }).save(); const vulnerability2 = await Vulnerability.create({ @@ -271,7 +261,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const title = 'test-' + Math.random(); @@ -324,7 +313,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -359,7 +347,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -394,12 +381,10 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const domain2 = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -464,7 +449,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -501,7 +485,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization: organization2 }).save(); const vulnerability = await Vulnerability.create({ @@ -528,7 +511,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization: organization }).save(); const vulnerability = await Vulnerability.create({ @@ -558,7 +540,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -595,7 +576,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ @@ -638,7 +618,6 @@ describe('vulnerabilities', () => { }).save(); const domain = await Domain.create({ name: 'test-' + Math.random(), - isFceb: true, organization }).save(); const vulnerability = await Vulnerability.create({ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fcdb8622..b0c86e01 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,7 +9,6 @@ "@elastic/react-search-ui-views": "^1.20.2", "@emotion/react": "^11.11.1", "@emotion/styled": "^11.11.0", - "@fontsource/roboto": "^5.1.0", "@jonkoops/matomo-tracker-react": "^0.7.0", "@mui/icons-material": "^5.14.1", "@mui/lab": "^5.0.0-alpha.135", @@ -13282,11 +13281,6 @@ "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.1.1.tgz", "version": "0.1.1" }, - "node_modules/@fontsource/roboto": { - "integrity": "sha512-cFRRC1s6RqPygeZ8Uw/acwVHqih8Czjt6Q0MwoUoDe9U3m4dH1HmNDRBZyqlMSFwgNAUKgFImncKdmDHyKpwdg==", - "resolved": "https://registry.npmjs.org/@fontsource/roboto/-/roboto-5.1.0.tgz", - "version": "5.1.0" - }, "node_modules/@hapi/hoek": { "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==", "peer": true, diff --git a/frontend/package.json b/frontend/package.json index 3c14325a..4bcea01f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,7 +17,6 @@ "@elastic/react-search-ui-views": "^1.20.2", "@emotion/react": "^11.11.1", "@emotion/styled": "^11.11.0", - "@fontsource/roboto": "^5.1.0", "@jonkoops/matomo-tracker-react": "^0.7.0", "@mui/icons-material": "^5.14.1", "@mui/lab": "^5.0.0-alpha.135", diff --git a/frontend/public/index.html b/frontend/public/index.html index fa2a5d11..3ffb6a80 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -6,6 +6,14 @@ + + CyHy Dashboard diff --git a/frontend/scripts/api.js b/frontend/scripts/api.js index 3f1332dd..57f6c411 100644 --- a/frontend/scripts/api.js +++ b/frontend/scripts/api.js @@ -65,30 +65,14 @@ app.use( }) ); -//Middleware to set Cache-Control headers -app.use((req, res, next) => { - res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate'); - next(); -}); - app.use((req, res, next) => { res.setHeader('X-XSS-Protection', '0'); next(); }); -app.use( - express.static(path.join(__dirname, '../build'), { - setHeaders: (res, path) => { - if (path.endsWith('.js')) { - res.setHeader('Content-Type', 'application/javascript'); - } - }, - maxAge: 'no-cache, no-store, must-revalidate' - }) -); +app.use(express.static(path.join(__dirname, '../build'))); app.use((req, res) => { - res.setHeader('Content-Type', 'text/html'); res.sendFile(path.join(__dirname, '../build/index.html')); }); diff --git a/frontend/scripts/docs.js b/frontend/scripts/docs.js index 4c9dc283..1858db31 100644 --- a/frontend/scripts/docs.js +++ b/frontend/scripts/docs.js @@ -38,12 +38,6 @@ app.use( }) ); -//Middleware to set Cache-Control headers -app.use((req, res, next) => { - res.setHeader('Cache-Control', 'private, max-age=3600'); - next(); -}); - app.use((req, res, next) => { res.setHeader('X-XSS-Protection', '0'); next(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 86d30e49..3b04f09f 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -3,6 +3,7 @@ import { BrowserRouter as Router, Switch, Route, + Redirect, useLocation } from 'react-router-dom'; import { API, Auth } from 'aws-amplify'; @@ -38,9 +39,9 @@ import { Authenticator } from '@aws-amplify/ui-react'; import { RSCDashboard } from 'components/ReadySetCyber/RSCDashboard'; import { RSCDetail } from 'components/ReadySetCyber/RSCDetail'; import { RSCLogin } from 'components/ReadySetCyber/RSCLogin'; +import { RSCAuthLoginCreate } from 'components/ReadySetCyber/RSCAuthLoginCreate'; import { RiskWithSearch } from 'pages/Risk/Risk'; import { StaticsContextProvider } from 'context/StaticsContextProvider'; -import { SavedSearchContextProvider } from 'context/SavedSearchContextProvider'; API.configure({ endpoints: [ @@ -93,167 +94,140 @@ const App: React.FC = () => ( - - - - - - - - - - - - - - - ( - - )} - permissions={[ - 'globalView', - 'regionalAdmin', - 'standard' - ]} - /> - - - - - - - - - - - - - - - - + + + + + } + unauth={AuthLogin} + component={RiskWithSearch} + /> + + + + + + + + + ( + + )} + permissions={['globalView', 'regionalAdmin', 'standard']} + /> + + + + + + + + + + } + unauth={RSCLogin} + component={RSCDashboard} + /> + } + unauth={RSCAuthLoginCreate} + component={RSCDashboard} + /> + } + permissions={[ + 'globalView', + 'readySetCyber', + 'regionalAdmin', + 'standard' + ]} + unauth={RSCLogin} + /> + + + + diff --git a/frontend/src/assets/nvd.jpeg b/frontend/src/assets/nvd.jpeg deleted file mode 100644 index a0f859753adea14f44b8ca7480c3ccb57b779da8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5938 zcmcgwXH-*NmktR~YICLtvQ0WbZzs!9VO0)l{K6cm?8 zKqP+zhyXwkn3nh!2`QcCtJut|Pyv}IPo12z%E%z>f-V7X%A4oe;Ic?9vz8%xA??7p z?22s$4!Pk4f-xon2N3jMJOpbXVj>_3Dd8F%3;+@Vfh3e91d+dv5QxFVv~;%wG@VEU zWspw@Y|TU4S0PZLGPZek=YaDmz-59pAQ%J&+y`vlM!$qYL6}937&fc$CU)|Y)GHr1 z0|hemNaLEQbIUt_U2nVHliiG?7;I_5JA! zhIv@ak~*2dC^9f{9rhTG9bc>S^N3uQB~0(g^tnOaW7VaTKdhh!p549|o%Qy5r>5a? z&dC3(P^JcM<9I>n?L?7n*HmU-7veLbAIJ2M|kQV68OBatW*32$`kk`qOG7#k)S-&m) ztA;4wQioB|cf|(gInUQD(o#vZ97lCZ-H@{jyi!?MMmx^c7NC&=b5ON+FI6mm+24v@ z3rhfZ3aV6;b!2Y1oB7M&e;7kdOJ|ZfST+_oaeLzDfN4wAL;YOaDm{hwI_n0lz1DE0 z%(2v@p-j)qbDV@*}qjM&=-5A*X=oe{Kv3L47w$L*_)4v?ve8uZXDDX1&y3kH{bL*J4|33evGx+^& zIl~w#4?)Kt2h}kG9hYULzklfCdQCI`y_lMN(zFA8 z+cv1nv~19A=>M{do__1`dWNb^mzWdYCB_&aCI-QfSA8>+i)b6CsKu?=a(JBss3sdJ zKhPa&+Px+grH>}!_D~;2&f{iAQ@d<#uRH0A*AAEkV^^IN>360a96Ak?BHX7-u#BAM z*_@l0Y2h#y?uT@N@c{=;u_F7yicxNN!byQ^py7vVN1hh}14-x-rIKGeX?e5ppS!kB;=XB<8Hgq_4Y(wS6b9({3xPV<`=KNzQua& z3&wN6^$xGf5}S~d!EK~Sfs#@O z6iFMeu*Rd)QtLkMUB?*t6BS{oTm36GJ>-*lzU_smmT$v;ed8o9-d+I0p=!)ZEaQK6 zcO#9}G-wyB>PBrLhTIu=OES6`FQohZ1ne6JoUu}#Qn(H$O& zJn`}uE>MEgf3a+y%OcY3gud4OeDDWIkKKDctg| zpymtt<2=2UNdIQzzLXW}TD_5_sFGxDZtLO_Q7b<$_t4;3Qc|Z*jIC-__s}iYN|=KZ{s$ZK zvu@2ts>*L!6Dc}j)qvlvjkyDN!QvzlbZ406UM z8q3ns=zw>_$&7PjfV%BzmJZ&r)JW#Z!zrU;?#PE{a(PG5hlaUM z)f`f5$36;lzj+Lw@PBJxmzaUEMtq9LvGU3pp{kCwj=>Ggr!A%U8|)=C8q07S!J~Wu z`!TDJU$yPj{rFd8hIF5kUEvQG(sjd}eOIeEASHw+UKW{3IqosHW4W{0$YBY_4E^iO zwC;(|B_Fue*k_BYJN1!|GDtPOJMoZmKP)^NKOQmum_L=);c8Aj6ZE8+2TTqbP|yM* zQ}%-e%**Cx$i`181~{|x)cak%u`(vQ!rXz(^QUyI9pkEU*yoibo|OF#4>q-rX7>G! z8zGS@4R;)AR@iPKcPhiN3bC2oDhVV-MS?>+5!KUCIJ)lFoC*8JPU(o|!CLHMSu#|1 z?FcxTP|=E3>xVZMe~EG3D?-KS6pyB}Dy}VCp`*L$Ezx!6rf4*9>5QoqHTk@%(le+T z{b0fUIA|AmUCM0~A}wOtFm6_9uq1ducjev8p7e-pbg_$kwwH4+`v6xMrbe8d?I750 zje#Y|+OQ|#UNa`;tuA6NXRDTKU_#~VXia{dra^plHF8&I7G7dcTBS!4v|WkxGJ4?} z9iP-fm6DoKkodte)<>v=n+3X9ZkV9*wnFAUmhjvyLMD3VYiinOpxYI9XP^|ry7_jc zYu=$(m&K{R$M#;`cvHAKw%*{QyuJH{ZZ@Yt!DjQZ4tK!}Y6g=?GOI=QjzJmK$uP6@ z$ZOtp>GQX9z~`eAC`H{P9XuzgTBpOxz1GY=vya@}KEFM@<5!p~yU=*Oznh|FESgu~ zR-Mq4n;2y1ZsXYE3D>V+=Y{>gB{D1d9j@%MR8KAa$)(SM$BqXxCpwj)eV`Oj$%6{f zp6v9Vz%BXH1cj`+7*++@S`Az7;|Jj`a{hQ-GGJZL%@c*gl8*u(lX=YgG38|KrlrjF zIIOk{^B*uQ~t&O#lGOFzkmr>c{33axtk$Fw>)VJc*@i=fbb zj+=zd0lj%&((Hh88ZSF^nOu0YS8l!SE2A@+wv>+5D?0-v9g zMh{Zr@ng>FPlPgt7x=`lAb)yzn$S<9u%C^dey=YiL-FBiq%JS654B9ue|@&7zi}nsFR(<#V0_B5Xk96NUzEei7IEPFe#v#|lHz~rjFFpJlJ6iq@}NZ^7k50p+jN8LUy@)J-oW@LxB(CQK@+r zUR}}@>1sT~!JSKW&-~3(r&`O@fQ2s1O1+Q&3bd?5v)PxAOt6xWbV7G;#&TaZ3djBw zrk)U^4qjSsl?#6>eq-R&ruF%YC+_@kIv#;1vwF|>Ub&5RIvLCwJl_jksH6R{>o2gg zC+}6K8PE}7qDcNAtchvD%}sTdr8K?))!HJkTeh0KCFOdV+`89bu6@y)$@nwom+k0V zt~0~fxl^e~b-z22_x5cRGsDg%QC_(dcelpuZiI0CvPW6u8od?iY#f}$bhx1u!gRtJ z;Pw9bi#vvpxNQeocfFW8##W1EA^m63+qTU=C6`m$o#Vm|t#8`?#lZ z`f$app$7Smc};TqZMSDlTQL3z``x7Xyj!^s0Xuctk| z5%jJcUr9#D@5Y2;G06rlsMhNjHN%V7$gND}?qt{Ly))$FBYa;@WqnNvkC2~%5vmsk zzU{F9T(49~)&U*x7AS1Qh5Q?_nAi9GLLa+k?{}S+6+QOeS}m?|+|%d>-8B)BP|ZaAxdnS_+qSmm#6%ONs=h7nBw^N0 zOk>Z%mCf{j{g(rNue;IU4;{yA^_^0~3 z>Kh6)@0cZ{LgZ&_$)TejMyD_Cid0+TQ>_?!)!N$UW~trw8k7-GC;*MVGH~=O;E<1| zQVyu@T|RhH_F|lnQoiaamdh6ezFEw4_$@qFP)<^k-N#8R*R@vjw;<{J0p<`{X>hSt z|{NJ2oY^TX?Cc{ZZS1)Qywh)h`&# zj++I6cf2D~%S0I@o?2zr59lJ~`JG& z$~EO-^^$Er@z=|PUL+eoUc+Ii7~PS0-ji|nytO$+G@;Eo1~d2+5_)x1U8~9E`)njhnX?$BpH+%=`$g6bu32Tq#ZTi%mZA{YT76PveQ6XVT6EQht-s8W)gA zu!adPchllP8LKAqdZzdk-eHNDW_@6-b}p9utwr%CeTP`=KB4+7gb!PbPpAl|S;y9G z?OFC&t8jP4)q2)xx&Cy|A5qTt`x3nF**pD3UFuuvNdHuFUUHQJ;j7W#Rja)Ef5_&2 zact1I=N$z)Y?AHevHWq{>z@BV1?fam$v(GZFxhTU?W?dGu^IbUvg;^J~FF3 zBRNw*;M2Cs8QH*_`M1mDu zk~pyniX}NhJb97txy`r2-TqIrNfs+xMheAQ#YxP0zWJ3q2|`u78jLOyk2&-(_I`W4 zqW8T9(!BdDF$*IdsDbfrN%Ou9pLi34nqMP4)2C~BX**d)5f=*V7rHv01cPc7E{C`@ z?~X8g-V6D-3ss$^mUi-`mO-*Q>4)c>DSRS+D^Gzv?TOt>u6%cXY6mkgo- zO^ugNP?~_S2x=Nc^Un%CQ6?>Rr+15UMHHKnH#+D<4Z>b}WhvYUSGrMvd{lHtxj-WE zy^28iTOIV*z14Go?)~O-z;JDQq>)?j5cT8R6EA_&skFSD3T`)`x>1nwQUrnsOhZWI zU@(zhd$9hVR!l0kKajI?A4 o)Ad@qKMTwiJBa2LPunQ~D=QvA#cSsVhS3rEF0zES<9zZz034%c-2eap diff --git a/frontend/src/components/DomainDetails.tsx b/frontend/src/components/DomainDetails.tsx index 66b934ef..6d7b8fc1 100644 --- a/frontend/src/components/DomainDetails.tsx +++ b/frontend/src/components/DomainDetails.tsx @@ -24,8 +24,9 @@ import { useDomainApi } from 'hooks'; import { DefinitionList } from './DefinitionList'; // @ts-ignore:next-line import { differenceInCalendarDays, parseISO } from 'date-fns'; -import { Webpage } from 'types'; +import { Webpage } from 'types/webpage'; import { useAuthContext } from 'context'; +import { Stack } from '@mui/system'; const PREFIX = 'DomainDetails'; @@ -335,15 +336,21 @@ export const DomainDetails: React.FC = (props) => { const webpageTree = generateWebpageTree(webpages); const webpageList = generateWebpageList(webpageTree); + const backToResults = () => { + history.push('/inventory'); + }; + return ( <> - - {/* */} + + +
diff --git a/frontend/src/components/DrawerInterior.tsx b/frontend/src/components/DrawerInterior.tsx index 3b43e432..1777496e 100644 --- a/frontend/src/components/DrawerInterior.tsx +++ b/frontend/src/components/DrawerInterior.tsx @@ -1,21 +1,18 @@ -import React, { useMemo } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { AccordionDetails, Accordion as MuiAccordion, AccordionSummary as MuiAccordionSummary, IconButton, + Paper, Divider, Stack, Toolbar, Typography, Box, - Button, - List, - FormControlLabel, - ListItem, - FormGroup, - Radio + Button } from '@mui/material'; +import { DataGrid } from '@mui/x-data-grid'; import { classes, StyledWrapper @@ -23,14 +20,15 @@ import { import { Delete, ExpandMore, - FiberManualRecordRounded, - FilterAlt, - Save + FiberManualRecordRounded } from '@mui/icons-material'; -import { FacetFilter, TaggedArrayInput } from 'components'; +import FilterAltIcon from '@mui/icons-material/FilterAlt'; +import { SearchBar } from 'components'; +import { TaggedArrayInput, FacetFilter } from 'components'; import { ContextType } from '../context/SearchProvider'; +import { SavedSearch } from '../types/saved-search'; import { useAuthContext } from '../context'; -import { useSavedSearchContext } from 'context/SavedSearchContext'; +import { useHistory, useLocation } from 'react-router-dom'; import { withSearch } from '@elastic/react-search-ui'; interface Props { @@ -41,7 +39,6 @@ interface Props { clearFilters: ContextType['clearFilters']; searchTerm: ContextType['searchTerm']; setSearchTerm: ContextType['setSearchTerm']; - initialFilters: any[]; } const FiltersApplied: React.FC = () => { @@ -62,18 +59,27 @@ export const DrawerInterior: React.FC = (props) => { removeFilter, facets, clearFilters, - setSearchTerm, - initialFilters + searchTerm, + setSearchTerm } = props; const { apiGet, apiDelete } = useAuthContext(); + const [savedSearches, setSavedSearches] = useState([]); + const [savedSearchCount, setSavedSearchCount] = useState(0); + const history = useHistory(); + const location = useLocation(); - const { - savedSearches, - setSavedSearches, - setSavedSearchCount, - activeSearchId, - setActiveSearchId - } = useSavedSearchContext(); + useEffect(() => { + const fetchSearches = async () => { + try { + const response = await apiGet('/saved-searches'); + setSavedSearches(response.result); + setSavedSearchCount(response.result.length); + } catch (error) { + console.error('Error fetching searches:', error); + } + }; + fetchSearches(); + }, [apiGet]); const deleteSearch = async (id: string) => { try { @@ -81,62 +87,10 @@ export const DrawerInterior: React.FC = (props) => { const updatedSearches = await apiGet('/saved-searches'); // Get current saved searches setSavedSearches(updatedSearches.result); // Update the saved searches setSavedSearchCount(updatedSearches.result.length); // Update the count - localStorage.removeItem('savedSearch'); } catch (e) { console.log(e); } }; - const displaySavedSearch = (id: string) => { - const savedSearch = savedSearches.find((search) => search.id === id); - if (savedSearch) { - setSearchTerm(savedSearch.searchTerm, { - shouldClearFilters: true, - autocompleteResults: false - }); - } - - savedSearch?.filters?.forEach((filter) => { - filter.values.forEach((value: string) => { - addFilter(filter.field, value, 'any'); - }); - }); - setActiveSearchId(id); - }; - const restoreInitialFilters = () => { - initialFilters.forEach((filter) => { - filter.values.forEach((value: string) => { - addFilter(filter.field, value, 'any'); - }); - }); - }; - - const revertSearch = () => { - setSearchTerm('', { - shouldClearFilters: true, - autocompleteResults: false - }); - restoreInitialFilters(); - setActiveSearchId(''); - }; - const toggleSavedSearches = (id: string) => { - const savedSearch = savedSearches.filter((search) => search.id === id); - - if (savedSearch) { - if (!isSavedSearchActive(id)) { - displaySavedSearch(id); - } else { - revertSearch(); - } - } - }; - - const isSavedSearchActive = (id: string): boolean => { - return activeSearchId === id; - }; - - const ascendingSavedSearches = savedSearches.sort((a, b) => - a.name.localeCompare(b.name) - ); const filtersByColumn = useMemo( () => @@ -179,11 +133,24 @@ export const DrawerInterior: React.FC = (props) => { Advanced Filters - + - +
+ { + if (location.pathname !== '/inventory') + history.push('/inventory?q=' + value); + setSearchTerm(value, { + shouldClearFilters: false, + autocompleteResults: false + }); + }} + /> +
{clearFilters && ( @@ -398,56 +365,140 @@ export const DrawerInterior: React.FC = (props) => { )} - - - - Saved Searches - - - - - - - }> - Saved Searches + + } + classes={{ + root: classes.root2, + content: classes.content, + disabled: classes.disabled2, + expanded: classes.expanded2 + }} + > +
+

Saved Searches

+
- - {ascendingSavedSearches.length > 0 ? ( - - {ascendingSavedSearches.map((search) => ( - - - toggleSavedSearches(search.id)} /> + + + + {savedSearches.length > 0 ? ( + ({ ...search }))} + rowCount={savedSearchCount} + columns={[ + { + field: 'name', + headerName: 'Name', + flex: 1, + width: 100, + description: 'Name', + renderCell: (cellValues) => { + const applyFilter = () => { + // if (clearFilters) clearFilters(); + localStorage.setItem( + 'savedSearch', + JSON.stringify(cellValues.row) + ); + setSearchTerm(cellValues.row.searchTerm, { + shouldClearFilters: false, + autocompleteResults: false + }); + if (location.pathname !== '/inventory') + history.push( + '/inventory?q=' + cellValues.row.searchTerm + ); + + // Apply the filters + cellValues.row.filters.forEach((filter) => { + filter.values.forEach((value) => { + addFilter(filter.field, value, 'any'); + }); + }); + }; + return ( +
{ + if (e.key === 'Enter') { + applyFilter(); + } + }} + style={{ + cursor: 'pointer', + textAlign: 'left', + width: '100%' + }} + > + {cellValues.value} +
+ ); } - label={search.name} - sx={{ padding: '0px' }} - value={search.id} - checked={isSavedSearchActive(search.id)} - /> - - deleteSearch(search.id)} - > - - - - ))} - - ) : ( - - - No Saved Searches - - - )} - + }, + { + field: 'actions', + headerName: '', + flex: 0.1, + renderCell: (cellValues) => { + const searchId = cellValues.id.toString(); + return ( +
+ { + e.stopPropagation(); + deleteSearch(searchId); + }} + tabIndex={0} + onKeyDown={(e) => { + if (e.key === 'Enter') { + deleteSearch(searchId); + } + }} + > + + +
+ ); + } + } + ]} + initialState={{ + pagination: { + paginationModel: { + pageSize: 5 + } + } + }} + pageSizeOptions={[5, 10]} + disableRowSelectionOnClick + sx={{ + disableColumnfilter: 'true', + '& .MuiDataGrid-row:hover': { + cursor: 'pointer' + } + }} + /> + ) : ( +
No Saved Searches
+ )} + + + ); diff --git a/frontend/src/components/FilterDrawerV2.tsx b/frontend/src/components/FilterDrawerV2.tsx index 20bfe0a4..abdcc9a5 100644 --- a/frontend/src/components/FilterDrawerV2.tsx +++ b/frontend/src/components/FilterDrawerV2.tsx @@ -4,7 +4,7 @@ import { withSearch } from '@elastic/react-search-ui'; import Box from '@mui/material/Box'; import Drawer from '@mui/material/Drawer'; import { DrawerInterior } from './DrawerInterior'; -import { RegionAndOrganizationFilters } from './RegionAndOrganizationFilters'; +import { OrganizationSearch } from './OrganizationSearch'; import FilterAltIcon from '@mui/icons-material/FilterAlt'; import { matchPath } from 'utils/matchPath'; import { useLocation } from 'react-router-dom'; @@ -18,7 +18,6 @@ export const FilterDrawer: FC< isFilterDrawerOpen: boolean; isMobile: boolean; setIsFilterDrawerOpen: (isOpen: boolean) => void; - initialFilters: any[]; } > = (props) => { const { @@ -31,8 +30,7 @@ export const FilterDrawer: FC< clearFilters, searchTerm, setSearchTerm, - filters, - initialFilters + filters } = props; const { pathname } = useLocation(); @@ -46,12 +44,10 @@ export const FilterDrawer: FC< - {matchPath( ['/inventory', '/inventory/domains', '/inventory/vulnerabilities'], @@ -65,7 +61,6 @@ export const FilterDrawer: FC< clearFilters={filters.length > 0 ? () => clearFilters([]) : undefined} searchTerm={searchTerm} setSearchTerm={setSearchTerm} - initialFilters={initialFilters} /> ) : ( <> diff --git a/frontend/src/components/Footer/Footer.tsx b/frontend/src/components/Footer/Footer.tsx index 2b59112b..41a87977 100644 --- a/frontend/src/components/Footer/Footer.tsx +++ b/frontend/src/components/Footer/Footer.tsx @@ -41,7 +41,6 @@ export const CrossfeedFooter: React.FC = (props) => { className={footerClasses.footerNavLink} href="https://www.cisa.gov" target="_blank" - rel="noopener noreferrer" > CISA Homepage diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index fcfb91df..83704996 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -7,10 +7,7 @@ import { IconButton, Drawer, ListItem, - List, - Box, - Typography, - useMediaQuery + List } from '@mui/material'; import { ChevronLeft, FilterAlt, Menu as MenuIcon } from '@mui/icons-material'; import { NavItem } from './NavItem'; @@ -19,10 +16,123 @@ import logo from '../assets/cyhydashboard.svg'; import cisaLogo from '../assets/cisaSeal.svg'; import { UserMenu } from './UserMenu'; import { matchPath } from 'utils/matchPath'; -import { useTheme } from '@mui/system'; -import { useUserLevel } from 'hooks/useUserLevel'; -const Root = styled('div')(() => ({})); +const PREFIX = 'Header'; + +const classes = { + inner: `${PREFIX}-inner`, + menuButton: `${PREFIX}-menuButton`, + logo: `${PREFIX}-logo`, + cisaLogo: `${PREFIX}-1cisaLogo`, + spacing: `${PREFIX}-spacing`, + activeMobileLink: `${PREFIX}-activeMobileLink`, + link: `${PREFIX}-link`, + userLink: `${PREFIX}-userLink`, + lgNav: `${PREFIX}-lgNav`, + selectOrg: `${PREFIX}-selectOrg`, + option: `${PREFIX}-option` +}; + +const Root = styled('div')(({ theme }) => ({ + [`.${classes.inner}`]: { + maxWidth: '1440px', + width: '100%', + margin: '0 auto' + }, + + [`.${classes.menuButton}`]: { + marginLeft: theme.spacing(2), + display: 'flex' + }, + [`.${classes.cisaLogo}`]: { + height: 40, + marginRight: theme.spacing(1) + }, + [`.${classes.logo}`]: { + width: 175, + minWidth: 175, + padding: theme.spacing(), + paddingLeft: 0, + [theme.breakpoints.down('xl')]: { + display: 'flex' + } + }, + [`.${classes.spacing}`]: { + flexGrow: 1 + }, + [`.${classes.activeMobileLink}`]: { + fontWeight: 700, + '&:after': { + content: "''", + position: 'absolute', + top: 0, + bottom: 0, + left: 0, + height: '100%', + width: 2, + backgroundColor: theme.palette.primary.main + } + }, + + [`.${classes.link}`]: { + position: 'relative', + color: 'white', + textDecoration: 'none', + margin: `0 ${theme.spacing()}px`, + padding: theme.spacing(), + borderBottom: '2px solid transparent', + fontWeight: 600 + }, + [`.${classes.userLink}`]: { + [theme.breakpoints.down('md')]: { + display: 'flex' + }, + [theme.breakpoints.up('lg')]: { + display: 'flex', + alignItems: 'center', + marginLeft: '1rem', + '& svg': { + marginRight: theme.spacing() + }, + border: 'none', + textDecoration: 'none' + } + }, + [`.${classes.lgNav}`]: { + display: 'flex', + [theme.breakpoints.down('sm')]: { + display: 'flex' + } + }, + + [`.${classes.selectOrg}`]: { + border: '1px solid #FFFFFF', + borderRadius: '5px', + width: '200px', + padding: '3px', + marginLeft: '20px', + '& svg': { + color: 'white' + }, + '& input': { + color: 'white', + width: '100%' + }, + '& input:focus': { + outlineWidth: 0 + }, + '& fieldset': { + borderStyle: 'none' + }, + '& div div': { + paddingTop: '0 !important' + }, + '& div div div': { + marginTop: '-3px !important' + }, + height: '45px' + } +})); const GLOBAL_ADMIN = 3; const REGIONAL_ADMIN = 2; @@ -55,7 +165,6 @@ export const Header: React.FC = ({ }) => { const { pathname } = useLocation(); const { user, logout } = useAuthContext(); - const theme = useTheme(); const [drawerOpen, setDrawerOpen] = useState(false); const [isMobile, setIsMobile] = useState(false); @@ -65,7 +174,19 @@ export const Header: React.FC = ({ setDrawerOpen(newOpen); }; - const { userLevel, formattedUserType } = useUserLevel(); + let userLevel = 0; + if (user && user.isRegistered) { + if (user.userType === 'standard') { + userLevel = STANDARD_USER; + } else if (user.userType === 'globalAdmin') { + userLevel = GLOBAL_ADMIN; + } else if ( + user.userType === 'regionalAdmin' || + user.userType === 'globalView' + ) { + userLevel = REGIONAL_ADMIN; + } + } const navItems: NavItemType[] = [ { @@ -151,111 +272,51 @@ export const Header: React.FC = ({ drawerItems = [...navItems, ...userMenuItems]; } - const isSmallerThanMds = useMediaQuery(theme.breakpoints.down('mds')); - return ( - - - - {matchPath(['/', '/inventory'], pathname) && user ? ( - - ) : ( - <> - )} - Cybersecurity and Infrastructure Security Agency Logo + + {matchPath(['/', '/inventory'], pathname) && user ? ( + - - CyHy Dashboard Icon Navigate Home - - {!isMobile && ( - - {desktopNavItems.slice()} - - )} - - {!isSmallerThanMds ? ( - - {user && userLevel > 0 ? ( - {formattedUserType} - ) : ( - <> - )} - ) : ( <> )} - - {userLevel > 0 && ( - <>{!isMobile && } - )} - {user && isMobile && ( - - - - )} - + Cybersecurity and Infrastructure Security Agency Logo + + CyHy Dashboard Icon Navigate Home + + {!isMobile && ( +
{desktopNavItems.slice()}
+ )} +
+ {userLevel > 0 && ( + <>{!isMobile && } + )} + {user && isMobile && ( + + + + )} - +
= ({ exact component={NavLink} to={path} + activeClassName={classes.activeMobileLink} > {title} diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx index 942f2a82..e9c8fc58 100644 --- a/frontend/src/components/Layout.tsx +++ b/frontend/src/components/Layout.tsx @@ -8,7 +8,7 @@ import React, { import { styled } from '@mui/material/styles'; import { useLocation } from 'react-router-dom'; import { Box, Drawer, ScopedCssBaseline, useMediaQuery } from '@mui/material'; -import { GovBanner, Header } from 'components'; +import { Header, GovBanner } from 'components'; import { useUserActivityTimeout } from 'hooks/useUserActivityTimeout'; import { useAuthContext } from 'context/AuthContext'; import UserInactiveModal from './UserInactivityModal/UserInactivityModal'; @@ -77,8 +77,6 @@ export const Layout: React.FC> = ({ const { regions } = useStaticsContext(); - const [initialFilters, setInitialFilters] = useState([]); - const theme = useTheme(); const [isFilterDrawerOpen, setIsFilterDrawerOpen] = usePersistentState( 'isFilterDrawerOpen', @@ -143,7 +141,6 @@ export const Layout: React.FC> = ({ filter.values.forEach((val) => { addFilter(filter.field, val, filter.type); }); - setInitialFilters(initialFiltersForUser); }); // eslint-disable-next-line react-hooks/exhaustive-deps }, [regions, user]); @@ -176,7 +173,6 @@ export const Layout: React.FC> = ({ setIsFilterDrawerOpen={setIsFilterDrawerOpen} isFilterDrawerOpen={isFilterDrawerOpen} isMobile={isMobile} - initialFilters={initialFilters} /> ) : ( > = ({ isFilterDrawerOpen={isFilterDrawerOpen} setIsFilterDrawerOpen={setIsFilterDrawerOpen} /> -
- { - return ( - - - - - - ); -}; - -export const Logs: FC = () => { - const { apiPost } = useAuthContext(); - const [filters, setFilters] = useState>([]); - const [openDialog, setOpenDialog] = useState(false); - const [dialogDetails, setDialogDetails] = useState< - (LogDetails & { id: number }) | null - >(null); - const [logs, setLogs] = useState<{ - count: Number; - result: Array; - }>({ - count: 0, - result: [] - }); - - const fetchLogs = useCallback(async () => { - const tableFilters = filters.reduce( - (acc: { [key: string]: { value: any; operator: any } }, cur) => { - return { - ...acc, - [cur.field]: { - value: cur.value, - operator: cur.operator - } - }; - }, - {} - ); - const results = await apiPost('/logs/search', { - body: { - ...tableFilters - } - }); - setLogs(results); - }, [apiPost, filters]); - - useEffect(() => { - fetchLogs(); - }, [fetchLogs]); - - const logCols: GridColDef[] = [ - { - field: 'eventType', - headerName: 'Event', - minWidth: 100, - flex: 1 - }, - { - field: 'result', - headerName: 'Result', - minWidth: 100, - flex: 1 - }, - { - field: 'createdAt', - headerName: 'Timestamp', - type: 'dateTime', - minWidth: 100, - flex: 1, - valueFormatter: (e) => { - return `${differenceInCalendarDays( - Date.now(), - parseISO(e.value) - )} days ago`; - } - }, - { - field: 'payload', - headerName: 'Payload', - description: 'Click any payload cell to expand.', - sortable: false, - minWidth: 300, - flex: 2, - renderCell: (cellValues) => { - return ( - - 
{JSON.stringify(cellValues.row.payload, null, 2)}
- - ); - }, - valueFormatter: (e) => { - return JSON.stringify(e.value, null, 2); - } - }, - { - field: 'details', - headerName: 'Details', - maxWidth: 70, - flex: 1, - renderCell: (cellValues: GridRenderEditCellParams) => { - return ( - { - setOpenDialog(true); - setDialogDetails(cellValues.row); - }} - > - - - ); - } - } - ]; - - useEffect(() => { - fetchLogs(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [filters]); - - return ( - - - { - setFilters(model.items); - }} - initialState={{ - pagination: { paginationModel: { pageSize: 15 } } - }} - pageSizeOptions={[15, 30, 50, 100]} - /> - - setOpenDialog(false)} - scroll="paper" - fullWidth - maxWidth="lg" - > - Payload Details - - - 
{JSON.stringify(dialogDetails?.payload, null, 2)}
- - - - - ); -}; diff --git a/frontend/src/components/OrganizationList/OrganizationList.tsx b/frontend/src/components/OrganizationList/OrganizationList.tsx index 55f42330..86da8246 100644 --- a/frontend/src/components/OrganizationList/OrganizationList.tsx +++ b/frontend/src/components/OrganizationList/OrganizationList.tsx @@ -113,7 +113,7 @@ export const OrganizationList: React.FC<{ return ( <> - + void; filters: any[]; - setSearchTerm: (s: string, opts?: any) => void; - searchTerm: string; } -export const RegionAndOrganizationFilters: React.FC< - RegionAndOrganizationFiltersProps -> = ({ +export const OrganizationSearch: React.FC = ({ addFilter, removeFilter, - filters, - searchTerm: domainSearchTerm, - setSearchTerm: setDomainSearchTerm + filters }) => { const { setShowMaps, user, apiPost } = useAuthContext(); const { regions } = useStaticsContext(); + + //Are we still using this? + // const [tags, setTags] = useState([]); const [searchTerm, setSearchTerm] = useState(''); const [orgResults, setOrgResults] = useState([]); - const [isOpen, setIsOpen] = useState(false); let userLevel = 0; if (user && user.isRegistered) { @@ -99,18 +91,8 @@ export const RegionAndOrganizationFilters: React.FC< } }); const orgs = results.body.hits.hits.map((hit) => hit._source); - // Filter out organizations that match the exclusions - const refinedOrgs = orgs.filter((org) => { - let exlude = false; - ORGANIZATION_EXCLUSIONS.forEach((exc) => { - if (org.name.toLowerCase().includes(exc)) { - exlude = true; - } - }); - return !exlude; - }); // Filter out organizations that are already in the filters - const filteredOrgs = refinedOrgs.filter( + const filteredOrgs = orgs.filter( (org) => !filters.find( (filter) => @@ -120,30 +102,7 @@ export const RegionAndOrganizationFilters: React.FC< ) ) ); - // Sort filtered orgs by name - const sortedOrgs = filteredOrgs.sort((a, b) => - a.name.localeCompare(b.name) - ); - - // Utility function to replce HTML encodings - const decodeHtml = (orgName: string): string => { - const encodings: { [key: string]: string } = { - '&': '&', - '<': '<', - '>': '>', - '"': '"', - ''': "'" - }; - return orgName.replace(/&|<|>|"|'/g, (m) => { - return encodings[m]; - }); - }; - // Decode HTML encodings in org names - sortedOrgs.forEach((org) => { - org.name = decodeHtml(org.name); - }); - - setOrgResults(sortedOrgs); + setOrgResults(filteredOrgs); } catch (e) { console.log(e); } @@ -169,8 +128,13 @@ export const RegionAndOrganizationFilters: React.FC< } }; - const handleTextChange = (v: string) => { - setSearchTerm(v); + const handleTextChange = (e: React.ChangeEvent) => { + const newSearchTerm = e.target.value; + setSearchTerm(newSearchTerm); + }; + + const handleChange = (v: string) => { + debounce(searchOrganizations(v, regionFilterValues ?? []) as any, 400); }; useEffect(() => { @@ -203,49 +167,10 @@ export const RegionAndOrganizationFilters: React.FC< }, [regionFilterValues] ); - const history = useHistory(); - const location = useLocation(); - - const handleAddOrganization = (org: OrganizationShallow) => { - if (org) { - const exists = organizationsInFilters?.find((o) => o.id === org.id); - if (exists) { - removeFilter(ORGANIZATION_FILTER_KEY, org, 'any'); - } else { - addFilter(ORGANIZATION_FILTER_KEY, org, 'any'); - } - setSearchTerm(''); - setIsOpen(false); - if (org.name === 'Election') { - setShowMaps(true); - } else { - setShowMaps(false); - } - } else { - } - }; return ( <> - - { - if (location.pathname !== '/inventory') { - history.push(`/inventory?q=${value}`); - setDomainSearchTerm(value, { - shouldClearFilters: false, - refresh: true - }); - } - setDomainSearchTerm(value, { - shouldClearFilters: false - }); - }} - /> - { - if (e && e.type === 'change') { - handleTextChange(v); - } - }} - inputValue={searchTerm} + onInputChange={(_, v) => handleChange(v)} // freeSolo disableClearable - open={isOpen} - onOpen={() => { - setIsOpen(true); - }} options={orgResults} - onChange={(e, v) => { - setTimeout(() => { - handleAddOrganization(v); - }, 250); - return; - }} getOptionLabel={(option) => option.name} - ListboxProps={{ - sx: { - ':active': { - bgcolor: 'transparent' - } - } - }} renderOption={(params, option) => { return ( -
  • - +
  • + {option.name}
    • ); }} isOptionEqualToValue={(option, value) => option?.name === value?.name } + onChange={(event, value) => { + if (value) { + const exists = organizationsInFilters?.find( + (org) => org.id === value.id + ); + if (exists) { + // setSelectedOrgs(selectedOrgs.filter((org) => org.id !== value.id)) + removeFilter(ORGANIZATION_FILTER_KEY, value, 'any'); + } else { + // setSelectedOrgs([...selectedOrgs, value]) + addFilter(ORGANIZATION_FILTER_KEY, value, 'any'); + } + setSearchTerm(''); + if (value.name === 'Election') { + setShowMaps(true); + } else { + setShowMaps(false); + } + } else { + } + }} renderInput={(params) => ( setIsOpen(false)} + value={searchTerm} + onChange={handleTextChange} /> )} /> diff --git a/frontend/src/components/ReadySetCyber/RSCAuthLoginCreate.tsx b/frontend/src/components/ReadySetCyber/RSCAuthLoginCreate.tsx new file mode 100644 index 00000000..dce70908 --- /dev/null +++ b/frontend/src/components/ReadySetCyber/RSCAuthLoginCreate.tsx @@ -0,0 +1,240 @@ +import React, { useEffect, useState } from 'react'; +import { AuthForm } from 'components'; +import { Button } from '@trussworks/react-uswds'; +import { + Box, + Dialog, + DialogContent, + DialogTitle, + Link, + Typography +} from '@mui/material'; +import { useAuthContext } from 'context'; +import { + Authenticator, + ThemeProvider, + useAuthenticator +} from '@aws-amplify/ui-react'; +import { I18n } from 'aws-amplify'; + +import { RSCRegisterForm } from 'components/ReadySetCyber/RSCRegisterForm'; + +const TOTP_ISSUER = process.env.REACT_APP_TOTP_ISSUER; + +// Strings come from https://github.com/aws-amplify/amplify-ui/blob/main/packages/ui/src/i18n/dictionaries/authenticator/en.ts +I18n.putVocabulariesForLanguage('en-US', { + 'Setup TOTP': 'Set up 2FA', + 'Confirm TOTP Code': 'Enter 2FA Code' +}); + +const amplifyTheme = { + name: 'my-theme' +}; + +interface Errors extends Partial { + global?: string; +} + +export const RSCAuthLoginCreate: React.FC<{ showSignUp?: boolean }> = ({ + showSignUp = true +}) => { + const { apiPost, refreshUser } = useAuthContext(); + const [errors, setErrors] = useState({}); + const [open, setOpen] = useState(false); + const [registerSuccess, setRegisterSuccess] = useState(false); + // Once a user signs in, call refreshUser() so that the callback is called and the user gets signed in. + const { authStatus } = useAuthenticator((context) => [context.isPending]); + useEffect(() => { + refreshUser(); + }, [refreshUser, authStatus]); + + const formFields = { + signIn: { + username: { + label: 'Email', + placeholder: 'Enter your email address', + required: true, + autoFocus: true + }, + password: { + label: 'Password', + placeholder: 'Enter your password', + required: true + } + }, + confirmSignIn: { + confirmation_code: { + label: 'Confirmation Code', + placeholder: 'Enter code from your authenticator app', + autoFocus: true + } + }, + resetPassword: { + username: { + label: 'Email', + placeholder: 'Enter your email address', + required: true, + autoFocus: true + } + }, + confirmResetPassword: { + confirmation_code: { + label: 'Confirmation Code', + placeholder: 'Enter code sent to your email address', + autoFocus: true + } + }, + confirmSignUp: { + confirmation_code: { + label: 'Confirmation Code', + placeholder: 'Enter code sent to your email address', + autoFocus: true + } + }, + setupTOTP: { + QR: { + // Set the issuer and name so that the authenticator app shows them. + // TODO: Set the issuer to the email, once this is resolved: https://github.com/aws-amplify/amplify-ui/issues/3387. + totpIssuer: TOTP_ISSUER + // totpUsername: email, + }, + confirmation_code: { + label: + 'Set up 2FA by scanning the QR code with an authenticator app on your phone.', + autoFocus: true + } + } + }; + + const onSubmit: React.FormEventHandler = async (e) => { + e.preventDefault(); + try { + const { redirectUrl, state, nonce } = await apiPost('/auth/login', { + body: {} + }); + localStorage.setItem('state', state); + localStorage.setItem('nonce', nonce); + window.location.href = redirectUrl; + } catch (e) { + console.error(e); + setErrors({ + global: 'Something went wrong logging in.' + }); + } + }; + + const onClose = () => { + setOpen(false); + }; + + const RegistrationSuccessDialog = ( + setRegisterSuccess(false)} + maxWidth="xs" + > + REQUEST SENT + + Thank you for requesting a ReadySetCyber Dashboard account, you will + receive notification once this request is approved. + + + ); + + if (process.env.REACT_APP_USE_COGNITO) { + return ( + +

    Welcome to ReadySetCyber Dashboard

    + + + + {/* alert('hello')}> + Register + */} + + {open && ( + + )} + {RegistrationSuccessDialog} + + + New to ReadySetCyber Dashboard?  + + setOpen(true)} + > + Register Now + + + +
    **Warning**
    +
    + {' '} + This system contains U.S. Government Data. Unauthorized use of this + system is prohibited. Use of this computer system, authorized or + unauthorized, constitutes consent to monitoring of this system. +
    +
    + {' '} + This computer system, including all related equipment, networks, and + network devices (specifically including Internet access) are + provided only for authorized U.S. Government use. U.S. Government + computer systems may be monitored for all lawful purposes, including + to ensure that their use is authorized, for management of the + system, to facilitate protection against unauthorized access, and to + verify security procedures, survivability, and operational security. + Monitoring includes active attacks by authorized U.S. Government + entities to test or verify the security of this system. During + monitoring, information may be examined, recorded, copied and used + for authorized purposes. All information, including personal + information, placed or sent over this system may be monitored. +
    +
    + {' '} + Unauthorized use may subject you to criminal prosecution. Evidence + of unauthorized use collected during monitoring may be used for + administrative, criminal, or other adverse action. Use of this + system constitutes consent to monitoring for these purposes. +
    +     +     + ); + } + + return ( + +

    Welcome to ReadySetCyber Dashboard

    + {errors.global &&

    {errors.global}

    } + + +
    New to ReadySetCyber Dashboard? Register with Login.gov
    +     + {open && ( + + )} + {RegistrationSuccessDialog} + +     + ); +}; diff --git a/frontend/src/components/ReadySetCyber/RSCFooter.tsx b/frontend/src/components/ReadySetCyber/RSCFooter.tsx index c3c4877a..f68ae09a 100644 --- a/frontend/src/components/ReadySetCyber/RSCFooter.tsx +++ b/frontend/src/components/ReadySetCyber/RSCFooter.tsx @@ -117,7 +117,16 @@ export const RSCFooter: React.FC = () => { } }} > - {/* TODO: CRASM-738 Determine if the widget https://www.dhs.gov/ntas-widget removed here is needed. */} +