From 00564698ece9f92d71554a0195c648e80d2d8593 Mon Sep 17 00:00:00 2001 From: Hank Doupe Date: Sun, 6 Sep 2020 14:21:07 -0400 Subject: [PATCH 1/9] Add Tag table to keep track of deployments --- .../comp/migrations/0028_simulation_tag.py | 25 ++++++++ webapp/apps/comp/models.py | 5 ++ webapp/apps/publish/urls.py | 4 +- webapp/apps/publish/views.py | 60 +++++++++++++---- .../migrations/0017_auto_20200906_1719.py | 64 +++++++++++++++++++ .../migrations/0018_auto_20200906_1719.py | 41 ++++++++++++ .../migrations/0019_auto_20200906_1819.py | 19 ++++++ webapp/apps/users/models.py | 36 +++++++++-- webapp/apps/users/serializers.py | 38 +++++++++-- workers/cs_workers/models/manage.py | 6 +- 10 files changed, 271 insertions(+), 27 deletions(-) create mode 100755 webapp/apps/comp/migrations/0028_simulation_tag.py create mode 100755 webapp/apps/users/migrations/0017_auto_20200906_1719.py create mode 100755 webapp/apps/users/migrations/0018_auto_20200906_1719.py create mode 100755 webapp/apps/users/migrations/0019_auto_20200906_1819.py diff --git a/webapp/apps/comp/migrations/0028_simulation_tag.py b/webapp/apps/comp/migrations/0028_simulation_tag.py new file mode 100755 index 00000000..c077bd56 --- /dev/null +++ b/webapp/apps/comp/migrations/0028_simulation_tag.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.10 on 2020-09-06 17:19 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0017_auto_20200906_1719"), + ("comp", "0027_auto_20200406_1547"), + ] + + operations = [ + migrations.AddField( + model_name="simulation", + name="tag", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="sims", + to="users.Tag", + ), + ), + ] diff --git a/webapp/apps/comp/models.py b/webapp/apps/comp/models.py index b1b02bb3..27c3bb9c 100755 --- a/webapp/apps/comp/models.py +++ b/webapp/apps/comp/models.py @@ -267,6 +267,7 @@ def new_sim(self, user, project, inputs_status=None): sim = self.create( owner=user.profile, project=project, + tag=project.latest_tag, model_pk=model_pk, inputs=inputs, status="STARTED", @@ -319,6 +320,7 @@ def fork(self, sim, user): traceback=sim.traceback, sponsor=sim.sponsor, project=sim.project, + tag=sim.tag, run_time=sim.run_time, run_cost=0, exp_comp_datetime=sim.exp_comp_datetime, @@ -381,6 +383,9 @@ class Simulation(models.Model): project = models.ForeignKey( "users.Project", on_delete=models.SET_NULL, related_name="sims", null=True ) + tag = models.ForeignKey( + "users.Tag", on_delete=models.SET_NULL, related_name="sims", null=True + ) notify_on_completion = models.BooleanField(default=False) # run-time in seconds run_time = models.IntegerField(default=0) diff --git a/webapp/apps/publish/urls.py b/webapp/apps/publish/urls.py index 689c15cb..415ad895 100644 --- a/webapp/apps/publish/urls.py +++ b/webapp/apps/publish/urls.py @@ -6,7 +6,7 @@ ProjectDetailView, ProjectDetailAPIView, ProjectAPIView, - TagAPIView, + TagsAPIView, EmbedApprovalView, EmbedApprovalDetailView, DeploymentsView, @@ -39,7 +39,7 @@ ), path( "api/v1///tags/", - TagAPIView.as_view(), + TagsAPIView.as_view(), name="project_tags_api", ), path( diff --git a/webapp/apps/publish/views.py b/webapp/apps/publish/views.py index 6af028b3..fcc78313 100644 --- a/webapp/apps/publish/views.py +++ b/webapp/apps/publish/views.py @@ -32,6 +32,7 @@ Cluster, Deployment, EmbedApproval, + Tag, is_profile_active, ) from webapp.apps.users.permissions import StrictRequiresActive, RequiresActive @@ -40,6 +41,7 @@ ProjectSerializer, ProjectWithVersionSerializer, TagSerializer, + TagUpdateSerializer, EmbedApprovalSerializer, DeploymentSerializer, ) @@ -179,27 +181,63 @@ def post(self, request, *args, **kwargs): return Response(status=status.HTTP_401_UNAUTHORIZED) -class TagAPIView(GetProjectMixin, APIView): +class TagsAPIView(GetProjectMixin, APIView): authentication_classes = ( SessionAuthentication, BasicAuthentication, TokenAuthentication, ) + permission_classes = (StrictRequiresActive,) def get(self, request, *args, **kwargs): - ser = TagSerializer(self.get_object(**kwargs), context={"request": request}) - return Response(ser.data, status=status.HTTP_200_OK) + project = self.get_object(**kwargs) + if not project.has_write_access(request.user): + raise PermissionDenied() + return Response( + { + "staging_tag": TagSerializer(instance=project.staging_tag).data, + "latest_tag": TagSerializer(instance=project.latest_tag).data, + }, + status=status.HTTP_200_OK, + ) def post(self, request, *args, **kwargs): project = self.get_object(**kwargs) - if request.user.is_authenticated and project.has_write_access(request.user): - serializer = TagSerializer(project, data=request.data) - if serializer.is_valid(): - serializer.save() - return Response(serializer.data, status=status.HTTP_200_OK) - else: - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - return Response(status=status.HTTP_401_UNAUTHORIZED) + if not project.has_write_access(request.user): + raise PermissionDenied() + serializer = TagUpdateSerializer(data=request.data) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + data = serializer.validated_data + print("data", data) + if data.get("staging_tag") is not None: + tag, _ = Tag.objects.get_or_create( + project=project, + image_tag=data.get("staging_tag"), + defaults=dict(cpu=project.cpu, memory=project.memory), + ) + project.staging_tag = tag + elif "staging_tag" in data: + project.staging_tag = None + + if data.get("latest_tag") is not None: + tag, _ = Tag.objects.get_or_create( + project=project, + image_tag=data.get("latest_tag"), + defaults=dict(cpu=project.cpu, memory=project.memory), + ) + project.latest_tag = tag + + project.save() + + return Response( + { + "staging_tag": TagSerializer(instance=project.staging_tag).data, + "latest_tag": TagSerializer(instance=project.latest_tag).data, + }, + status=status.HTTP_200_OK, + ) class RecentModelsAPIView(generics.ListAPIView): diff --git a/webapp/apps/users/migrations/0017_auto_20200906_1719.py b/webapp/apps/users/migrations/0017_auto_20200906_1719.py new file mode 100755 index 00000000..81f391c5 --- /dev/null +++ b/webapp/apps/users/migrations/0017_auto_20200906_1719.py @@ -0,0 +1,64 @@ +# Generated by Django 3.0.10 on 2020-09-06 17:19 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0016_auto_20200904_1054"), + ] + + operations = [ + migrations.RenameField( + model_name="deployment", old_name="tag", new_name="tag_deprecated", + ), + migrations.RenameField( + model_name="project", + old_name="latest_tag", + new_name="latest_tag_deprecated", + ), + migrations.RenameField( + model_name="project", + old_name="staging_tag", + new_name="staging_tag_deprecated", + ), + migrations.CreateModel( + name="Tag", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("image_tag", models.CharField(max_length=64, null=True)), + ( + "cpu", + models.DecimalField( + decimal_places=1, default=2, max_digits=5, null=True + ), + ), + ( + "memory", + models.DecimalField( + decimal_places=1, default=6, max_digits=5, null=True + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "project", + models.ForeignKey( + null=django.db.models.deletion.CASCADE, + on_delete=django.db.models.deletion.SET_NULL, + related_name="tags", + to="users.Project", + ), + ), + ], + ), + ] diff --git a/webapp/apps/users/migrations/0018_auto_20200906_1719.py b/webapp/apps/users/migrations/0018_auto_20200906_1719.py new file mode 100755 index 00000000..a988656b --- /dev/null +++ b/webapp/apps/users/migrations/0018_auto_20200906_1719.py @@ -0,0 +1,41 @@ +# Generated by Django 3.0.10 on 2020-09-06 17:19 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0017_auto_20200906_1719"), + ] + + operations = [ + migrations.AddField( + model_name="deployment", + name="tag", + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.SET_NULL, to="users.Tag" + ), + ), + migrations.AddField( + model_name="project", + name="latest_tag", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="latest", + to="users.Tag", + ), + ), + migrations.AddField( + model_name="project", + name="staging_tag", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="staging", + to="users.Tag", + ), + ), + ] diff --git a/webapp/apps/users/migrations/0019_auto_20200906_1819.py b/webapp/apps/users/migrations/0019_auto_20200906_1819.py new file mode 100755 index 00000000..a5957850 --- /dev/null +++ b/webapp/apps/users/migrations/0019_auto_20200906_1819.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.10 on 2020-09-06 18:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0018_auto_20200906_1719"), + ] + + operations = [ + migrations.AddConstraint( + model_name="tag", + constraint=models.UniqueConstraint( + fields=("project", "image_tag"), name="unique_project_tag" + ), + ), + ] diff --git a/webapp/apps/users/models.py b/webapp/apps/users/models.py index ca02fbcb..1fa431e1 100755 --- a/webapp/apps/users/models.py +++ b/webapp/apps/users/models.py @@ -257,8 +257,15 @@ def callabledefault(): cluster_type = models.CharField(default="single-core", max_length=32) - latest_tag = models.CharField(null=True, max_length=64) - staging_tag = models.CharField(null=True, max_length=64) + latest_tag_deprecated = models.CharField(null=True, max_length=64) + staging_tag_deprecated = models.CharField(null=True, max_length=64) + + latest_tag = models.ForeignKey( + "Tag", null=True, on_delete=models.SET_NULL, related_name="latest" + ) + staging_tag = models.ForeignKey( + "Tag", null=True, on_delete=models.SET_NULL, related_name="staging" + ) objects = ProjectManager() @@ -387,6 +394,26 @@ class Meta: permissions = (("write_project", "Write project"),) +class Tag(models.Model): + project = models.ForeignKey( + "Project", on_delete=models.SET_NULL, related_name="tags", null=models.CASCADE + ) + image_tag = models.CharField(null=True, max_length=64) + cpu = models.DecimalField(max_digits=5, decimal_places=1, null=True, default=2) + memory = models.DecimalField(max_digits=5, decimal_places=1, null=True, default=6) + created_at = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return str(self.image_tag) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["project", "image_tag"], name="unique_project_tag" + ) + ] + + class DeploymentException(Exception): pass @@ -436,7 +463,8 @@ class Deployment(models.Model): last_ping_at = models.DateTimeField(auto_now_add=True) # Uses max length of django username field. name = models.CharField(null=True, max_length=150) - tag = models.CharField(null=True, max_length=64) + tag_deprecated = models.CharField(null=True, max_length=64) + tag = models.ForeignKey("Tag", null=True, on_delete=models.SET_NULL) status = models.CharField( default="creating", @@ -494,7 +522,7 @@ def create_deployment(self): resp = requests.post( f"{self.project.cluster.url}/deployments/{self.project}/", - json={"deployment_name": self.public_name, "tag": self.tag,}, + json={"deployment_name": self.public_name, "tag": str(self.tag)}, headers=self.project.cluster.headers(), ) diff --git a/webapp/apps/users/serializers.py b/webapp/apps/users/serializers.py index 549edfa8..90599822 100644 --- a/webapp/apps/users/serializers.py +++ b/webapp/apps/users/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from webapp.apps.users.models import Project, EmbedApproval, Deployment +from webapp.apps.users.models import Project, EmbedApproval, Deployment, Tag class DeploymentSerializer(serializers.ModelSerializer): @@ -32,7 +32,7 @@ class ProjectSerializer(serializers.ModelSerializer): cluster_type = serializers.CharField(required=False) sim_count = serializers.IntegerField(required=False) user_count = serializers.IntegerField(required=False) - latest_tag = serializers.CharField(allow_null=True, required=False) + latest_tag = serializers.StringRelatedField(required=False) # see to_representation # has_write_access = serializers.BooleanField(source="has_write_access") @@ -83,7 +83,7 @@ class ProjectWithVersionSerializer(serializers.ModelSerializer): sim_count = serializers.IntegerField(required=False) version = serializers.CharField(required=False) user_count = serializers.IntegerField(required=False) - latest_tag = serializers.CharField(allow_null=True, required=False) + latest_tag = serializers.StringRelatedField(required=False) # see to_representation # has_write_access = serializers.BooleanField(source="has_write_access") @@ -122,15 +122,39 @@ class Meta: "callable_name", "tech", ) - read_only = ("sim_count", "status", "user_count", "version") + read_only = ("sim_count", "status", "user_count", "version", "latest_tag") + + +class TagUpdateSerializer(serializers.Serializer): + latest_tag = serializers.CharField(allow_null=True, required=False) + staging_tag = serializers.CharField(allow_null=True, required=False) + + def validate(self, attrs): + if attrs.get("latest_tag") is None and attrs.get("staging_tag") is None: + raise serializers.ValidationError( + "Either latest_tag or staging_tag must be specificied" + ) + return attrs class TagSerializer(serializers.ModelSerializer): + project = serializers.StringRelatedField() + class Meta: - model = Project + model = Tag fields = ( - "latest_tag", - "staging_tag", + "project", + "image_tag", + "memory", + "cpu", + "created_at", + ) + + read_only = ( + "project", + "memory", + "cpu", + "created_at", ) diff --git a/workers/cs_workers/models/manage.py b/workers/cs_workers/models/manage.py index e8bccfed..52866399 100644 --- a/workers/cs_workers/models/manage.py +++ b/workers/cs_workers/models/manage.py @@ -255,7 +255,7 @@ def stage_app(self, app): resp.status_code == 200 ), f"Got: {resp.url} {resp.status_code} {resp.text}" - sys.stdout.write(resp.json()["staging_tag"]) + sys.stdout.write(resp.json()["staging_tag"]["image_tag"]) def promote_app(self, app): resp = httpx.get( @@ -265,7 +265,7 @@ def promote_app(self, app): assert ( resp.status_code == 200 ), f"Got: {resp.url} {resp.status_code} {resp.text}" - staging_tag = resp.json()["staging_tag"] + staging_tag = resp.json()["staging_tag"]["image_tag"] resp = httpx.post( f"{self.config.cs_url}/apps/api/v1/{app['owner']}/{app['title']}/tags/", json={"latest_tag": staging_tag or self.tag, "staging_tag": None}, @@ -275,7 +275,7 @@ def promote_app(self, app): resp.status_code == 200 ), f"Got: {resp.url} {resp.status_code} {resp.text}" - sys.stdout.write(resp.json()["latest_tag"]) + sys.stdout.write(resp.json()["latest_tag"]["image_tag"]) def write_secrets(self, app): secret_config = copy.deepcopy(self.secret_template) From 98e578ed8f24b7143a7b919a1645da746e1915b9 Mon Sep 17 00:00:00 2001 From: Hank Doupe Date: Tue, 8 Sep 2020 08:17:49 -0400 Subject: [PATCH 2/9] Add tests fro new tags --- .../management/commands/init_projects.py | 77 ------------------- webapp/apps/billing/tests/test_models.py | 1 - webapp/apps/billing/tests/test_utils.py | 1 + webapp/apps/publish/tests/test_views.py | 39 +++++++++- 4 files changed, 36 insertions(+), 82 deletions(-) delete mode 100644 webapp/apps/billing/management/commands/init_projects.py diff --git a/webapp/apps/billing/management/commands/init_projects.py b/webapp/apps/billing/management/commands/init_projects.py deleted file mode 100644 index b53a4c7b..00000000 --- a/webapp/apps/billing/management/commands/init_projects.py +++ /dev/null @@ -1,77 +0,0 @@ -from django.core.management.base import BaseCommand -from django.contrib.auth import get_user_model - -from webapp.apps.billing.models import Project, Product, Plan -from webapp.apps.billing.utils import get_billing_data - -User = get_user_model() - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument( - "--use_stripe", - action="store_true", - dest="use_stripe", - help="Use stripe when initializing the database", - ) - parser.add_argument( - "--include_mock_data", - action="store_true", - dest="include_mock_data", - help="Include mock data. Used for testing.", - ) - - def handle(self, *args, **options): - use_stripe = options["use_stripe"] - include_mock_data = options["include_mock_data"] - billing = get_billing_data(include_mock_data=options["include_mock_data"]) - for app_name, plan in billing.items(): - if plan["username"]: - try: - profile = User.objects.get(username=plan["username"]).profile - except User.DoesNotExist: - print(f"Username: {plan['username']} not found.") - profile = None - else: - profile = None - if plan["sponsor"]: - try: - sponsor = User.objects.get(username=plan["sponsor"]).profile - except User.DoesNotExist: - print(f"Sponsor: {plan['sponsor']} not found.") - sponsor = None - else: - sponsor = None - project, _ = Project.objects.update_or_create( - name=plan["name"], - defaults={ - "app_name": plan["app_name"], - "server_cost": plan["server_cost"], - "exp_task_time": plan["exp_task_time"], - "exp_num_tasks": plan["exp_num_tasks"], - "is_public": plan["is_public"], - "profile": profile, - "sponsor": sponsor, - }, - ) - if use_stripe: - if Product.objects.filter(name=plan["name"]).count() == 0: - stripe_product = Product.create_stripe_object(plan["name"]) - product = Product.construct(stripe_product, project) - stripe_plan_lic = Plan.create_stripe_object( - amount=plan["amount"], - product=product, - usage_type="licensed", - interval=plan["interval"], - currency=plan["currency"], - ) - Plan.construct(stripe_plan_lic, product) - stripe_plan_met = Plan.create_stripe_object( - amount=plan["metered_amount"], - product=product, - usage_type="metered", - interval=plan["interval"], - currency=plan["currency"], - ) - Plan.construct(stripe_plan_met, product) diff --git a/webapp/apps/billing/tests/test_models.py b/webapp/apps/billing/tests/test_models.py index 8e5fe0e0..853c4222 100644 --- a/webapp/apps/billing/tests/test_models.py +++ b/webapp/apps/billing/tests/test_models.py @@ -167,7 +167,6 @@ def test_customer_sync_subscriptions(self, db, client): "cpu": 3, "memory": 9, "listed": True, - "latest_tag": "v1", } with mock_sync_projects(): diff --git a/webapp/apps/billing/tests/test_utils.py b/webapp/apps/billing/tests/test_utils.py index be3da327..e164b21d 100644 --- a/webapp/apps/billing/tests/test_utils.py +++ b/webapp/apps/billing/tests/test_utils.py @@ -135,6 +135,7 @@ def test_charge_run_with_sponsored_model(db): ) +@pytest.mark.requires_stripe class TestChargeDeployments: def test_embed_approvals(self, db, profile): project = Project.objects.get(title="Test-Viz") diff --git a/webapp/apps/publish/tests/test_views.py b/webapp/apps/publish/tests/test_views.py index f742a14d..46bad1ca 100755 --- a/webapp/apps/publish/tests/test_views.py +++ b/webapp/apps/publish/tests/test_views.py @@ -7,7 +7,7 @@ from guardian.shortcuts import assign_perm, remove_perm from webapp.apps.comp.models import Simulation -from webapp.apps.users.models import Project, Profile, EmbedApproval +from webapp.apps.users.models import Project, Profile, EmbedApproval, Tag from webapp.apps.users.serializers import ProjectSerializer from .utils import mock_sync_projects, mock_get_version @@ -29,7 +29,6 @@ def test_post(self, client): "cpu": 3, "memory": 9, "listed": True, - "latest_tag": "v1", } with mock_sync_projects(): resp = client.post("/apps/api/v1/", post_data) @@ -61,7 +60,6 @@ def test_get_detail_api(self, api_client, client, test_models): "exp_task_time": 20, "listed": True, "status": "live", - "latest_tag": "v1", "tech": "python-paramtools", "callable_name": None, } @@ -266,7 +264,40 @@ def test_deployments_api(self, api_client, test_models): assert resp.status_code == 200 project.refresh_from_db() - assert project.latest_tag == "v5" + assert project.latest_tag == Tag.objects.get(project=project, image_tag="v5") + + resp = api_client.post( + f"/apps/api/v1/{project.owner}/{project.title}/tags/", + data={"staging_tag": "v6"}, + format="json", + ) + + assert resp.status_code == 200 + project.refresh_from_db() + assert project.staging_tag == Tag.objects.get(project=project, image_tag="v6") + assert project.latest_tag == Tag.objects.get(project=project, image_tag="v5") + + resp = api_client.post( + f"/apps/api/v1/{project.owner}/{project.title}/tags/", + data={"latest_tag": "v6", "staging_tag": None}, + format="json", + ) + + assert resp.status_code == 200 + project.refresh_from_db() + assert project.staging_tag is None + assert project.latest_tag == Tag.objects.get(project=project, image_tag="v6") + + resp = api_client.post( + f"/apps/api/v1/{project.owner}/{project.title}/tags/", + data={}, + format="json", + ) + + assert resp.status_code == 400 + project.refresh_from_db() + assert project.staging_tag is None + assert project.latest_tag == Tag.objects.get(project=project, image_tag="v6") @pytest.mark.django_db From 5ad8b6a843dfa51a538b67f70aeedae1edff6d6b Mon Sep 17 00:00:00 2001 From: Hank Doupe Date: Tue, 8 Sep 2020 08:20:00 -0400 Subject: [PATCH 3/9] Add command sketching out how invoices could be created at the end of the billing period --- .../billing/management/commands/invoice.py | 122 ++++++++++++++++++ webapp/apps/users/models.py | 16 ++- 2 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 webapp/apps/billing/management/commands/invoice.py diff --git a/webapp/apps/billing/management/commands/invoice.py b/webapp/apps/billing/management/commands/invoice.py new file mode 100644 index 00000000..2499ab64 --- /dev/null +++ b/webapp/apps/billing/management/commands/invoice.py @@ -0,0 +1,122 @@ +""" +Create invoices for previous month's usage. + +- For each customer: + - Loop over all simulations that they own or sponsored. + - Sum time * price / sec + - Loop over all deployments where they own the embed approval or are owners. + - Sum length of deployment * price / sec +""" +from collections import defaultdict +from datetime import datetime + +from django.core.management.base import BaseCommand, CommandError + +from webapp.apps.billing.models import Customer + + +def process_simulations(simulations): + results = defaultdict(list) + for simulation in simulations.all(): + project = simulation.project + if getattr(simulation, "tag"): + server_cost = simulation.tag.server_cost + else: + server_cost = project.server_cost + run_time = simulation.run_time + + results[str(project)].append( + { + "model_pk": simulation.model_pk, + "server_cost": server_cost, + "run_time": run_time, + } + ) + + return results + + +def process_deployments(deployments): + results = defaultdict(list) + for deployment in deployments: + project = deployment.project + if getattr(deployment, "tag"): + server_cost = deployment.tag.server_cost + else: + server_cost = project.server_cost + + run_time = (deployment.deleted_at - deployment.created_at).seconds + + results[str(project)].append( + { + "deployment_id": deployment.id, + "server_cost": server_cost, + "run_time": run_time, + } + ) + + return results + + +def aggregate_metrics(grouped): + results = {} + for project, metrics in grouped.items(): + results[project] = { + "total_cost": sum( + metric["server_cost"] * metric["run_time"] / 3600 for metric in metrics + ), + "total_time": sum(metric["run_time"] for metric in metrics) / 3600, + } + return results + + +class Command(BaseCommand): + help = "Closes the specified poll for voting" + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + start = "2020-09-01" + end = str(datetime.now().date()) + for customer in Customer.objects.all(): + if not customer.user: + # print("customer", customer) + continue + profile = customer.user.profile + owner_sims = process_simulations( + profile.sims.filter(sponsor__isnull=True, creation_date__gte=start) + ) + sponsor_sims = process_simulations( + customer.user.profile.sponsored_sims.filter(creation_date__gte=start) + ) + + owner_costs = aggregate_metrics(owner_sims) + sponsor_costs = aggregate_metrics(sponsor_sims) + + ea_deployments = process_deployments( + profile.deployments.filter( + embed_approval__owner=profile, + created_at__gte=start, + deleted_at__lte=end, + ) + ) + + # same as sponsored for now. + owner_deployments = process_deployments( + profile.deployments.filter( + owner=profile, + embed_approval__isnull=True, + created_at__gte=start, + deleted_at__lte=end, + ) + ) + + ea_deployment_costs = aggregate_metrics(ea_deployments) + owner_deployment_costs = aggregate_metrics(owner_deployments) + + print(profile) + print(owner_costs) + print(sponsor_costs) + print(ea_deployment_costs) + print(owner_deployment_costs) diff --git a/webapp/apps/users/models.py b/webapp/apps/users/models.py index 1fa431e1..99e07cee 100755 --- a/webapp/apps/users/models.py +++ b/webapp/apps/users/models.py @@ -191,6 +191,13 @@ def create(self, *args, **kwargs): return project +def get_server_cost(cpu, memory): + """Hourly compute costs""" + cpu_price = COMPUTE_PRICING["cpu"] + memory_price = COMPUTE_PRICING["memory"] + return float(cpu) * cpu_price + float(memory) * memory_price + + class Project(models.Model): SECS_IN_HOUR = 3600.0 title = models.CharField(max_length=255) @@ -312,9 +319,7 @@ def n_secs_per_penny(self): @property def server_cost(self): """Hourly compute costs""" - cpu_price = COMPUTE_PRICING["cpu"] - memory_price = COMPUTE_PRICING["memory"] - return float(self.cpu) * cpu_price + float(self.memory) * memory_price + return get_server_cost(self.cpu, self.memory) @property def server_cost_in_secs(self): @@ -406,6 +411,11 @@ class Tag(models.Model): def __str__(self): return str(self.image_tag) + @property + def server_cost(self): + """Hourly compute costs""" + return get_server_cost(self.cpu, self.memory) + class Meta: constraints = [ models.UniqueConstraint( From 9c09e59fc15f1411bef20e7a8c449103a83ff7dd Mon Sep 17 00:00:00 2001 From: Hank Doupe Date: Tue, 8 Sep 2020 09:19:37 -0400 Subject: [PATCH 4/9] Add code for generating Stripe invoices --- .../billing/management/commands/invoice.py | 69 +++++++++++++++++-- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/webapp/apps/billing/management/commands/invoice.py b/webapp/apps/billing/management/commands/invoice.py index 2499ab64..0c67ce28 100644 --- a/webapp/apps/billing/management/commands/invoice.py +++ b/webapp/apps/billing/management/commands/invoice.py @@ -7,13 +7,20 @@ - Loop over all deployments where they own the embed approval or are owners. - Sum length of deployment * price / sec """ +import math +import os from collections import defaultdict from datetime import datetime from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone from webapp.apps.billing.models import Customer +import stripe + +stripe.api_key = os.environ.get("STRIPE_SECRET") + def process_simulations(simulations): results = defaultdict(list) @@ -62,6 +69,7 @@ def aggregate_metrics(grouped): results = {} for project, metrics in grouped.items(): results[project] = { + "n": len(metrics), "total_cost": sum( metric["server_cost"] * metric["run_time"] / 3600 for metric in metrics ), @@ -70,6 +78,23 @@ def aggregate_metrics(grouped): return results +def create_invoice_items(customer, aggregated_metrics, description, period): + for name, metrics in aggregated_metrics.items(): + n, total_cost, total_time = ( + metrics["n"], + metrics["total_cost"], + metrics["total_time"], + ) + total_time_mins = round(total_time * 60, 2) + stripe.InvoiceItem.create( + customer=customer.stripe_id, + amount=int(total_cost * 100), + description=f"{name} ({n} {description} totalling {total_time_mins} minutes)", + period=period, + currency="usd", + ) + + class Command(BaseCommand): help = "Closes the specified poll for voting" @@ -77,8 +102,8 @@ def add_arguments(self, parser): pass def handle(self, *args, **options): - start = "2020-09-01" - end = str(datetime.now().date()) + start = timezone.make_aware(datetime.fromisoformat("2020-08-01")) + end = timezone.now() for customer in Customer.objects.all(): if not customer.user: # print("customer", customer) @@ -91,8 +116,8 @@ def handle(self, *args, **options): customer.user.profile.sponsored_sims.filter(creation_date__gte=start) ) - owner_costs = aggregate_metrics(owner_sims) - sponsor_costs = aggregate_metrics(sponsor_sims) + owner_sim_costs = aggregate_metrics(owner_sims) + sponsor_sim_costs = aggregate_metrics(sponsor_sims) ea_deployments = process_deployments( profile.deployments.filter( @@ -116,7 +141,39 @@ def handle(self, *args, **options): owner_deployment_costs = aggregate_metrics(owner_deployments) print(profile) - print(owner_costs) - print(sponsor_costs) + print(owner_sim_costs) + print(sponsor_sim_costs) print(ea_deployment_costs) print(owner_deployment_costs) + + create_invoice = ( + bool(owner_sim_costs) + or bool(sponsor_sim_costs) + or bool(ea_deployment_costs) + or bool(owner_deployment_costs) + ) + + if not create_invoice: + continue + + start_ts = math.floor(start.timestamp()) + end_ts = math.floor(end.timestamp()) + + period = {"start": start_ts, "end": end_ts} + + create_invoice_items(customer, owner_sim_costs, "simulations", period) + create_invoice_items( + customer, sponsor_sim_costs, "sponsored simulations", period + ) + create_invoice_items( + customer, ea_deployment_costs, "embedded deployments", period + ) + create_invoice_items( + customer, owner_deployment_costs, "sponsored deployments", period + ) + + print("creating invoice for ", customer, customer.user.profile) + stripe.Invoice.create( + customer=customer.stripe_id, + description="Compute Studio Usage Subscription", + ) From 4e4a3a0661dddc0657c5a254ddd7e811538be635 Mon Sep 17 00:00:00 2001 From: Hank Doupe Date: Tue, 8 Sep 2020 09:26:27 -0400 Subject: [PATCH 5/9] Filter out deployments or sims with 0 run time --- webapp/apps/billing/management/commands/invoice.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/webapp/apps/billing/management/commands/invoice.py b/webapp/apps/billing/management/commands/invoice.py index 0c67ce28..ffd6a47d 100644 --- a/webapp/apps/billing/management/commands/invoice.py +++ b/webapp/apps/billing/management/commands/invoice.py @@ -32,6 +32,9 @@ def process_simulations(simulations): server_cost = project.server_cost run_time = simulation.run_time + if run_time <= 0: + continue + results[str(project)].append( { "model_pk": simulation.model_pk, @@ -54,6 +57,9 @@ def process_deployments(deployments): run_time = (deployment.deleted_at - deployment.created_at).seconds + if run_time <= 0: + continue + results[str(project)].append( { "deployment_id": deployment.id, From 1333f9fa575aff07dbfa9df8bfe1177e8c1d8e60 Mon Sep 17 00:00:00 2001 From: Hank Doupe Date: Thu, 10 Sep 2020 12:14:22 -0400 Subject: [PATCH 6/9] Add tests for new invoice flow and start removing usagerecord --- webapp/apps/billing/invoice.py | 208 ++++++++++++ .../billing/management/commands/invoice.py | 161 +-------- webapp/apps/billing/tests/test_invoice.py | 312 ++++++++++++++++++ webapp/apps/billing/tests/test_utils.py | 200 +---------- webapp/apps/billing/utils.py | 73 ---- webapp/apps/comp/views/core.py | 6 +- webapp/apps/publish/views.py | 5 +- webapp/apps/users/tests/utils.py | 1 + 8 files changed, 530 insertions(+), 436 deletions(-) create mode 100644 webapp/apps/billing/invoice.py create mode 100644 webapp/apps/billing/tests/test_invoice.py diff --git a/webapp/apps/billing/invoice.py b/webapp/apps/billing/invoice.py new file mode 100644 index 00000000..94a71629 --- /dev/null +++ b/webapp/apps/billing/invoice.py @@ -0,0 +1,208 @@ +""" +Create invoices for previous month's usage. + +- For each customer: + - Loop over all simulations that they own or sponsored. + - Sum time * price / sec + - Loop over all deployments where they own the embed approval or are owners. + - Sum length of deployment * price / sec +""" +import math +import os +from collections import defaultdict +from datetime import datetime + + +from django.utils import timezone + +from webapp.apps.billing.models import Customer + +import stripe + +stripe.api_key = os.environ.get("STRIPE_SECRET") + + +def process_simulations(simulations): + results = defaultdict(list) + for simulation in simulations.all(): + project = simulation.project + # Projects not doing pay per sim are handled elsewhere. + if not project.pay_per_sim: + continue + + if getattr(simulation, "tag"): + server_cost = simulation.tag.server_cost + else: + server_cost = project.server_cost + run_time = simulation.run_time + + if run_time <= 0: + continue + + results[str(project)].append( + { + "model_pk": simulation.model_pk, + "server_cost": server_cost, + "run_time": run_time, + } + ) + + return results + + +def process_deployments(deployments): + results = defaultdict(list) + for deployment in deployments: + project = deployment.project + # Projects not doing pay per sim are handled elsewhere. + if not project.pay_per_sim: + continue + + if getattr(deployment, "tag"): + server_cost = deployment.tag.server_cost + else: + server_cost = project.server_cost + + run_time = (deployment.deleted_at - deployment.created_at).seconds + + if run_time <= 0: + continue + + results[str(project)].append( + { + "deployment_id": deployment.id, + "server_cost": server_cost, + "run_time": run_time, + } + ) + + return results + + +def aggregate_metrics(grouped): + results = {} + for project, metrics in grouped.items(): + results[project] = { + "n": len(metrics), + "total_cost": round( + sum( + metric["server_cost"] * metric["run_time"] / 3600 + for metric in metrics + ), + 4, + ), + "total_time": round(sum(metric["run_time"] for metric in metrics) / 60, 4), + } + return results + + +def create_invoice_items(customer, aggregated_metrics, description, period): + for project, metrics in aggregated_metrics.items(): + n, total_cost, total_time = ( + metrics["n"], + metrics["total_cost"], + metrics["total_time"], + ) + if total_time > 60: + time_msg = f"{round(total_time / 60, 2)} hours" + else: + time_msg = f"{int(round(total_time / 60, 0))} minutes" + stripe.InvoiceItem.create( + customer=customer.stripe_id, + amount=int(total_cost * 100), + description=f"{project} ({n} {description} totalling {time_msg})", + period=period, + currency="usd", + metadata={"project": project, "description": description}, + ) + + +def invoice_customer(customer, start, end, send_invoice=True): + profile = customer.user.profile + owner_sims = process_simulations( + profile.sims.filter(sponsor__isnull=True, creation_date__gte=start) + ) + sponsor_sims = process_simulations( + customer.user.profile.sponsored_sims.filter(creation_date__gte=start) + ) + + owner_sim_costs = aggregate_metrics(owner_sims) + sponsor_sim_costs = aggregate_metrics(sponsor_sims) + + ea_deployments = process_deployments( + profile.deployments.filter( + embed_approval__owner=profile, created_at__gte=start, deleted_at__lte=end, + ) + ) + + # same as sponsored for now. + owner_deployments = process_deployments( + profile.deployments.filter( + owner=profile, + embed_approval__isnull=True, + created_at__gte=start, + deleted_at__lte=end, + ) + ) + + ea_deployment_costs = aggregate_metrics(ea_deployments) + owner_deployment_costs = aggregate_metrics(owner_deployments) + + summary = { + "detail": { + "simulations": {"owner": owner_sims, "sponsor": sponsor_sims,}, + "deployments": { + "owner": owner_deployments, + "embed_approval": ea_deployments, + }, + }, + "summary": { + "simulations": {"owner": owner_sim_costs, "sponsor": sponsor_sim_costs,}, + "deployments": { + "owner": owner_deployment_costs, + "embed_approval": ea_deployment_costs, + }, + }, + } + + print(profile) + print(summary["summary"]) + + create_invoice = ( + bool(owner_sim_costs) + or bool(sponsor_sim_costs) + or bool(ea_deployment_costs) + or bool(owner_deployment_costs) + ) + + if not create_invoice: + return summary + + start_ts = math.floor(start.timestamp()) + end_ts = math.floor(end.timestamp()) + + period = {"start": start_ts, "end": end_ts} + + if send_invoice: + create_invoice_items(customer, owner_sim_costs, "simulations", period) + create_invoice_items( + customer, sponsor_sim_costs, "sponsored simulations", period + ) + create_invoice_items( + customer, ea_deployment_costs, "embedded deployments", period + ) + create_invoice_items( + customer, owner_deployment_costs, "sponsored deployments", period + ) + + print("creating invoice for ", customer, customer.user.profile) + invoice = stripe.Invoice.create( + customer=customer.stripe_id, + description="Compute Studio Usage Subscription", + ) + else: + invoice = None + + summary["invoice"] = invoice + + return summary diff --git a/webapp/apps/billing/management/commands/invoice.py b/webapp/apps/billing/management/commands/invoice.py index ffd6a47d..6f9a85cc 100644 --- a/webapp/apps/billing/management/commands/invoice.py +++ b/webapp/apps/billing/management/commands/invoice.py @@ -7,98 +7,11 @@ - Loop over all deployments where they own the embed approval or are owners. - Sum length of deployment * price / sec """ -import math -import os -from collections import defaultdict -from datetime import datetime - from django.core.management.base import BaseCommand, CommandError from django.utils import timezone from webapp.apps.billing.models import Customer - -import stripe - -stripe.api_key = os.environ.get("STRIPE_SECRET") - - -def process_simulations(simulations): - results = defaultdict(list) - for simulation in simulations.all(): - project = simulation.project - if getattr(simulation, "tag"): - server_cost = simulation.tag.server_cost - else: - server_cost = project.server_cost - run_time = simulation.run_time - - if run_time <= 0: - continue - - results[str(project)].append( - { - "model_pk": simulation.model_pk, - "server_cost": server_cost, - "run_time": run_time, - } - ) - - return results - - -def process_deployments(deployments): - results = defaultdict(list) - for deployment in deployments: - project = deployment.project - if getattr(deployment, "tag"): - server_cost = deployment.tag.server_cost - else: - server_cost = project.server_cost - - run_time = (deployment.deleted_at - deployment.created_at).seconds - - if run_time <= 0: - continue - - results[str(project)].append( - { - "deployment_id": deployment.id, - "server_cost": server_cost, - "run_time": run_time, - } - ) - - return results - - -def aggregate_metrics(grouped): - results = {} - for project, metrics in grouped.items(): - results[project] = { - "n": len(metrics), - "total_cost": sum( - metric["server_cost"] * metric["run_time"] / 3600 for metric in metrics - ), - "total_time": sum(metric["run_time"] for metric in metrics) / 3600, - } - return results - - -def create_invoice_items(customer, aggregated_metrics, description, period): - for name, metrics in aggregated_metrics.items(): - n, total_cost, total_time = ( - metrics["n"], - metrics["total_cost"], - metrics["total_time"], - ) - total_time_mins = round(total_time * 60, 2) - stripe.InvoiceItem.create( - customer=customer.stripe_id, - amount=int(total_cost * 100), - description=f"{name} ({n} {description} totalling {total_time_mins} minutes)", - period=period, - currency="usd", - ) +from webapp.apps.billing.invoice import invoice_customer class Command(BaseCommand): @@ -108,78 +21,10 @@ def add_arguments(self, parser): pass def handle(self, *args, **options): - start = timezone.make_aware(datetime.fromisoformat("2020-08-01")) + start = timezone.make_aware(datetime.fromisoformat("2019-08-01")) end = timezone.now() for customer in Customer.objects.all(): if not customer.user: # print("customer", customer) continue - profile = customer.user.profile - owner_sims = process_simulations( - profile.sims.filter(sponsor__isnull=True, creation_date__gte=start) - ) - sponsor_sims = process_simulations( - customer.user.profile.sponsored_sims.filter(creation_date__gte=start) - ) - - owner_sim_costs = aggregate_metrics(owner_sims) - sponsor_sim_costs = aggregate_metrics(sponsor_sims) - - ea_deployments = process_deployments( - profile.deployments.filter( - embed_approval__owner=profile, - created_at__gte=start, - deleted_at__lte=end, - ) - ) - - # same as sponsored for now. - owner_deployments = process_deployments( - profile.deployments.filter( - owner=profile, - embed_approval__isnull=True, - created_at__gte=start, - deleted_at__lte=end, - ) - ) - - ea_deployment_costs = aggregate_metrics(ea_deployments) - owner_deployment_costs = aggregate_metrics(owner_deployments) - - print(profile) - print(owner_sim_costs) - print(sponsor_sim_costs) - print(ea_deployment_costs) - print(owner_deployment_costs) - - create_invoice = ( - bool(owner_sim_costs) - or bool(sponsor_sim_costs) - or bool(ea_deployment_costs) - or bool(owner_deployment_costs) - ) - - if not create_invoice: - continue - - start_ts = math.floor(start.timestamp()) - end_ts = math.floor(end.timestamp()) - - period = {"start": start_ts, "end": end_ts} - - create_invoice_items(customer, owner_sim_costs, "simulations", period) - create_invoice_items( - customer, sponsor_sim_costs, "sponsored simulations", period - ) - create_invoice_items( - customer, ea_deployment_costs, "embedded deployments", period - ) - create_invoice_items( - customer, owner_deployment_costs, "sponsored deployments", period - ) - - print("creating invoice for ", customer, customer.user.profile) - stripe.Invoice.create( - customer=customer.stripe_id, - description="Compute Studio Usage Subscription", - ) + invoice_customer(customer, start, end, send_invoice=False) diff --git a/webapp/apps/billing/tests/test_invoice.py b/webapp/apps/billing/tests/test_invoice.py new file mode 100644 index 00000000..38f04794 --- /dev/null +++ b/webapp/apps/billing/tests/test_invoice.py @@ -0,0 +1,312 @@ +from datetime import timedelta + +import pytest + +from django.utils import timezone +from django.db.models import F, Sum + +from webapp.settings import USE_STRIPE +from webapp.apps.billing import invoice + +from webapp.apps.comp.models import Simulation +from webapp.apps.users.models import Deployment, Project, Profile, EmbedApproval +from webapp.apps.users.tests.utils import mock_post_to_cluster + + +def round4(val): + return round(val, 4) + + +def sim_time_sum(simulations): + return round4(simulations.aggregate(Sum("run_time"))["run_time__sum"]) + + +def deployment_time_sum(deployments): + res = 0 + for deployment in deployments.all(): + res += (deployment.deleted_at - deployment.created_at).seconds + return round4(res) + + +def gen_simulations(owner, sponsor, project, run_times): + results = [] + for run_time in run_times: + sim = Simulation.objects.new_sim(owner.user, project, inputs_status="SUCCESS",) + sim.run_time = run_time + sim.sponsor = sponsor + sim.status = "SUCCESS" + sim.save() + results.append(sim) + + qs = Simulation.objects.filter(owner=owner, sponsor=sponsor, project=project) + assert qs.count() == len(run_times) + + return qs + + +def gen_deployments(owner, embed_approval, project, run_times): + with mock_post_to_cluster(): + results = [] + for i, run_time in enumerate(run_times): + deployment, _ = Deployment.objects.get_or_create_deployment( + project, f"test-{i}", owner=owner, embed_approval=embed_approval + ) + deployment.status = "terminated" + deployment.created_at = timezone.now() - timedelta(seconds=run_time) + deployment.deleted_at = timezone.now() + deployment.save() + results.append(deployment) + + qs = Deployment.objects.filter( + owner=owner, embed_approval=embed_approval, project=project + ) + assert qs.count() == len(run_times) + + return qs + + +@pytest.fixture +def owner_sims(profile): + """ + Generate simulations owned by profile that is being tested. + """ + project = Project.objects.get(title="Used-for-testing") + return gen_simulations( + owner=profile, sponsor=None, project=project, run_times=[60] * 10 + ) + + +@pytest.fixture +def other_profiles_sims(): + """ + Generate simulations owned by a profile that is NOT the one that + is primarily being tested. + """ + project = Project.objects.get(title="Used-for-testing") + modeler = Profile.objects.get(user__username="modeler") + return gen_simulations( + owner=modeler, sponsor=None, project=project, run_times=[45] * 8 + ) + + +@pytest.fixture +def sponsored_sims(profile): + """ + Generate simulations sponsored by a profile that is not the one being tested. + """ + sponsored_project = Project.objects.get(title="Used-for-testing-sponsored-apps") + return gen_simulations( + owner=profile, + sponsor=sponsored_project.sponsor, + project=sponsored_project, + run_times=[60] * 20, + ) + + +@pytest.fixture +def deployments(profile): + """ + Generate deployments where the owner is the primary one being tested. + """ + viz_project = Project.objects.get(title="Test-Viz") + return gen_deployments( + owner=profile, embed_approval=None, project=viz_project, run_times=[60] * 10 + ) + + +@pytest.fixture +def ea_deployments(profile): + """ + Generate deployments with embed approvals where the owner is the primary + one being tested. + """ + viz_project = Project.objects.get(title="Test-Viz") + ea = EmbedApproval.objects.create( + project=viz_project, + owner=profile, + url="https://test.compute.studio", + name="my-embed", + ) + return gen_deployments( + owner=profile, embed_approval=ea, project=viz_project, run_times=[60] * 10 + ) + + +class TestInvoice: + def test_owner_resources( + self, + profile, + owner_sims, + other_profiles_sims, + sponsored_sims, + deployments, + ea_deployments, + ): + start = timezone.now() - timedelta(days=7) + end = timezone.now() + + profile_invoices = invoice.invoice_customer( + profile.user.customer, start, end, send_invoice=USE_STRIPE + ) + + owner_sims_cost = round4( + sim_time_sum(owner_sims) * owner_sims.first().project.server_cost / 3600 + ) + owner_deployments_cost = round4( + deployment_time_sum(deployments) + * deployments.first().project.server_cost + / 3600 + ) + ea_deployments_cost = round4( + deployment_time_sum(ea_deployments) + * ea_deployments.first().project.server_cost + / 3600 + ) + + assert profile_invoices["summary"] == { + "deployments": { + "embed_approval": { + "modeler/Test-Viz": { + "n": ea_deployments.count(), + "total_cost": ea_deployments_cost, + "total_time": round4(deployment_time_sum(ea_deployments) / 60), + } + }, + "owner": { + "modeler/Test-Viz": { + "n": deployments.count(), + "total_cost": owner_deployments_cost, + "total_time": round4(deployment_time_sum(deployments) / 60), + } + }, + }, + "simulations": { + "owner": { + "modeler/Used-for-testing": { + "n": owner_sims.count(), + "total_cost": owner_sims_cost, + "total_time": round4(sim_time_sum(owner_sims) / 60), + } + }, + "sponsor": {}, + }, + } + stripe_invoice = profile_invoices["invoice"] + + assert profile_invoices["invoice"].amount_due == int( + 100 * (owner_sims_cost + owner_deployments_cost + ea_deployments_cost) + ) + + assert len(stripe_invoice.lines.data) == 3 + + for line in stripe_invoice.lines.data: + if ( + line.metadata.project == "modeler/Test-Viz" + and line.metadata.description == "embedded deployments" + ): + assert line.amount == int(100 * ea_deployments_cost) + elif ( + line.metadata.project == "modeler/Test-Viz" + and line.metadata.description == "sponsored deployments" + ): + assert line.amount == int(100 * owner_deployments_cost) + elif line.metadata.project == "modeler/Used-for-testing": + assert line.amount == int(100 * owner_sims_cost) + else: + raise ValueError(f"{line.name} {line.metadata}") + + def test_sponsor_resources( + self, + profile, + owner_sims, + other_profiles_sims, + sponsored_sims, + deployments, + ea_deployments, + ): + start = timezone.now() - timedelta(days=7) + end = timezone.now() + sponsor = Profile.objects.get(user__username="sponsor") + profile_invoices = invoice.invoice_customer( + sponsor.user.customer, start, end, send_invoice=USE_STRIPE + ) + + sponsored_sims_cost = round4( + sim_time_sum(sponsored_sims) + * sponsored_sims.first().project.server_cost + / 3600 + ) + + assert profile_invoices["summary"] == { + "deployments": {"embed_approval": {}, "owner": {},}, + "simulations": { + "owner": {}, + "sponsor": { + "modeler/Used-for-testing-sponsored-apps": { + "n": sponsored_sims.count(), + "total_cost": sponsored_sims_cost, + "total_time": round4(sim_time_sum(sponsored_sims) / 60), + } + }, + }, + } + stripe_invoice = profile_invoices["invoice"] + + assert profile_invoices["invoice"].amount_due == int(100 * sponsored_sims_cost) + + assert len(stripe_invoice.lines.data) == 1 + + for line in stripe_invoice.lines.data: + if line.metadata.project == "modeler/Used-for-testing-sponsored-apps": + assert line.amount == int(100 * sponsored_sims_cost) + else: + raise ValueError(f"{line.name} {line.metadata}") + + def test_other_profile_resources( + self, + profile, + owner_sims, + other_profiles_sims, + sponsored_sims, + deployments, + ea_deployments, + ): + start = timezone.now() - timedelta(days=7) + end = timezone.now() + modeler = Profile.objects.get(user__username="modeler") + profile_invoices = invoice.invoice_customer( + modeler.user.customer, start, end, send_invoice=USE_STRIPE + ) + + other_profiles_sims_cost = round4( + sim_time_sum(other_profiles_sims) + * other_profiles_sims.first().project.server_cost + / 3600 + ) + + assert profile_invoices["summary"] == { + "deployments": {"embed_approval": {}, "owner": {},}, + "simulations": { + "owner": { + "modeler/Used-for-testing": { + "n": other_profiles_sims.count(), + "total_cost": other_profiles_sims_cost, + "total_time": round4(sim_time_sum(other_profiles_sims) / 60), + } + }, + "sponsor": {}, + }, + } + stripe_invoice = profile_invoices["invoice"] + + assert profile_invoices["invoice"].amount_due == int( + 100 * other_profiles_sims_cost + ) + + assert len(stripe_invoice.lines.data) == 1 + + for line in stripe_invoice.lines.data: + if line.metadata.project == "modeler/Used-for-testing": + assert line.amount == int(100 * other_profiles_sims_cost) + else: + raise ValueError(f"{line.name} {line.metadata}") diff --git a/webapp/apps/billing/tests/test_utils.py b/webapp/apps/billing/tests/test_utils.py index e164b21d..42e81e29 100644 --- a/webapp/apps/billing/tests/test_utils.py +++ b/webapp/apps/billing/tests/test_utils.py @@ -1,21 +1,9 @@ -from datetime import timedelta - import pytest -from django.utils import timezone from django.contrib.auth import get_user_model -from webapp.apps.billing.utils import ( - has_payment_method, - ChargeRunMixin, - ChargeDeploymentMixin, -) -from webapp.apps.billing.models import UsageRecord, SubscriptionItem -from webapp.apps.comp.models import Inputs, Simulation -from webapp.apps.users.models import Profile, Project, EmbedApproval, Deployment -from webapp.apps.users.tests.utils import mock_post_to_cluster +from webapp.apps.billing.utils import has_payment_method -from .utils import gen_blank_customer User = get_user_model() @@ -28,189 +16,3 @@ def test_has_payment_method(db, customer): username="pmt-test", email="testyo@email.com", password="testtest2222" ) assert not has_payment_method(u) - - -@pytest.mark.requires_stripe -@pytest.mark.parametrize("pay_per_sim", [True, False]) -def test_pay_per_sim(db, customer, pay_per_sim): - owner = Profile.objects.get(user__username="modeler") - project = Project.objects.get(title="Used-for-testing", owner=owner) - project.pay_per_sim = pay_per_sim - project.save() - inputs = Inputs.objects.create(inputs_style="paramtools", project=project) - sim = Simulation.objects.create( - inputs=inputs, - project=project, - model_pk=Simulation.objects.next_model_pk(project), - owner=owner, - ) - - ur_count = UsageRecord.objects.count() - - cr = ChargeRunMixin() - cr.charge_run(sim, {"task_times": [100]}) - - if pay_per_sim: - assert ur_count + 1 == UsageRecord.objects.count() - else: - assert ur_count == UsageRecord.objects.count() - - -@pytest.mark.requires_stripe -def test_charge_run_adds_plans(db): - """ - Create new user and make sure that the model's plan is added with ChargeRun. - """ - - # setup - project = Project.objects.get(title="Used-for-testing") - profile = gen_blank_customer( - username="no_plans", email="tester@email.com", password="heyhey2222" - ) - inputs = Inputs.objects.create(inputs_style="paramtools", project=project) - sim = Simulation.objects.create( - inputs=inputs, - project=project, - model_pk=Simulation.objects.next_model_pk(project), - owner=profile, - ) - - # No plans set for model right now. - plan = project.product.plans.get(usage_type="metered") - assert ( - SubscriptionItem.objects.filter( - subscription__customer=profile.user.customer, plan=plan - ).count() - == 0 - ) - - cr = ChargeRunMixin() - cr.charge_run(sim, {"task_times": [100]}) - - # model's plan has been added. - assert ( - SubscriptionItem.objects.filter( - subscription__customer=profile.user.customer, plan=plan - ).count() - == 1 - ) - - -@pytest.mark.requires_stripe -def test_charge_run_with_sponsored_model(db): - """ - Create new user and make sure no plans are added after running sponsored model. - """ - # setup - project = Project.objects.get(title="Used-for-testing-sponsored-apps") - profile = gen_blank_customer( - username="no_plans_sponsored", email="tester@email.com", password="heyhey2222" - ) - inputs = Inputs.objects.create(inputs_style="paramtools", project=project) - sim = Simulation.objects.create( - inputs=inputs, - project=project, - model_pk=Simulation.objects.next_model_pk(project), - owner=profile, - ) - - # No plans set for model. - plan = project.product.plans.get(usage_type="metered") - assert ( - SubscriptionItem.objects.filter( - subscription__customer=profile.user.customer, plan=plan - ).count() - == 0 - ) - - cr = ChargeRunMixin() - cr.charge_run(sim, {"task_times": [100]}) - - # Still no plans. - assert ( - SubscriptionItem.objects.filter( - subscription__customer=profile.user.customer, plan=plan - ).count() - == 0 - ) - - -@pytest.mark.requires_stripe -class TestChargeDeployments: - def test_embed_approvals(self, db, profile): - project = Project.objects.get(title="Test-Viz") - - ea = EmbedApproval.objects.create( - project=project, - owner=profile, - url="https://embed.compute.studio", - name="my-test-embed", - ) - - with mock_post_to_cluster(): - deployment, _ = Deployment.objects.get_or_create_deployment( - project=project, name="my-deployment", owner=None, embed_approval=ea, - ) - - elapsed = timedelta(hours=2) - deleted_at = timezone.now() + elapsed - - deployment.deleted_at = deleted_at - deployment.status = "terminated" - deployment.save() - - ur_count = UsageRecord.objects.count() - - cd = ChargeDeploymentMixin() - cd.charge_deployment(deployment, use_stripe=True) - - si = SubscriptionItem.objects.get( - subscription__customer=ea.owner.user.customer, - plan=project.product.plans.get(usage_type="metered"), - ) - - assert ur_count + 1 == UsageRecord.objects.count() - - assert si.usage_records.count() == 1 - - ur = si.usage_records.first() - - quantity = deployment.project.run_cost(elapsed.seconds, adjust=True) - assert ur.quantity == quantity * 100 - - def test_sponsored(self, db, profile): - project = Project.objects.get(title="Test-Viz") - sponsor = Profile.objects.get(user__username="sponsor") - project.sponsor = sponsor - project.save() - - with mock_post_to_cluster(): - deployment, _ = Deployment.objects.get_or_create_deployment( - project=project, name="my-deployment", owner=None, embed_approval=None, - ) - - elapsed = timedelta(hours=2) - deleted_at = timezone.now() + elapsed - - deployment.deleted_at = deleted_at - deployment.status = "terminated" - deployment.save() - - ur_count = UsageRecord.objects.count() - - cd = ChargeDeploymentMixin() - cd.charge_deployment(deployment, use_stripe=True) - - si = SubscriptionItem.objects.get( - subscription__customer=sponsor.user.customer, - plan=project.product.plans.get(usage_type="metered"), - ) - - assert ur_count + 1 == UsageRecord.objects.count() - - assert si.usage_records.count() == 1 - - ur = si.usage_records.first() - - quantity = deployment.project.run_cost(elapsed.seconds, adjust=True) - assert ur.quantity == quantity * 100 diff --git a/webapp/apps/billing/utils.py b/webapp/apps/billing/utils.py index fed7c5f4..a5866f43 100755 --- a/webapp/apps/billing/utils.py +++ b/webapp/apps/billing/utils.py @@ -1,75 +1,2 @@ -import json - -from webapp.apps.users.models import Project -from .models import SubscriptionItem, UsageRecord, Plan - - -class ChargeRunMixin: - """ - Add charge_run method to outputs view. This class makes it easy to test - the logic for charging users for model runs. - """ - - def charge_run(self, sim, meta_dict, use_stripe=True): - sim.run_time = sum(meta_dict["task_times"]) - sim.run_cost = sim.project.run_cost(sim.run_time) - if use_stripe and sim.project.pay_per_sim: - quantity = sim.project.run_cost(sim.run_time, adjust=True) - plan = sim.project.product.plans.get(usage_type="metered") - # The sponsor is also stored on the Simulation object. However, the - # Project object should be considered the single source of truth - # for sending usage records. - sponsor = sim.project.sponsor - if sponsor is not None: - customer = sponsor.user.customer - else: - customer = sim.owner.user.customer - try: - si = SubscriptionItem.objects.get( - subscription__customer=customer, plan=plan - ) - except SubscriptionItem.DoesNotExist: - customer.sync_subscriptions(plans=Plan.objects.filter(pk=plan.pk)) - si = SubscriptionItem.objects.get( - subscription__customer=customer, plan=plan - ) - stripe_ur = UsageRecord.create_stripe_object( - quantity=Project.dollar_to_penny(quantity), - timestamp=None, - subscription_item=si, - ) - UsageRecord.construct(stripe_ur, si) - - -class ChargeDeploymentMixin: - def charge_deployment(self, deployment, use_stripe=True): - if use_stripe: - deployment_time = (deployment.deleted_at - deployment.created_at).seconds - quantity = deployment.project.run_cost(deployment_time, adjust=True) - plan = deployment.project.product.plans.get(usage_type="metered") - - if getattr(deployment, "embed_approval", None) is not None: - customer = deployment.embed_approval.owner.user.customer - elif getattr(deployment.project, "sponsor", None) is not None: - customer = deployment.project.sponsor.user.customer - else: - customer = deployment.owner - try: - si = SubscriptionItem.objects.get( - subscription__customer=customer, plan=plan - ) - except SubscriptionItem.DoesNotExist: - customer.sync_subscriptions(plans=Plan.objects.filter(pk=plan.pk)) - si = SubscriptionItem.objects.get( - subscription__customer=customer, plan=plan - ) - stripe_ur = UsageRecord.create_stripe_object( - quantity=Project.dollar_to_penny(quantity), - timestamp=None, - subscription_item=si, - ) - UsageRecord.construct(stripe_ur, si) - - def has_payment_method(user): return hasattr(user, "customer") and user.customer is not None diff --git a/webapp/apps/comp/views/core.py b/webapp/apps/comp/views/core.py index 3c463a95..13064f38 100755 --- a/webapp/apps/comp/views/core.py +++ b/webapp/apps/comp/views/core.py @@ -7,7 +7,7 @@ from webapp.apps.users.permissions import RequiresActive, RequiresPayment from webapp.settings import USE_STRIPE -from webapp.apps.billing.utils import has_payment_method, ChargeRunMixin +from webapp.apps.billing.utils import has_payment_method from webapp.apps.users.models import is_profile_active @@ -96,9 +96,9 @@ def get_object(self, model_pk, username, title): return obj -class RecordOutputsMixin(ChargeRunMixin): +class RecordOutputsMixin: def record_outputs(self, sim, data): - self.charge_run(sim, data["meta"], use_stripe=USE_STRIPE) + sim.run_time = sum(data["meta"]["task_times"]) sim.meta_data = data["meta"] sim.model_version = data.get("model_version", "NA") # successful run diff --git a/webapp/apps/publish/views.py b/webapp/apps/publish/views.py index 1c00a76e..33eb1030 100644 --- a/webapp/apps/publish/views.py +++ b/webapp/apps/publish/views.py @@ -28,7 +28,6 @@ # from webapp.settings import DEBUG from webapp.settings import USE_STRIPE -from webapp.apps.billing.utils import ChargeDeploymentMixin from webapp.apps.users.models import ( Project, Cluster, @@ -422,7 +421,7 @@ def get_queryset(self): return self.queryset -class DeploymentsDetailView(ChargeDeploymentMixin, APIView): +class DeploymentsDetailView(APIView): authentication_classes = ( SessionAuthentication, BasicAuthentication, @@ -473,7 +472,7 @@ def delete(self, request, *args, **kwargs): deployment.delete_deployment() - self.charge_deployment(deployment, use_stripe=USE_STRIPE) + # self.charge_deployment(deployment, use_stripe=USE_STRIPE) return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/webapp/apps/users/tests/utils.py b/webapp/apps/users/tests/utils.py index a17a6f92..81d96877 100644 --- a/webapp/apps/users/tests/utils.py +++ b/webapp/apps/users/tests/utils.py @@ -11,4 +11,5 @@ def mock_post_to_cluster(): matcher = re.compile(Cluster.objects.default().url) with requests_mock.Mocker(real_http=True) as mock: mock.register_uri("POST", matcher, json={}) + mock.register_uri("GET", matcher, json={}) yield From 7c73ef8cb5bdd6ad8246953e975fb3e000138c33 Mon Sep 17 00:00:00 2001 From: Hank Doupe Date: Thu, 10 Sep 2020 17:37:22 -0400 Subject: [PATCH 7/9] Make sure tag is backwards compatible until can be removed post-migration --- webapp/apps/comp/asyncsubmit.py | 10 ++++++---- webapp/apps/comp/views/views.py | 1 - webapp/apps/users/models.py | 9 +++++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/webapp/apps/comp/asyncsubmit.py b/webapp/apps/comp/asyncsubmit.py index 745dc50c..67d9dec1 100755 --- a/webapp/apps/comp/asyncsubmit.py +++ b/webapp/apps/comp/asyncsubmit.py @@ -125,11 +125,13 @@ def submit(self): "adjustment": inputs.deserialized_inputs, } print("submit", data) + project = self.sim.project + if project.latest_tag is None: + tag = project.latest_tag_deprecated + else: + tag = str(project.latest_tag) self.submitted_id = self.compute.submit_job( - project=inputs.project, - task_name=actions.SIM, - task_kwargs=data, - tag=self.sim.project.latest_tag, + project=inputs.project, task_name=actions.SIM, task_kwargs=data, tag=tag, ) print(f"job id: {self.submitted_id}") diff --git a/webapp/apps/comp/views/views.py b/webapp/apps/comp/views/views.py index 1fb8f55e..19c878d6 100755 --- a/webapp/apps/comp/views/views.py +++ b/webapp/apps/comp/views/views.py @@ -30,7 +30,6 @@ from webapp.settings import DEBUG, DEFAULT_VIZ_HOST -from webapp.apps.billing.models import SubscriptionItem, UsageRecord from webapp.apps.billing.utils import has_payment_method from webapp.apps.users.models import ( Project, diff --git a/webapp/apps/users/models.py b/webapp/apps/users/models.py index 99e07cee..4f9f2482 100755 --- a/webapp/apps/users/models.py +++ b/webapp/apps/users/models.py @@ -527,12 +527,17 @@ def public_name(self): def create_deployment(self): if self.tag is None: - self.tag = self.project.latest_tag + if self.project.latest_tag is None: + self.tag_deprecated = self.project.latest_tag_deprecated + tag = self.tag_deprecated + else: + self.tag = self.project.latest_tag + tag = str(self.tag) self.save() resp = requests.post( f"{self.project.cluster.url}/deployments/{self.project}/", - json={"deployment_name": self.public_name, "tag": str(self.tag)}, + json={"deployment_name": self.public_name, "tag": tag}, headers=self.project.cluster.headers(), ) From bc9a99980e53e32bf7f9ea081fc92c136076e45c Mon Sep 17 00:00:00 2001 From: Hank Doupe Date: Thu, 10 Sep 2020 18:15:31 -0400 Subject: [PATCH 8/9] Add cli args for invoice command and fix some date shenanigans --- webapp/apps/billing/invoice.py | 26 +++++++++++------ .../billing/management/commands/invoice.py | 28 ++++++++++++++++--- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/webapp/apps/billing/invoice.py b/webapp/apps/billing/invoice.py index 94a71629..cbdb9f79 100644 --- a/webapp/apps/billing/invoice.py +++ b/webapp/apps/billing/invoice.py @@ -11,7 +11,7 @@ import os from collections import defaultdict from datetime import datetime - +from pprint import pprint from django.utils import timezone @@ -26,6 +26,9 @@ def process_simulations(simulations): results = defaultdict(list) for simulation in simulations.all(): project = simulation.project + # Project is deleted. + if project is None: + continue # Projects not doing pay per sim are handled elsewhere. if not project.pay_per_sim: continue @@ -106,7 +109,7 @@ def create_invoice_items(customer, aggregated_metrics, description, period): if total_time > 60: time_msg = f"{round(total_time / 60, 2)} hours" else: - time_msg = f"{int(round(total_time / 60, 0))} minutes" + time_msg = f"{int(round(total_time, 1))} minutes" stripe.InvoiceItem.create( customer=customer.stripe_id, amount=int(total_cost * 100), @@ -120,10 +123,12 @@ def create_invoice_items(customer, aggregated_metrics, description, period): def invoice_customer(customer, start, end, send_invoice=True): profile = customer.user.profile owner_sims = process_simulations( - profile.sims.filter(sponsor__isnull=True, creation_date__gte=start) + profile.sims.filter(sponsor__isnull=True, creation_date__date__gte=start.date()) ) sponsor_sims = process_simulations( - customer.user.profile.sponsored_sims.filter(creation_date__gte=start) + customer.user.profile.sponsored_sims.filter( + creation_date__date__gte=start.date() + ) ) owner_sim_costs = aggregate_metrics(owner_sims) @@ -131,7 +136,9 @@ def invoice_customer(customer, start, end, send_invoice=True): ea_deployments = process_deployments( profile.deployments.filter( - embed_approval__owner=profile, created_at__gte=start, deleted_at__lte=end, + embed_approval__owner=profile, + created_at__gte=start.date(), + deleted_at__lte=end.date(), ) ) @@ -140,8 +147,8 @@ def invoice_customer(customer, start, end, send_invoice=True): profile.deployments.filter( owner=profile, embed_approval__isnull=True, - created_at__gte=start, - deleted_at__lte=end, + created_at__date__gte=start.date(), + deleted_at__date__lte=end.date(), ) ) @@ -165,8 +172,9 @@ def invoice_customer(customer, start, end, send_invoice=True): }, } - print(profile) - print(summary["summary"]) + print() + print("Customer username:", profile) + pprint(summary["summary"]) create_invoice = ( bool(owner_sim_costs) diff --git a/webapp/apps/billing/management/commands/invoice.py b/webapp/apps/billing/management/commands/invoice.py index 6f9a85cc..3e04dc06 100644 --- a/webapp/apps/billing/management/commands/invoice.py +++ b/webapp/apps/billing/management/commands/invoice.py @@ -7,6 +7,9 @@ - Loop over all deployments where they own the embed approval or are owners. - Sum length of deployment * price / sec """ +import calendar +from datetime import datetime + from django.core.management.base import BaseCommand, CommandError from django.utils import timezone @@ -14,17 +17,34 @@ from webapp.apps.billing.invoice import invoice_customer +def parse_date(date_str): + return timezone.make_aware(datetime.fromisoformat(date_str)) + + class Command(BaseCommand): help = "Closes the specified poll for voting" def add_arguments(self, parser): - pass + parser.add_argument("--start") + parser.add_argument("--end") + parser.add_argument("--dryrun", action="store_true") def handle(self, *args, **options): - start = timezone.make_aware(datetime.fromisoformat("2019-08-01")) - end = timezone.now() + print(options) + if options.get("start"): + start = parse_date(options["start"]) + else: + start = timezone.now().replace( + day=1, hour=0, minute=0, second=0, microsecond=0 + ) + if options.get("end"): + end = parse_date(options["end"]) + else: + _, end_day = calendar.monthrange(start.year, start.month) + end = start.replace(day=end_day) + print(f"Billing period: {str(start.date())} to {str(end.date())}") for customer in Customer.objects.all(): if not customer.user: # print("customer", customer) continue - invoice_customer(customer, start, end, send_invoice=False) + invoice_customer(customer, start, end, send_invoice=not options["dryrun"]) From e375cfca602349863da42c27875841b1627bb590 Mon Sep 17 00:00:00 2001 From: Hank Doupe Date: Fri, 11 Sep 2020 09:01:45 -0400 Subject: [PATCH 9/9] Fix deployments query --- webapp/apps/billing/invoice.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/apps/billing/invoice.py b/webapp/apps/billing/invoice.py index cbdb9f79..dcb509fa 100644 --- a/webapp/apps/billing/invoice.py +++ b/webapp/apps/billing/invoice.py @@ -137,8 +137,8 @@ def invoice_customer(customer, start, end, send_invoice=True): ea_deployments = process_deployments( profile.deployments.filter( embed_approval__owner=profile, - created_at__gte=start.date(), - deleted_at__lte=end.date(), + deleted_at__date__gte=start.date(), + deleted_at__date__lte=end.date(), ) ) @@ -147,7 +147,7 @@ def invoice_customer(customer, start, end, send_invoice=True): profile.deployments.filter( owner=profile, embed_approval__isnull=True, - created_at__date__gte=start.date(), + deleted_at__date__gte=start.date(), deleted_at__date__lte=end.date(), ) )