feat: resource changelog tracking & API (#252)

* feat: add resource changelog API service & repo

* feat: unit test for service

* feat: move to a separate service

* feat: add handler & proto

* feat: reorganize functions & track resource updates

* feat: fix metric counter & fix job changelog query

* feat: skip when there's no changes

* feat: update queries structure

* feat: fix query & handler function name

* fix: handler naming

* feat: update on code review

* feat: remove sorting from get query

* fix: lint

* feat: update proton commit

Makefile

@@ -5,7 +5,7 @@ NAME = "github.com/goto/optimus"
 LAST_COMMIT := $(shell git rev-parse --short HEAD)
 LAST_TAG := "$(shell git rev-list --tags --max-count=1)"
 OPMS_VERSION := "$(shell git describe --tags ${LAST_TAG})-next"
-PROTON_COMMIT := "0278f0402ee9d1b5f392e03c7f45160879924d78"
+PROTON_COMMIT := "01ffd0ca223431ae24a8de44827de0d96afef9b2"
 
 
 .PHONY: build test test-ci generate-proto unit-test-ci integration-test vet coverage clean install lint

core/job/service/changelog_service.go

@@ -3,10 +3,18 @@ package service
 import (
 	"context"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
 	"github.com/goto/optimus/core/job"
 	"github.com/goto/optimus/core/tenant"
 )
 
+var getChangelogFailures = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "get_changelog_errors",
+	Help: "errors occurred in get changelog",
+}, []string{"project", "job", "error"})
+
 type ChangeLogService struct {
 	jobRepo JobRepository
 }
@@ -20,10 +28,6 @@ func (cl *ChangeLogService) GetChangelog(ctx context.Context, projectName tenant
 			err.Error(),
 		).Inc()
 	}
-	getChangelogFeatureAdoption.WithLabelValues(
-		projectName.String(),
-		jobName.String(),
-	).Inc()
 
 	return changelog, err
 }

core/job/service/job_service.go

@@ -9,8 +9,6 @@ import (
 
 	"github.com/goto/salt/log"
 	"github.com/kushsharma/parallel"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/goto/optimus/core/event"
 	"github.com/goto/optimus/core/event/moderator"
@@ -38,20 +36,6 @@ const (
 	projectConfigPrefix = "GLOBAL__"
 )
 
-var (
-
-	// right now this is done to capture the feature adoption
-	getChangelogFeatureAdoption = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "get_changelog_total",
-		Help: "number of requests received for viewing changelog",
-	}, []string{"project", "job"})
-
-	getChangelogFailures = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "get_changelog_errors",
-		Help: "errors occurred in get changelog",
-	}, []string{"project", "job", "error"})
-)
-
 type JobService struct {
 	jobRepo        JobRepository
 	upstreamRepo   UpstreamRepository
@@ -453,23 +437,6 @@ func (j *JobService) ChangeNamespace(ctx context.Context, jobTenant, jobNewTenan
 	return nil
 }
 
-func (j *JobService) GetChangelog(ctx context.Context, projectName tenant.ProjectName, jobName job.Name) ([]*job.ChangeLog, error) {
-	changelog, err := j.jobRepo.GetChangelog(ctx, projectName, jobName)
-	if err != nil {
-		getChangelogFailures.WithLabelValues(
-			projectName.String(),
-			jobName.String(),
-			err.Error(),
-		).Inc()
-	}
-	getChangelogFeatureAdoption.WithLabelValues(
-		projectName.String(),
-		jobName.String(),
-	).Inc()
-
-	return changelog, err
-}
-
 func (j *JobService) Get(ctx context.Context, jobTenant tenant.Tenant, jobName job.Name) (*job.Job, error) {
 	jobs, err := j.GetByFilter(ctx,
 		filter.WithString(filter.ProjectName, jobTenant.ProjectName().String()),

core/resource/changelog.go

@@ -0,0 +1,22 @@
+package resource
+
+import "time"
+
+const (
+	EntityResourceChangelog = "resource_changelog"
+
+	ChangelogChangeTypeCreate = "create"
+	ChangelogChangeTypeUpdate = "update"
+	ChangelogChangeTypeDelete = "delete"
+)
+
+type Change struct {
+	Property string
+	Diff     string
+}
+
+type ChangeLog struct {
+	Change []Change
+	Type   string
+	Time   time.Time
+}

core/resource/handler/v1beta1/resource.go

@@ -35,9 +35,14 @@ type ResourceService interface {
 	SyncResources(ctx context.Context, tnnt tenant.Tenant, store resource.Store, names []string) (*resource.SyncResponse, error)
 }
 
+type ResourceChangeLogService interface {
+	GetChangelogs(ctx context.Context, projectName tenant.ProjectName, resourceName resource.Name) ([]*resource.ChangeLog, error)
+}
+
 type ResourceHandler struct {
-	l       log.Logger
-	service ResourceService
+	l                log.Logger
+	service          ResourceService
+	changelogService ResourceChangeLogService
 
 	pb.UnimplementedResourceServiceServer
 }
@@ -424,6 +429,29 @@ func (rh ResourceHandler) ApplyResources(ctx context.Context, req *pb.ApplyResou
 	return &pb.ApplyResourcesResponse{Statuses: respStatuses}, nil
 }
 
+func (rh ResourceHandler) GetResourceChangelogs(ctx context.Context, req *pb.GetResourceChangelogsRequest) (*pb.GetResourceChangelogsResponse, error) {
+	projectName, err := tenant.ProjectNameFrom(req.GetProjectName())
+	if err != nil {
+		return nil, errors.GRPCErr(err, "invalid project name")
+	}
+
+	resourceName, err := resource.NameFrom(req.GetResourceName())
+	if err != nil {
+		return nil, errors.GRPCErr(err, "invalid resource name")
+	}
+
+	changelogs, err := rh.changelogService.GetChangelogs(ctx, projectName, resourceName)
+	if err != nil {
+		return nil, errors.GRPCErr(err, fmt.Sprintf("unable to get changelog for resource %s", resourceName.String()))
+	}
+
+	responseChangelogs := make([]*pb.ResourceChangelog, len(changelogs))
+	for i, changelog := range changelogs {
+		responseChangelogs[i] = toChangelogProto(changelog)
+	}
+	return &pb.GetResourceChangelogsResponse{History: responseChangelogs}, nil
+}
+
 func writeError(logWriter writer.LogWriter, err error) {
 	if err == nil {
 		return
@@ -523,9 +551,26 @@ func raiseResourceDatastoreEventMetric(jobTenant tenant.Tenant, datastoreName, r
 	}).Inc()
 }
 
-func NewResourceHandler(l log.Logger, resourceService ResourceService) *ResourceHandler {
+func toChangelogProto(cl *resource.ChangeLog) *pb.ResourceChangelog {
+	pbChange := &pb.ResourceChangelog{
+		EventType: cl.Type,
+		Timestamp: cl.Time.String(),
+	}
+
+	pbChange.Change = make([]*pb.ResourceChange, len(cl.Change))
+	for i, change := range cl.Change {
+		pbChange.Change[i] = &pb.ResourceChange{
+			AttributeName: change.Property,
+			Diff:          change.Diff,
+		}
+	}
+	return pbChange
+}
+
+func NewResourceHandler(l log.Logger, resourceService ResourceService, changelogService ResourceChangeLogService) *ResourceHandler {
 	return &ResourceHandler{
-		l:       l,
-		service: resourceService,
+		l:                l,
+		service:          resourceService,
+		changelogService: changelogService,
 	}
 }