Export active repos + Fix ratelimiting

README.md

@@ -30,6 +30,16 @@ github_actions_workflow_billable_time_seconds{owner="totocorp",platform="UBUNTU"
 github_actions_workflow_billable_time_seconds{owner="totocorp",platform="UBUNTU",repo="repo-C",workflow="test",workflow_id="2"} 15
 ```
 
+### Active Repositories
+
+How many repositories under the organization are considered active. Depends on the `-max-last-push` setting.
+
+```
+# HELP github_actions_workflow_active_repos Last reported total of active repositories in the monitored org
+# TYPE github_actions_workflow_active_repos gauge
+github_actions_workflow_active_repos 174
+```
+
 ### Last Refresh Timestamp
 
 Last timestamp in seconds where the exported managed to refresh the data. Usefull for detecting stale data.

actions/collector.go

@@ -30,6 +30,7 @@ type UsageCollector struct {
 	billableTimeDesc        *prometheus.Desc
 	lastRefreshTimeDesc     *prometheus.Desc
 	lastRefreshDurationDesc *prometheus.Desc
+	activeReposDesc         *prometheus.Desc
 
 	refreshTicker *time.Ticker
 	cancelFunc    func()
@@ -39,7 +40,7 @@ type UsageCollector struct {
 	ready chan struct{}
 
 	lastUsageDataMu     sync.RWMutex
-	lastUsageData       Usage
+	lastUsageData       *Usage
 	lastRefreshTime     time.Time
 	lastRefreshDuration time.Duration
 
@@ -78,6 +79,12 @@ func NewUsageCollector(usagefetcher WorkflowUsageFetcher, logger *zap.Logger, re
 			nil,
 			nil,
 		),
+		activeReposDesc: prometheus.NewDesc(
+			"github_actions_workflow_active_repos",
+			"Last reported total of active repositories in the monitored org",
+			nil,
+			nil,
+		),
 	}
 
 	for _, opt := range opts {
@@ -105,25 +112,34 @@ func (c *UsageCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.billableTimeDesc
 	ch <- c.lastRefreshTimeDesc
 	ch <- c.lastRefreshDurationDesc
+	ch <- c.activeReposDesc
 }
 
 func (c *UsageCollector) Collect(ch chan<- prometheus.Metric) {
 	c.lastUsageDataMu.RLock()
 	defer c.lastUsageDataMu.RUnlock()
 
-	for _, workflowData := range c.lastUsageData {
-		for platform, value := range workflowData.BillableTime {
-			ch <- prometheus.MustNewConstMetric(
-				c.billableTimeDesc,
-				prometheus.GaugeValue,
-				value.Seconds(),
-				workflowData.Owner,
-				workflowData.Repo,
-				workflowData.Workflow,
-				strconv.FormatInt(workflowData.ID, 10),
-				platform,
-			)
+	if c.lastUsageData != nil {
+		for _, workflowData := range c.lastUsageData.Workflows {
+			for platform, value := range workflowData.BillableTime {
+				ch <- prometheus.MustNewConstMetric(
+					c.billableTimeDesc,
+					prometheus.GaugeValue,
+					value.Seconds(),
+					workflowData.Owner,
+					workflowData.Repo,
+					workflowData.Workflow,
+					strconv.FormatInt(workflowData.ID, 10),
+					platform,
+				)
+			}
 		}
+
+		ch <- prometheus.MustNewConstMetric(
+			c.activeReposDesc,
+			prometheus.GaugeValue,
+			float64(c.lastUsageData.ActiveRepos),
+		)
 	}
 
 	if !c.lastRefreshTime.IsZero() {

actions/collector_test.go

@@ -153,6 +153,15 @@ github_actions_workflow_last_refresh_duration_seconds 1
 github_actions_workflow_last_refresh_timestamp_seconds 1.697328e+09
 				`,
 		},
+		{
+			metricName:  "github_actions_workflow_active_repos",
+			mockOptions: defaultMockBehavior,
+			wantMetrics: `
+# HELP github_actions_workflow_active_repos Last reported total of active repositories in the monitored org
+# TYPE github_actions_workflow_active_repos gauge
+github_actions_workflow_active_repos 3
+				`,
+		},
 	} {
 		t.Run(testCase.metricName, func(t *testing.T) {
 			var (

actions/usage.go

@@ -12,7 +12,7 @@ import (
 )
 
 type WorkflowUsageFetcher interface {
-	Fetch(ctx context.Context) (Usage, error)
+	Fetch(ctx context.Context) (*Usage, error)
 }
 
 type WorkflowUsage struct {
@@ -24,7 +24,10 @@ type WorkflowUsage struct {
 	BillableTime map[string]time.Duration
 }
 
-type Usage []WorkflowUsage
+type Usage struct {
+	ActiveRepos int64
+	Workflows   []WorkflowUsage
+}
 
 type OrgUsageFetcher struct {
 	gh     *github.Client
@@ -45,7 +48,7 @@ func NewOrgUsageFetcher(concurencyLimit int, maxLastPushed time.Duration, org st
 	}
 }
 
-func (f *OrgUsageFetcher) Fetch(ctx context.Context) (Usage, error) {
+func (f *OrgUsageFetcher) Fetch(ctx context.Context) (*Usage, error) {
 	var (
 		usageMu sync.Mutex
 		usage   Usage
@@ -74,6 +77,9 @@ func (f *OrgUsageFetcher) Fetch(ctx context.Context) (Usage, error) {
 						continue
 					}
 
+					// No mutex needed here, only one goroutine in writing this integer.
+					usage.ActiveRepos++
+
 					repo := repo
 
 					group.Go(func() error {
@@ -115,7 +121,7 @@ func (f *OrgUsageFetcher) Fetch(ctx context.Context) (Usage, error) {
 										}
 
 										usageMu.Lock()
-										usage = append(usage, result)
+										usage.Workflows = append(usage.Workflows, result)
 										usageMu.Unlock()
 
 										f.logger.Debug(
@@ -146,7 +152,7 @@ func (f *OrgUsageFetcher) Fetch(ctx context.Context) (Usage, error) {
 		)
 	})
 
-	return usage, group.Wait()
+	return &usage, group.Wait()
 }
 
 func scanAllRepoWorkflows(ctx context.Context, org, repo string, workflowClient *github.ActionsService, cb func(*github.Workflows)) error {

cmd/exporter/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"net/http"
 	"os"
@@ -109,7 +110,7 @@ func run() int {
 
 	}()
 
-	if err := srv.ListenAndServe(); err != nil {
+	if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
 		logger.Error(
 			"Could not listen over HTTP",
 			zap.Error(err),

cmd/print/main.go

@@ -65,11 +65,13 @@ func run() int {
 		return 1
 	}
 
-	sort.Slice(usage, func(i, j int) bool {
-		return usage[i].Repo < usage[j].Repo
+	sort.Slice(usage.Workflows, func(i, j int) bool {
+		return usage.Workflows[i].Repo < usage.Workflows[j].Repo
 	})
 
-	for _, workflowUsage := range usage {
+	logger.Info("Reporting stats", zap.Int64("active repos", usage.ActiveRepos))
+
+	for _, workflowUsage := range usage.Workflows {
 		logger.Info(
 			"Got usage stats",
 			zap.String("owner", workflowUsage.Owner),

go.mod

@@ -33,3 +33,6 @@ require (
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+// Until https://github.com/gofri/go-github-ratelimit/pull/18 lands.
+replace github.com/gofri/go-github-ratelimit => github.com/zendesk-piotrpawluk/go-github-ratelimit v0.0.0-20231120163947-01b70bdcdf9a

go.sum

@@ -7,8 +7,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/gofri/go-github-ratelimit v1.0.5 h1:j+AS0Jh5baasOTLkWprpuEsDSuz6bAyE/HuoGH1JrZ4=
-github.com/gofri/go-github-ratelimit v1.0.5/go.mod h1:OnCi5gV+hAG/LMR7llGhU7yHt44se9sYgKPnafoL7RY=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
@@ -49,6 +47,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zendesk-piotrpawluk/go-github-ratelimit v0.0.0-20231120163947-01b70bdcdf9a h1:3oCnep3MIn2PwdtHHUMwc4toJmk20U7X2sOs9PWDpIA=
+github.com/zendesk-piotrpawluk/go-github-ratelimit v0.0.0-20231120163947-01b70bdcdf9a/go.mod h1:OnCi5gV+hAG/LMR7llGhU7yHt44se9sYgKPnafoL7RY=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=