Add flag to enable fully consistent mode for read requests

Signed-off-by: Jonathan Marcantonio <[email protected]>

configs/config.yaml

@@ -17,3 +17,4 @@ data:
     token: "${PRESHARED}" # token takes precedence over tokenFile
     tokenFile: "${PRESHARED_FILE:.secrets/local-spicedb-secret}"
     schemaFile: "${SCHEMA_FILE:deploy/schema.zed}"
+    fullyConsistent: false

deploy/kessel-relations.yaml

@@ -34,6 +34,7 @@ objects:
             token: "${PRESHARED}" # token takes precedence over tokenFile
             tokenFile: "${PRESHARED_FILE:.secrets/local-spicedb-secret}"
             schemaFile: "${SCHEMA_FILE:deploy/schema.zed}"
+            fullyConsistent: false
   - apiVersion: v1
     kind: Secret
     metadata:

internal/conf/conf.proto

@@ -40,6 +40,7 @@ message Data {
     string token = 3;
     string tokenFile = 4;
     string schemaFile = 5;
+    bool fullyConsistent = 6;
   }
   SpiceDb spiceDb = 1;
 }

internal/data/LocalSpiceDbContainer.go

@@ -33,6 +33,9 @@ const (
 	SpicedbSchemaBootstrapFile = "spicedb-test-data/basic_schema.zed"
 	// SpicedbRelationsBootstrapFile specifies an optional bootstrap file containing relations to be used for testing
 	SpicedbRelationsBootstrapFile = ""
+	// FullyConsistent specifices the consistency mode used for our read API calls
+	// may experience different results between tests and manual probing if the values differ
+	FullyConsistent = true // Should probably be inline with our config file. (TODO: Can we make our tests grab the same value?)
 )
 
 // LocalSpiceDbContainer struct that holds pointers to the container, dockertest pool and exposes the port
@@ -136,7 +139,9 @@ func (l *LocalSpiceDbContainer) NewToken() (string, error) {
 
 // WaitForQuantizationInterval needed to avoid read-before-write when loading the schema
 func (l *LocalSpiceDbContainer) WaitForQuantizationInterval() {
-	time.Sleep(10 * time.Millisecond)
+	if !FullyConsistent {
+		time.Sleep(10 * time.Millisecond)
+	}
 }
 
 // CreateClient creates a new client that connects to the dockerized spicedb instance and the right store
@@ -162,10 +167,11 @@ func (l *LocalSpiceDbContainer) CreateSpiceDbRepository() (*SpiceDbRepository, e
 	defer os.RemoveAll(tmpDir)
 
 	spiceDbConf := &conf.Data_SpiceDb{
-		UseTLS:     false,
-		Endpoint:   "localhost:" + l.port,
-		Token:      tmpFile.Name(),
-		SchemaFile: l.schemaLocation,
+		UseTLS:          false,
+		Endpoint:        "localhost:" + l.port,
+		Token:           tmpFile.Name(),
+		SchemaFile:      l.schemaLocation,
+		FullyConsistent: FullyConsistent, // Should be inline with our config file
 	}
 	repo, _, err := NewSpiceDbRepository(&conf.Data{SpiceDb: spiceDbConf}, l.logger)
 	if err != nil {

internal/data/spicedb.go

@@ -35,6 +35,12 @@ const (
 	relationPrefix = "t_"
 )
 
+var (
+	// Default consistency for read APIs is minimize_latency
+	// will attempt to minimize the latency of the API call by selecting data that is most likely exist in the cache.
+	consistency = &v1.Consistency{Requirement: &v1.Consistency_MinimizeLatency{MinimizeLatency: true}}
+)
+
 // NewSpiceDbRepository .
 func NewSpiceDbRepository(c *conf.Data, logger log.Logger) (*SpiceDbRepository, func(), error) {
 	log.NewHelper(logger).Info("creating spicedb connection")
@@ -90,6 +96,11 @@ func NewSpiceDbRepository(c *conf.Data, logger log.Logger) (*SpiceDbRepository,
 		log.NewHelper(logger).Info("spicedb connection cleanup requested (nothing to clean up)")
 	}
 
+	if c.SpiceDb.FullyConsistent {
+		// will ensure that all data used is fully consistent with the latest data available within the SpiceDB datastore.
+		consistency = &v1.Consistency{Requirement: &v1.Consistency_FullyConsistent{FullyConsistent: true}}
+	}
+
 	return &SpiceDbRepository{client, healthClient, c.SpiceDb.SchemaFile, false}, cleanup, nil
 }
 
@@ -128,6 +139,7 @@ func (s *SpiceDbRepository) LookupSubjects(ctx context.Context, subject_type *ap
 	}
 
 	req := &v1.LookupSubjectsRequest{
+		Consistency: consistency,
 		Resource: &v1.ObjectReference{
 			ObjectType: kesselTypeToSpiceDBType(object.Type),
 			ObjectId:   object.Id,
@@ -194,6 +206,7 @@ func (s *SpiceDbRepository) LookupResources(ctx context.Context, resouce_type *a
 		}
 	}
 	client, err := s.client.LookupResources(ctx, &v1.LookupResourcesRequest{
+		Consistency:        consistency,
 		ResourceObjectType: kesselTypeToSpiceDBType(resouce_type),
 		Permission:         relation,
 		Subject: &v1.SubjectReference{
@@ -347,6 +360,7 @@ func (s *SpiceDbRepository) ReadRelationships(ctx context.Context, filter *apiV1
 	}
 
 	req := &v1.ReadRelationshipsRequest{
+		Consistency:        consistency,
 		RelationshipFilter: relationshipFilter,
 		OptionalLimit:      limit,
 		OptionalCursor:     cursor,
@@ -448,9 +462,10 @@ func (s *SpiceDbRepository) Check(ctx context.Context, check *apiV1beta1.CheckRe
 		ObjectId:   check.GetResource().GetId(),
 	}
 	req := &v1.CheckPermissionRequest{
-		Resource:   resource,
-		Permission: check.GetRelation(),
-		Subject:    subject,
+		Consistency: consistency,
+		Resource:    resource,
+		Permission:  check.GetRelation(),
+		Subject:     subject,
 	}
 	checkResponse, err := s.client.CheckPermission(ctx, req)
 	if err != nil {