diff --git a/subsystem/inventory/storage/diskv/diskv.go b/subsystem/inventory/storage/diskv/diskv.go index 1f79a93..9e6a7ea 100644 --- a/subsystem/inventory/storage/diskv/diskv.go +++ b/subsystem/inventory/storage/diskv/diskv.go @@ -1,87 +1,27 @@ -// Package diskv implements a diskv-backed inventory subsystem storage backend. +// Package diskv implements an inventory subsystem backend using diskv. package diskv import ( - "context" - "encoding/json" - "fmt" "path/filepath" - "github.com/micromdm/nanocmd/subsystem/inventory/storage" + "github.com/micromdm/nanocmd/subsystem/inventory/storage/kv" + + "github.com/micromdm/nanolib/storage/kv/kvdiskv" "github.com/peterbourgon/diskv/v3" ) -// Diskv is an on-disk enrollment inventory data store. +// Diskv is an inventory subsystem backend which uses diskv as the key-value store. type Diskv struct { - diskv *diskv.Diskv + *kv.KV } -// New creates a new initialized inventory data store. +// New creates a new profile store at on disk at path. func New(path string) *Diskv { - flatTransform := func(s string) []string { return []string{} } return &Diskv{ - diskv: diskv.New(diskv.Options{ + KV: kv.New(kvdiskv.New(diskv.New(diskv.Options{ BasePath: filepath.Join(path, "inventory"), - Transform: flatTransform, + Transform: kvdiskv.FlatTransform, CacheSizeMax: 1024 * 1024, - }), - } -} - -// RetrieveInventory retrieves the inventory data for enrollment IDs. -func (s *Diskv) RetrieveInventory(ctx context.Context, opt *storage.SearchOptions) (map[string]storage.Values, error) { - ret := make(map[string]storage.Values) - for _, id := range opt.IDs { - if !s.diskv.Has(id) { - continue - } - raw, err := s.diskv.Read(id) - if err != nil { - return ret, fmt.Errorf("reading values for %s: %w", id, err) - } - var vals storage.Values - if err = json.Unmarshal(raw, &vals); err != nil { - return ret, fmt.Errorf("unmarshal values for %s: %w", id, err) - } - ret[id] = vals - } - return ret, nil -} - -// StoreInventoryValues stores inventory data about the specified ID. -func (s *Diskv) StoreInventoryValues(ctx context.Context, id string, values storage.Values) error { - var err error - var raw []byte - var vals storage.Values - if s.diskv.Has(id) { - // this is likely race-prone as we perform a read-process-write on the same key. - if raw, err = s.diskv.Read(id); err != nil { - return fmt.Errorf("reading values: %w", err) - } - if len(raw) > 0 { - if err = json.Unmarshal(raw, &vals); err != nil { - return fmt.Errorf("unmarshal values: %w", err) - } - if vals != nil { - for k := range values { - vals[k] = values[k] - } - } - } - } - if vals == nil { - vals = values - } - if raw, err = json.Marshal(vals); err != nil { - return fmt.Errorf("marshal values: %w", err) + }))), } - if err = s.diskv.Write(id, raw); err != nil { - return fmt.Errorf("write values: %w", err) - } - return nil -} - -// DeleteInventory deletes all inventory data for an enrollment ID. -func (s *Diskv) DeleteInventory(ctx context.Context, id string) error { - return s.diskv.Erase(id) } diff --git a/subsystem/inventory/storage/diskv/diskv_test.go b/subsystem/inventory/storage/diskv/diskv_test.go index 263b480..0817aa3 100644 --- a/subsystem/inventory/storage/diskv/diskv_test.go +++ b/subsystem/inventory/storage/diskv/diskv_test.go @@ -1,7 +1,6 @@ package diskv import ( - "os" "testing" "github.com/micromdm/nanocmd/subsystem/inventory/storage" @@ -9,6 +8,5 @@ import ( ) func TestDiskv(t *testing.T) { - test.TestStorage(t, func() storage.Storage { return New("teststor") }) - os.RemoveAll("teststor") + test.TestStorage(t, func() storage.Storage { return New(t.TempDir()) }) } diff --git a/subsystem/inventory/storage/inmem/inmem.go b/subsystem/inventory/storage/inmem/inmem.go index 605682e..e8412f5 100644 --- a/subsystem/inventory/storage/inmem/inmem.go +++ b/subsystem/inventory/storage/inmem/inmem.go @@ -2,60 +2,17 @@ package inmem import ( - "context" - "sync" + "github.com/micromdm/nanocmd/subsystem/inventory/storage/kv" - "github.com/micromdm/nanocmd/subsystem/inventory/storage" + "github.com/micromdm/nanolib/storage/kv/kvmap" ) -// InMem represents the in-memory enrollment inventory data store. +// InMem is an in-memory inventory subsystem storage system backend. type InMem struct { - mu sync.RWMutex - inv map[string]storage.Values + *kv.KV } -// New creates a new initialized inventory data store. +// New creates a new inventory subsystem storage system backend. func New() *InMem { - return &InMem{inv: make(map[string]storage.Values)} -} - -// RetrieveInventory retrieves the inventory data for enrollment IDs. -func (s *InMem) RetrieveInventory(ctx context.Context, opt *storage.SearchOptions) (map[string]storage.Values, error) { - s.mu.RLock() - defer s.mu.RUnlock() - if opt == nil || len(opt.IDs) <= 0 { - return nil, nil - } - ret := make(map[string]storage.Values) - for _, id := range opt.IDs { - if vals, ok := s.inv[id]; ok { - ret[id] = make(storage.Values) - for k, v := range vals { - ret[id][k] = v - } - } - } - return ret, nil -} - -// StoreInventoryValues stores inventory data about the specified ID. -func (s *InMem) StoreInventoryValues(ctx context.Context, id string, values storage.Values) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.inv[id] == nil { - s.inv[id] = values - } else { - for k, v := range values { - s.inv[id][k] = v - } - } - return nil -} - -// DeleteInventory deletes all inventory data for an enrollment ID. -func (s *InMem) DeleteInventory(ctx context.Context, id string) error { - s.mu.Lock() - defer s.mu.Unlock() - delete(s.inv, id) - return nil + return &InMem{KV: kv.New(kvmap.New())} } diff --git a/subsystem/inventory/storage/kv/kv.go b/subsystem/inventory/storage/kv/kv.go new file mode 100644 index 0000000..6b56dc3 --- /dev/null +++ b/subsystem/inventory/storage/kv/kv.go @@ -0,0 +1,101 @@ +// Package kv implements an inventory subsystem storage backend using a key-value store. +package kv + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + + "github.com/micromdm/nanocmd/subsystem/inventory/storage" + + "github.com/micromdm/nanolib/storage/kv" +) + +// KV is an inventory subsystem storage backend using a key-value store. +type KV struct { + mu sync.RWMutex + b kv.KeysPrefixTraversingBucket +} + +// New creates a new inventory subsystem backend. +func New(b kv.KeysPrefixTraversingBucket) *KV { + return &KV{b: b} +} + +// RetrieveInventory queries and returns the inventory values by mapped +// by enrollment ID from the key-value store. Must provide opt and IDs. +func (s *KV) RetrieveInventory(ctx context.Context, opt *storage.SearchOptions) (map[string]storage.Values, error) { + if opt == nil || len(opt.IDs) < 1 { + return nil, storage.ErrNoIDs + } + + s.mu.RLock() + defer s.mu.RUnlock() + + r := make(map[string]storage.Values) + for _, id := range opt.IDs { + jsonValues, err := s.b.Get(ctx, id) + if errors.Is(err, kv.ErrKeyNotFound) { + continue + } else if err != nil { + return r, fmt.Errorf("getting values for %s: %w", id, err) + } + + var values storage.Values + if err = json.Unmarshal(jsonValues, &values); err != nil { + return r, fmt.Errorf("unmarshal values for %s: %w", id, err) + } + r[id] = values + } + return r, nil +} + +// StoreInventoryValues stores inventory data about the specified ID. +func (s *KV) StoreInventoryValues(ctx context.Context, id string, newValues storage.Values) error { + if id == "" { + return storage.ErrNoIDs + } + if len(newValues) == 0 { + return nil + } + + s.mu.Lock() + defer s.mu.Unlock() + + jsonValues, err := s.b.Get(ctx, id) + if err != nil && !errors.Is(err, kv.ErrKeyNotFound) { + return fmt.Errorf("get values: %w", err) + } + + var values storage.Values + if len(jsonValues) < 1 { + values = newValues + } else { + // load existing values + if err = json.Unmarshal(jsonValues, &values); err != nil { + return fmt.Errorf("unmarshal values: %w", err) + } + + // merge the new values in + for k := range newValues { + values[k] = newValues[k] + } + } + + if jsonValues, err = json.Marshal(&values); err != nil { + return fmt.Errorf("marshal values: %w", err) + } + + if err = s.b.Set(ctx, id, jsonValues); err != nil { + return fmt.Errorf("set values: %w", err) + } + + return nil +} + +// DeleteInventory deletes all inventory data for an enrollment ID. +func (s *KV) DeleteInventory(ctx context.Context, id string) error { + return s.b.Delete(ctx, id) +} diff --git a/subsystem/inventory/storage/storage.go b/subsystem/inventory/storage/storage.go index 211b1be..9327613 100644 --- a/subsystem/inventory/storage/storage.go +++ b/subsystem/inventory/storage/storage.go @@ -3,6 +3,11 @@ package storage import ( "context" + "errors" +) + +var ( + ErrNoIDs = errors.New("no ids supplied") ) // SearchOptions is a basic query for inventory of enrollment IDs. @@ -15,11 +20,18 @@ type Values map[string]interface{} type ReadStorage interface { // RetrieveInventory queries and returns the inventory values by mapped by enrollment ID. + // If no search opt nor IDs are provided an ErrNoIDs should be returned. + // If IDs are have no inventory data then they should be skipped and + // omitted from the output with no error. RetrieveInventory(ctx context.Context, opt *SearchOptions) (map[string]Values, error) } type Storage interface { ReadStorage + + // StoreInventoryValues stores inventory data about the specified ID. StoreInventoryValues(ctx context.Context, id string, values Values) error + + // DeleteInventory deletes all inventory data for an enrollment ID. DeleteInventory(ctx context.Context, id string) error } diff --git a/subsystem/inventory/storage/test/test.go b/subsystem/inventory/storage/test/test.go index 053a5da..c7cf4d3 100644 --- a/subsystem/inventory/storage/test/test.go +++ b/subsystem/inventory/storage/test/test.go @@ -57,6 +57,6 @@ func TestStorage(t *testing.T, newStorage func() storage.Storage) { _, ok = idVals[id] if ok { - t.Error("expected id to missing in id values map") + t.Error("expected id to be missing in id values map") } } diff --git a/subsystem/profile/storage/diskv/diskv.go b/subsystem/profile/storage/diskv/diskv.go index bc3301d..a257928 100644 --- a/subsystem/profile/storage/diskv/diskv.go +++ b/subsystem/profile/storage/diskv/diskv.go @@ -1,107 +1,27 @@ -// Package inmem implements a storage backend for the Profile subsystem backed by diskv. +// Package diskv implements a storage backend for the Profile subsystem backed by diskv. package diskv import ( - "context" - "fmt" "path/filepath" - "strings" - "github.com/micromdm/nanocmd/subsystem/profile/storage" + "github.com/micromdm/nanocmd/subsystem/profile/storage/kv" + + "github.com/micromdm/nanolib/storage/kv/kvdiskv" "github.com/peterbourgon/diskv/v3" ) -// Diskv is a storage backend for the Profile subsystem backed by diskv. +// Diskv is a profile storage backend that uses an on-disk key-valye store. type Diskv struct { - diskv *diskv.Diskv + *kv.KV } -// New creates a new initialized profile data store. +// New creates a new profile store at on disk at path. func New(path string) *Diskv { - flatTransform := func(s string) []string { return []string{} } return &Diskv{ - diskv: diskv.New(diskv.Options{ + KV: kv.New(kvdiskv.New(diskv.New(diskv.Options{ BasePath: filepath.Join(path, "profile"), - Transform: flatTransform, + Transform: kvdiskv.FlatTransform, CacheSizeMax: 1024 * 1024, - }), - } -} - -// RetrieveProfileInfos implements the storage interface. -func (s *Diskv) RetrieveProfileInfos(ctx context.Context, names []string) (map[string]storage.ProfileInfo, error) { - if len(names) < 1 { - for name := range s.diskv.Keys(nil) { - if strings.HasSuffix(name, ".identifier") { - names = append(names, name[:len(name)-11]) - } - } - } - ret := make(map[string]storage.ProfileInfo) - for _, name := range names { - if !s.diskv.Has(name + ".identifier") { - return ret, fmt.Errorf("profile not found for %s: %w", name, storage.ErrProfileNotFound) - } - idBytes, err := s.diskv.Read(name + ".identifier") - if err != nil { - return ret, fmt.Errorf("reading identifier for %s: %w", name, err) - } - uuidBytes, err := s.diskv.Read(name + ".uuid") - if err != nil { - return ret, fmt.Errorf("reading uuid for %s: %w", name, err) - } - ret[name] = storage.ProfileInfo{ - Identifier: string(idBytes), - UUID: string(uuidBytes), - } - } - return ret, nil -} - -// RetrieveRawProfiles implements the storage interface. -func (s *Diskv) RetrieveRawProfiles(ctx context.Context, names []string) (map[string][]byte, error) { - if len(names) < 1 { - return nil, storage.ErrNoNames - } - ret := make(map[string][]byte) - for _, name := range names { - if !s.diskv.Has(name + ".raw") { - continue - } - var err error - if ret[name], err = s.diskv.Read(name + ".raw"); err != nil { - return ret, fmt.Errorf("reading raw for %s: %w", name, err) - } - } - return ret, nil -} - -// StoreProfile implements the storage interface. -func (s *Diskv) StoreProfile(ctx context.Context, name string, info storage.ProfileInfo, raw []byte) error { - err := s.diskv.Write(name+".raw", raw) - if err != nil { - return fmt.Errorf("writing raw: %w", err) - } - if err = s.diskv.Write(name+".identifier", []byte(info.Identifier)); err != nil { - return fmt.Errorf("writing identifier: %w", err) - } - if err = s.diskv.Write(name+".uuid", []byte(info.UUID)); err != nil { - return fmt.Errorf("writing uuid: %w", err) - } - return nil -} - -// DeleteProfile implements the storage interface. -func (s *Diskv) DeleteProfile(ctx context.Context, name string) error { - err := s.diskv.Erase(name + ".identifier") - if err != nil { - return fmt.Errorf("delete identifier for %s: %w", name, err) - } - if err := s.diskv.Erase(name + ".uuid"); err != nil { - return fmt.Errorf("delete uuid for %s: %w", name, err) - } - if err := s.diskv.Erase(name + ".raw"); err != nil { - return fmt.Errorf("delete raw for %s: %w", name, err) + }))), } - return nil } diff --git a/subsystem/profile/storage/diskv/diskv_test.go b/subsystem/profile/storage/diskv/diskv_test.go index 13e06c3..3604021 100644 --- a/subsystem/profile/storage/diskv/diskv_test.go +++ b/subsystem/profile/storage/diskv/diskv_test.go @@ -1,7 +1,6 @@ package diskv import ( - "os" "testing" "github.com/micromdm/nanocmd/subsystem/profile/storage" @@ -9,6 +8,5 @@ import ( ) func TestDiskv(t *testing.T) { - test.TestProfileStorage(t, func() storage.Storage { return New("teststor") }) - os.RemoveAll("teststor") + test.TestProfileStorage(t, func() storage.Storage { return New(t.TempDir()) }) } diff --git a/subsystem/profile/storage/inmem/inmem.go b/subsystem/profile/storage/inmem/inmem.go index a79bf5b..ebd5f09 100644 --- a/subsystem/profile/storage/inmem/inmem.go +++ b/subsystem/profile/storage/inmem/inmem.go @@ -2,83 +2,16 @@ package inmem import ( - "context" - "fmt" - "sync" + "github.com/micromdm/nanocmd/subsystem/profile/storage/kv" - "github.com/micromdm/nanocmd/subsystem/profile/storage" + "github.com/micromdm/nanolib/storage/kv/kvmap" ) -type profile struct { - info storage.ProfileInfo - raw []byte -} - -// InMem is an in-memory storage backend for the Profile subsystem. +// InMem is a profile storage backend using an in-memory key-valye store. type InMem struct { - m sync.RWMutex - p map[string]profile + *kv.KV } func New() *InMem { - return &InMem{p: make(map[string]profile)} -} - -// RetrieveProfileInfos implements the storage interface. -func (s *InMem) RetrieveProfileInfos(ctx context.Context, names []string) (map[string]storage.ProfileInfo, error) { - s.m.RLock() - defer s.m.RUnlock() - if len(names) < 1 { - names = make([]string, 0, len(s.p)) - for key := range s.p { - names = append(names, key) - } - } - ret := make(map[string]storage.ProfileInfo) - for _, name := range names { - profile, ok := s.p[name] - if !ok { - return ret, fmt.Errorf("%w: %s", storage.ErrProfileNotFound, name) - } - ret[name] = profile.info - } - return ret, nil -} - -// RetrieveRawProfiles implements the storage interface. -func (s *InMem) RetrieveRawProfiles(ctx context.Context, names []string) (map[string][]byte, error) { - if len(names) < 1 { - return nil, storage.ErrNoNames - } - s.m.RLock() - defer s.m.RUnlock() - ret := make(map[string][]byte) - for _, name := range names { - profile, ok := s.p[name] - if !ok { - return ret, fmt.Errorf("%w: %s", storage.ErrProfileNotFound, name) - } - ret[name] = profile.raw - } - return ret, nil -} - -// StoreProfile implements the storage interface. -func (s *InMem) StoreProfile(ctx context.Context, name string, info storage.ProfileInfo, raw []byte) error { - s.m.Lock() - defer s.m.Unlock() - s.p[name] = profile{info: info, raw: raw} - return nil -} - -// DeleteProfile implements the storage interface. -func (s *InMem) DeleteProfile(ctx context.Context, name string) error { - s.m.Lock() - defer s.m.Unlock() - _, ok := s.p[name] - if !ok { - return storage.ErrProfileNotFound - } - delete(s.p, name) - return nil + return &InMem{KV: kv.New(kvmap.New())} } diff --git a/subsystem/profile/storage/kv/kv.go b/subsystem/profile/storage/kv/kv.go new file mode 100644 index 0000000..6f807bf --- /dev/null +++ b/subsystem/profile/storage/kv/kv.go @@ -0,0 +1,96 @@ +// Package kv implements a profile storage backend using a key-value store. +package kv + +import ( + "context" + "errors" + "fmt" + + "github.com/micromdm/nanocmd/subsystem/profile/storage" + + "github.com/micromdm/nanolib/storage/kv" +) + +const ( + keyPfxUUID = "uuid." + keyPfxID = "id." + keyPfxRaw = "raw." +) + +// KV is a profile storage backend using a key-value store. +type KV struct { + b kv.KeysPrefixTraversingBucket +} + +func New(b kv.KeysPrefixTraversingBucket) *KV { + return &KV{b: b} +} + +// RetrieveProfileInfos returns the profile metadata in the key-value store by name. +// Will return all keys. +func (s *KV) RetrieveProfileInfos(ctx context.Context, names []string) (map[string]storage.ProfileInfo, error) { + if len(names) < 1 { + for k := range s.b.KeysPrefix(ctx, keyPfxID, nil) { + names = append(names, k[len(keyPfxID):]) + } + } + + r := make(map[string]storage.ProfileInfo) + for _, name := range names { + id, err := s.b.Get(ctx, keyPfxID+name) + if errors.Is(err, kv.ErrKeyNotFound) { + return r, fmt.Errorf("%w: %s: %v", storage.ErrProfileNotFound, name, err) + } else if err != nil { + return r, err + } + + uuid, err := s.b.Get(ctx, keyPfxUUID+name) + if errors.Is(err, kv.ErrKeyNotFound) { + return r, fmt.Errorf("%w: %s: %v", storage.ErrProfileNotFound, name, err) + } else if err != nil { + return r, err + } + + r[name] = storage.ProfileInfo{ + Identifier: string(id), + UUID: string(uuid), + } + } + return r, nil +} + +// RetrieveRawProfiles returns the raw profile bytes in the key-value store by name. +func (s *KV) RetrieveRawProfiles(ctx context.Context, names []string) (map[string][]byte, error) { + if len(names) < 1 { + return nil, storage.ErrNoNames + } + r := make(map[string][]byte) + for _, name := range names { + profile, err := s.b.Get(ctx, keyPfxRaw+name) + if errors.Is(err, kv.ErrKeyNotFound) { + return r, fmt.Errorf("%w: %s: %v", storage.ErrProfileNotFound, name, err) + } else if err != nil { + return r, err + } + r[name] = profile + } + return r, nil +} + +// StoreProfile stores a raw profile and associated info in the key-value store by name. +func (s *KV) StoreProfile(ctx context.Context, name string, info storage.ProfileInfo, raw []byte) error { + return kv.SetMap(ctx, s.b, map[string][]byte{ + keyPfxID + name: []byte(info.Identifier), + keyPfxUUID + name: []byte(info.UUID), + keyPfxRaw + name: raw, + }) +} + +// DeleteProfile deletes a profile from the key-value store by name. +func (s *KV) DeleteProfile(ctx context.Context, name string) error { + return kv.DeleteSlice(ctx, s.b, []string{ + keyPfxID + name, + keyPfxUUID + name, + keyPfxRaw + name, + }) +} diff --git a/subsystem/profile/storage/storage.go b/subsystem/profile/storage/storage.go index d21d5c6..122eece 100644 --- a/subsystem/profile/storage/storage.go +++ b/subsystem/profile/storage/storage.go @@ -31,12 +31,12 @@ type ReadStorage interface { // RetrieveProfileInfos returns the profile metadata by name. // Implementations have the choice to return all profile metadata if // no names were provided or not. ErrProfileNotFound is returned for - // a name that hasn't been stored. + // any name that hasn't been stored. RetrieveProfileInfos(ctx context.Context, names []string) (map[string]ProfileInfo, error) // RetrieveRawProfiles returns the raw profile bytes by name. // Implementations should not return all profiles if no names were provided. - // ErrProfileNotFound is returned for a name that hasn't been stored. + // ErrProfileNotFound is returned for any name that hasn't been stored. // ErrNoNames is returned if names is empty. RetrieveRawProfiles(ctx context.Context, names []string) (map[string][]byte, error) } @@ -49,7 +49,7 @@ type Storage interface { // and matches the raw profile bytes. StoreProfile(ctx context.Context, name string, info ProfileInfo, raw []byte) error - // DeleteProfile deletes a profile from profile storage. + // DeleteProfile deletes a profile from profile storage by name. // ErrProfileNotFound is returned for a name that hasn't been stored. DeleteProfile(ctx context.Context, name string) error }