-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
CLI | Add parallel GetSastMetadetaByID functionality (AST-40381) (#737)
* Add Pagination req * Add Pagination req * Add GetSastMetadataByIDsInParallel func in sast-metadata-http.go * revert pagination * add sastMetaDataInParallel func to mock * change signature of func in mock * Add integration test * encapsulate getSastMetadata function * revert changes in util command * added integration test * Update internal/wrappers/sast-metadata.go Co-authored-by: tamarleviCm <[email protected]> * Added semaphore to limit go routines * Added semaphore to limit go routines * add sast metadata service * change sast metadata mock * add unit tests * add unit tests * add unit tests * add unit tests * add unit tests * remove sorting response functionality * remove sorting response functionality * add service name explanation * change to semaphore pkg * change to semaphore pkg * change to semaphore pkg * add missing dependencies * re trigger pipeline * check * check * skip failing tests * revert skipping --------- Co-authored-by: AlvoBen <[email protected]> Co-authored-by: tamarleviCm <[email protected]> Co-authored-by: Or Shamir Checkmarx <[email protected]>
- Loading branch information
4 people
authored
Jun 18, 2024
1 parent
2e7141a
commit d78cef0
Showing
8 changed files
with
415 additions
and
205 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| package services | ||
|
|
||
| import ( | ||
| "context" | ||
| "strings" | ||
| "sync" | ||
|
|
||
| commonParams "github.com/checkmarx/ast-cli/internal/params" | ||
| "github.com/checkmarx/ast-cli/internal/wrappers" | ||
| "golang.org/x/sync/semaphore" | ||
| ) | ||
|
|
||
| const ( | ||
| BatchSize = 200 | ||
| ) | ||
|
|
||
| func GetSastMetadataByIDs(sastMetaDataWrapper wrappers.SastMetadataWrapper, scanIDs []string) (*wrappers.SastMetadataModel, error) { | ||
| totalBatches := (len(scanIDs) + BatchSize - 1) / BatchSize | ||
| maxConcurrentGoRoutines := 10 | ||
| sem := semaphore.NewWeighted(int64(maxConcurrentGoRoutines)) | ||
|
|
||
| var wg sync.WaitGroup | ||
| results := make(chan wrappers.SastMetadataModel, totalBatches) | ||
| errors := make(chan error, totalBatches) | ||
| ctx := context.Background() | ||
|
|
||
| for i := 0; i < totalBatches; i++ { | ||
| start := i * BatchSize | ||
| end := start + BatchSize | ||
| if end > len(scanIDs) { | ||
| end = len(scanIDs) | ||
| } | ||
|
|
||
| batchParams := map[string]string{ | ||
| commonParams.ScanIDsQueryParam: strings.Join(scanIDs[start:end], ","), | ||
| } | ||
|
|
||
| wg.Add(1) | ||
| err := sem.Acquire(ctx, 1) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| go func() { | ||
| defer wg.Done() | ||
| defer sem.Release(1) | ||
|
|
||
| result, err := sastMetaDataWrapper.GetSastMetadataByIDs(batchParams) | ||
| if err != nil { | ||
| errors <- err | ||
| return | ||
| } | ||
| results <- *result | ||
| }() | ||
| } | ||
|
|
||
| go func() { | ||
| wg.Wait() | ||
| close(results) | ||
| close(errors) | ||
| }() | ||
|
|
||
| if len(errors) > 0 { | ||
| return nil, <-errors | ||
| } | ||
|
|
||
| var models []wrappers.SastMetadataModel | ||
| for result := range results { | ||
| models = append(models, result) | ||
| } | ||
| return makeSastMetadataModelFromResults(models), nil | ||
| } | ||
|
|
||
| func makeSastMetadataModelFromResults(results []wrappers.SastMetadataModel) *wrappers.SastMetadataModel { | ||
| finalResult := &wrappers.SastMetadataModel{} | ||
| for _, result := range results { | ||
| finalResult.TotalCount += result.TotalCount | ||
| finalResult.Scans = append(finalResult.Scans, result.Scans...) | ||
| finalResult.Missing = append(finalResult.Missing, result.Missing...) | ||
| } | ||
| return finalResult | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| package services | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "strconv" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/checkmarx/ast-cli/internal/wrappers" | ||
| "github.com/checkmarx/ast-cli/internal/wrappers/mock" | ||
| "github.com/stretchr/testify/assert" //nolint:depguard | ||
| ) | ||
|
|
||
| func TestGetSastMetadataByIDs(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| scanIDs []string | ||
| }{ | ||
| { | ||
| name: "Multiple batches", | ||
| scanIDs: createScanIDs(5000), | ||
| }, | ||
| { | ||
| name: "Single batch", | ||
| scanIDs: createScanIDs(100), | ||
| }, | ||
| { | ||
| name: "Empty slice", | ||
| scanIDs: []string{}, | ||
| }, | ||
| { | ||
| name: "Multiple batches with partial last batch", | ||
| scanIDs: createScanIDs(893), | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| tt := tt | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| wrapper := &mock.SastMetadataMockWrapper{} | ||
| actual, err := GetSastMetadataByIDs(wrapper, tt.scanIDs) | ||
| if err != nil { | ||
| t.Errorf("GetSastMetadataByIDs(%v) returned an error: %v", tt.scanIDs, err) | ||
| } | ||
| assert.True(t, checkAllScanExists(actual.Scans, "ConcurrentTest", len(tt.scanIDs))) | ||
| assert.Equal(t, len(tt.scanIDs), len(actual.Scans)) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func createScanIDs(count int) []string { | ||
| scanIDs := make([]string, count) | ||
| for i := 0; i < count; i++ { | ||
| scanIDs[i] = fmt.Sprintf("ConcurrentTest%d", i) | ||
| } | ||
| return scanIDs | ||
| } | ||
|
|
||
| func checkAllScanExists(scans []wrappers.Scans, prefix string, count int) bool { | ||
| existingScans := make(map[int]bool) | ||
|
|
||
| for i := range scans { | ||
| scan := &scans[i] | ||
| if strings.HasPrefix(scan.ScanID, prefix) { | ||
| scanNumberStr := scan.ScanID[len(prefix):] | ||
| if scanNumber, err := strconv.Atoi(scanNumberStr); err == nil { | ||
| existingScans[scanNumber] = true | ||
| } | ||
| } | ||
| } | ||
|
|
||
| for i := 0; i < count; i++ { | ||
| if !existingScans[i] { | ||
| return false | ||
| } | ||
| } | ||
| return true | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.