From aaa62e89092da25885168c5e8eb2bd9568741e29 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 13 Oct 2023 12:59:58 +0600 Subject: [PATCH 01/11] feat(pub): add graph support --- .../analyzer/language/dart/pub/pubspec.go | 158 +++++++++++++++++- .../language/dart/pub/pubspec_test.go | 158 ++++++++++++++---- .../{broken.lock => broken/pubspec.lock} | 0 .../{empty.lock => empty/pubspec.lock} | 0 .../language/dart/pub/testdata/happy.lock | 25 --- .../pub.dev/collection-1.18.0/pubspec.yaml | 16 ++ .../hosted/pub.dev/crypto-3.0.3/pubspec.yaml | 17 ++ .../hosted/pub.dev/meta-1.11.0/pubspec.yaml | 17 ++ .../pub.dev/typed_data-1.3.2/pubspec.yaml | 18 ++ .../dart/pub/testdata/happy/pubspec.lock | 37 ++++ 10 files changed, 376 insertions(+), 70 deletions(-) rename pkg/fanal/analyzer/language/dart/pub/testdata/{broken.lock => broken/pubspec.lock} (100%) rename pkg/fanal/analyzer/language/dart/pub/testdata/{empty.lock => empty/pubspec.lock} (100%) delete mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy.lock create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/collection-1.18.0/pubspec.yaml create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/crypto-3.0.3/pubspec.yaml create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/meta-1.11.0/pubspec.yaml create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/typed_data-1.3.2/pubspec.yaml create mode 100644 pkg/fanal/analyzer/language/dart/pub/testdata/happy/pubspec.lock diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index 5961d9ae2a77..abb0781ba42a 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -2,35 +2,175 @@ package pub import ( "context" + "io" + "io/fs" "os" + "path" "path/filepath" + "runtime" + "sort" + "github.com/samber/lo" "golang.org/x/xerrors" + "gopkg.in/yaml.v3" "github.com/aquasecurity/go-dep-parser/pkg/dart/pub" + godeptypes "github.com/aquasecurity/go-dep-parser/pkg/types" + "github.com/aquasecurity/go-dep-parser/pkg/utils" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language" "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/utils/fsutils" ) func init() { - analyzer.RegisterAnalyzer(&pubSpecLockAnalyzer{}) + analyzer.RegisterPostAnalyzer(analyzer.TypePubSpecLock, newPubSpecLockAnalyzer) } const ( - version = 1 + version = 2 + pubSpecYamlFileName = "pubspec.yaml" ) -// pubSpecLockAnalyzer analyzes pubspec.lock -type pubSpecLockAnalyzer struct{} +// pubSpecLockAnalyzer analyzes `pubspec.lock` +type pubSpecLockAnalyzer struct { + parser godeptypes.Parser +} + +func newPubSpecLockAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) { + return pubSpecLockAnalyzer{ + parser: pub.NewParser(), + }, nil +} + +func (a pubSpecLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) { + var apps []types.Application + + // get all DependsOn from cache dir + // `lib_ID` -> `lib_names` + allDependsOn, err := findDependsOn() + if err != nil { + log.Logger.Warnf("Unable to parse cache dir: %s", err) + } + + required := func(path string, d fs.DirEntry) bool { + return filepath.Base(path) == types.PubSpecLock + } + + err = fsutils.WalkDir(input.FS, ".", required, func(path string, _ fs.DirEntry, r io.Reader) error { + app, err := language.Parse(types.Pub, path, r, a.parser) + if err != nil { + return xerrors.Errorf("unable to parse %q: %w", path, err) + } + + if app == nil { + return nil + } + + // Required to search for library versions from DependsOn. + libs := lo.SliceToMap(app.Libraries, func(lib types.Package) (string, string) { + return lib.Name, lib.ID + }) + + for i, lib := range app.Libraries { + var dependsOn []string + for _, depName := range allDependsOn[lib.ID] { + if depID, ok := libs[depName]; ok { + dependsOn = append(dependsOn, depID) + } + } + app.Libraries[i].DependsOn = dependsOn + } + + sort.Sort(app.Libraries) + apps = append(apps, *app) + return nil + }) + if err != nil { + return nil, xerrors.Errorf("walk error: %w", err) + } + + return &analyzer.AnalysisResult{ + Applications: apps, + }, nil +} -func (a pubSpecLockAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { - p := pub.NewParser() - res, err := language.Analyze(types.Pub, input.FilePath, input.Content, p) +func findDependsOn() (map[string][]string, error) { + dir := cacheDir() + if !fsutils.DirExists(dir) { + log.Logger.Debugf("Cache dir (%s) not found. Need 'dart pub get' to fill dependency relationships", dir) + return nil, nil + } + + deps := make(map[string][]string) + if err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } else if !d.Type().IsRegular() { + return nil + } + // parse only `pubspec.yaml` files + if path.Base(p) != pubSpecYamlFileName { + return nil + } + + id, dependsOn, err := parsePubSpecYaml(p) + if err != nil { + return xerrors.Errorf("unable to parse %q: %s", p, err) + } + if id != "" { + deps[id] = dependsOn + } + return nil + + }); err != nil { + return nil, xerrors.Errorf("walk error: %w", err) + } + return deps, nil +} + +// https://dart.dev/tools/pub/glossary#system-cache +func cacheDir() string { + if dir := os.Getenv("PUB_CACHE"); dir != "" { + return dir + } + + // `%LOCALAPPDATA%\Pub\Cache` for Windows + if runtime.GOOS == "windows" { + return filepath.Join(os.Getenv("LOCALAPPDATA"), "Pub", "Cache") + } + + // `~/.pub-cache` for Linux or Mac + return filepath.Join(os.Getenv("HOME"), ".pub_cache") +} + +type pubSpecYaml struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Dependencies map[string]string `yaml:"dependencies,omitempty"` +} + +func parsePubSpecYaml(filePath string) (string, []string, error) { + f, err := os.Open(filePath) if err != nil { - return nil, xerrors.Errorf("%s parse error: %w", input.FilePath, err) + return "", nil, xerrors.Errorf("unable to open %q to get list of direct deps: %w", filePath, err) + } + defer func() { _ = f.Close() }() + + var spec pubSpecYaml + if err = yaml.NewDecoder(f).Decode(&spec); err != nil { + return "", nil, xerrors.Errorf("unable to decode %q: %w", filePath, err) + } + if len(spec.Dependencies) > 0 { + // pubspec.yaml uses version ranges + // save only dependencies names + dependsOn := lo.MapToSlice(spec.Dependencies, func(key string, _ string) string { + return key + }) + return utils.PackageID(spec.Name, spec.Version), dependsOn, nil } - return res, nil + return "", nil, nil } func (a pubSpecLockAnalyzer) Required(filePath string, _ os.FileInfo) bool { diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go b/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go index e03d9c1c8a64..86d70bbcb4e1 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go @@ -1,8 +1,9 @@ package pub import ( + "context" "os" - "sort" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -14,35 +15,51 @@ import ( func Test_pubSpecLockAnalyzer_Analyze(t *testing.T) { tests := []struct { - name string - inputFile string - want *analyzer.AnalysisResult - wantErr assert.ErrorAssertionFunc + name string + dir string + pubCacheEnv string + want *analyzer.AnalysisResult + wantErr assert.ErrorAssertionFunc }{ { - name: "happy path", - inputFile: "testdata/happy.lock", + // Supports only absolute paths for `rootUri` in package_config.json + // But for this test this field was changed + name: "happy path with cache", + dir: "testdata/happy", + pubCacheEnv: "testdata/happy/cache", want: &analyzer.AnalysisResult{ Applications: []types.Application{ { Type: types.Pub, - FilePath: "testdata/happy.lock", + FilePath: "pubspec.lock", Libraries: types.Packages{ { - ID: "crypto@3.0.2", + ID: "collection@1.17.0", + Name: "collection", + Version: "1.17.0", + Indirect: true, + }, + { + ID: "crypto@3.0.3", Name: "crypto", - Version: "3.0.2", + Version: "3.0.3", + DependsOn: []string{ + "typed_data@1.3.2", + }, }, { - ID: "flutter_test@0.0.0", - Name: "flutter_test", - Version: "0.0.0", + ID: "meta@1.11.0", + Name: "meta", + Version: "1.11.0", }, { - ID: "uuid@3.0.6", - Name: "uuid", - Version: "3.0.6", + ID: "typed_data@1.3.2", + Name: "typed_data", + Version: "1.3.2", Indirect: true, + DependsOn: []string{ + "collection@1.17.0", + }, }, }, }, @@ -51,39 +68,108 @@ func Test_pubSpecLockAnalyzer_Analyze(t *testing.T) { wantErr: assert.NoError, }, { - name: "empty file", - inputFile: "testdata/empty.lock", - wantErr: assert.NoError, + name: "happy path without cache", + dir: "testdata/happy", + pubCacheEnv: "testdata/happy/empty", + want: &analyzer.AnalysisResult{ + Applications: []types.Application{ + { + Type: types.Pub, + FilePath: "pubspec.lock", + Libraries: types.Packages{ + { + ID: "collection@1.17.0", + Name: "collection", + Version: "1.17.0", + Indirect: true, + }, + { + ID: "crypto@3.0.3", + Name: "crypto", + Version: "3.0.3", + }, + { + ID: "meta@1.11.0", + Name: "meta", + Version: "1.11.0", + }, + { + ID: "typed_data@1.3.2", + Name: "typed_data", + Version: "1.3.2", + Indirect: true, + }, + }, + }, + }, + }, + wantErr: assert.NoError, }, { - name: "broken file", - inputFile: "testdata/broken.lock", - wantErr: assert.Error, + name: "empty file", + dir: "testdata/empty", + want: &analyzer.AnalysisResult{}, + wantErr: assert.NoError, + }, + { + name: "broken file", + dir: "testdata/broken", + want: &analyzer.AnalysisResult{}, + wantErr: assert.Error, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - f, err := os.Open(tt.inputFile) + t.Setenv("PUB_CACHE", tt.pubCacheEnv) + a, err := newPubSpecLockAnalyzer(analyzer.AnalyzerOptions{}) require.NoError(t, err) - defer f.Close() - a := pubSpecLockAnalyzer{} - got, err := a.Analyze(nil, analyzer.AnalysisInput{ - FilePath: tt.inputFile, - Content: f, + got, err := a.PostAnalyze(context.Background(), analyzer.PostAnalysisInput{ + FS: os.DirFS(tt.dir), }) - if got != nil { - for _, app := range got.Applications { - sort.Sort(app.Libraries) - } - } + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} - if !tt.wantErr(t, err, tt.inputFile) { - return +func Test_pubSpecLockAnalyzer_cacheDir(t *testing.T) { + tests := []struct { + name string + pubCacheEnv string + localAppDataEnv string + windowsTest bool + wantDir string + }{ + { + name: "default cache dir for Linux/MacOS", + wantDir: "/root/.pub_cache", + }, + { + name: "default cache dir Windows", + localAppDataEnv: `C:\Users\User\AppData\Local`, + windowsTest: true, + wantDir: `C:\Users\User\AppData\Local\Pub\Cache`, + }, + { + name: "PUB_CACHE is used", + pubCacheEnv: "/root/cache", + wantDir: "/root/cache", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if runtime.GOOS != "windows" && tt.windowsTest { + t.Skipf("This test is not used for %s", runtime.GOOS) } - assert.Equal(t, tt.want, got) + t.Setenv("HOME", "/root") + t.Setenv("PUB_CACHE", tt.pubCacheEnv) + + dir := cacheDir() + assert.Equal(t, tt.wantDir, dir) }) } } diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/broken.lock b/pkg/fanal/analyzer/language/dart/pub/testdata/broken/pubspec.lock similarity index 100% rename from pkg/fanal/analyzer/language/dart/pub/testdata/broken.lock rename to pkg/fanal/analyzer/language/dart/pub/testdata/broken/pubspec.lock diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/empty.lock b/pkg/fanal/analyzer/language/dart/pub/testdata/empty/pubspec.lock similarity index 100% rename from pkg/fanal/analyzer/language/dart/pub/testdata/empty.lock rename to pkg/fanal/analyzer/language/dart/pub/testdata/empty/pubspec.lock diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy.lock b/pkg/fanal/analyzer/language/dart/pub/testdata/happy.lock deleted file mode 100644 index 3a37840aa3bb..000000000000 --- a/pkg/fanal/analyzer/language/dart/pub/testdata/happy.lock +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by pub -# See https://dart.dev/tools/pub/glossary#lockfile -packages: - crypto: - dependency: "direct main" - description: - name: crypto - url: "https://pub.flutter-io.cn" - source: hosted - version: "3.0.2" - flutter_test: - dependency: "direct dev" - description: flutter - source: sdk - version: "0.0.0" - uuid: - dependency: transitive - description: - name: uuid - url: "https://pub.flutter-io.cn" - source: hosted - version: "3.0.6" -sdks: - dart: ">=2.18.0 <3.0.0" - flutter: ">=3.3.0" \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/collection-1.18.0/pubspec.yaml b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/collection-1.18.0/pubspec.yaml new file mode 100644 index 000000000000..2e54e00344a6 --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/collection-1.18.0/pubspec.yaml @@ -0,0 +1,16 @@ +name: collection +version: 1.18.0 +description: >- + Collections and utilities functions and classes related to collections. +repository: https://github.com/dart-lang/collection + +topics: + - data-structures + - collections + +environment: + sdk: ">=2.18.0 <4.0.0" + +dev_dependencies: + lints: ^2.0.1 + test: ^1.16.0 \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/crypto-3.0.3/pubspec.yaml b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/crypto-3.0.3/pubspec.yaml new file mode 100644 index 000000000000..29cc438636a8 --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/crypto-3.0.3/pubspec.yaml @@ -0,0 +1,17 @@ +name: crypto +version: 3.0.3 +description: Implementations of SHA, MD5, and HMAC cryptographic functions. +repository: https://github.com/dart-lang/crypto +topics: + - crypto + +environment: + sdk: '>=2.19.0 <3.0.0' + +dependencies: + typed_data: ^1.3.0 + +dev_dependencies: + convert: ^3.0.0 + dart_flutter_team_lints: ^1.0.0 + test: ^1.16.0 \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/meta-1.11.0/pubspec.yaml b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/meta-1.11.0/pubspec.yaml new file mode 100644 index 000000000000..d4c6f0f86340 --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/meta-1.11.0/pubspec.yaml @@ -0,0 +1,17 @@ +name: meta +# Note, because version `2.0.0` was mistakenly released, the next major version must be `3.x.y`. +version: 1.11.0 +description: >- + Annotations used to express developer intentions that can't otherwise be + deduced by statically analyzing source code. +repository: https://github.com/dart-lang/sdk/tree/main/pkg/meta + +environment: + sdk: ">=2.12.0 <4.0.0" + +# We use 'any' version constraints here as we get our package versions from +# the dart-lang/sdk repo's DEPS file. Note that this is a special case; the +# best practice for packages is to specify their compatible version ranges. +# See also https://dart.dev/tools/pub/dependencies. +dev_dependencies: + lints: any diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/typed_data-1.3.2/pubspec.yaml b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/typed_data-1.3.2/pubspec.yaml new file mode 100644 index 000000000000..642bf644115c --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/cache/hosted/pub.dev/typed_data-1.3.2/pubspec.yaml @@ -0,0 +1,18 @@ +name: typed_data +version: 1.3.2 +description: >- + Utility functions and classes related to the dart:typed_data library. +repository: https://github.com/dart-lang/typed_data + +topics: + - data-structures + +environment: + sdk: '>=2.17.0 <4.0.0' + +dependencies: + collection: ^1.15.0 + +dev_dependencies: + dart_flutter_team_lints: ^0.1.0 + test: ^1.16.0 \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/dart/pub/testdata/happy/pubspec.lock b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/pubspec.lock new file mode 100644 index 000000000000..d3bd9670042c --- /dev/null +++ b/pkg/fanal/analyzer/language/dart/pub/testdata/happy/pubspec.lock @@ -0,0 +1,37 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + collection: + dependency: transitive + description: + name: collection + sha256: cfc915e6923fe5ce6e153b0723c753045de46de1b4d63771530504004a45fae0 + url: "https://pub.dev" + source: hosted + version: "1.17.0" + crypto: + dependency: "direct main" + description: + name: crypto + sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab + url: "https://pub.dev" + source: hosted + version: "3.0.3" + meta: + dependency: "direct dev" + description: + name: meta + sha256: d584fa6707a52763a52446f02cc621b077888fb63b93bbcb1143a7be5a0c0c04 + url: "https://pub.dev" + source: hosted + version: "1.11.0" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: facc8d6582f16042dd49f2463ff1bd6e2c9ef9f3d5da3d9b087e244a7b564b3c + url: "https://pub.dev" + source: hosted + version: "1.3.2" +sdks: + dart: ">=3.1.0 <4.0.0" \ No newline at end of file From 3863a5ec214cc365e8f69fe3fe26216a9ab86446 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 13 Oct 2023 14:34:22 +0600 Subject: [PATCH 02/11] fix unit tests --- .../analyzer/language/dart/pub/pubspec.go | 3 +-- .../language/dart/pub/pubspec_test.go | 27 ++++++++++++++----- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index abb0781ba42a..bc50f9b4086b 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -5,7 +5,6 @@ import ( "io" "io/fs" "os" - "path" "path/filepath" "runtime" "sort" @@ -111,7 +110,7 @@ func findDependsOn() (map[string][]string, error) { return nil } // parse only `pubspec.yaml` files - if path.Base(p) != pubSpecYamlFileName { + if filepath.Base(p) != pubSpecYamlFileName { return nil } diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go b/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go index 86d70bbcb4e1..d8ecad82fb3f 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec_test.go @@ -148,24 +148,37 @@ func Test_pubSpecLockAnalyzer_cacheDir(t *testing.T) { wantDir: "/root/.pub_cache", }, { - name: "default cache dir Windows", - localAppDataEnv: `C:\Users\User\AppData\Local`, - windowsTest: true, - wantDir: `C:\Users\User\AppData\Local\Pub\Cache`, + name: "default cache dir Windows", + windowsTest: true, + wantDir: "C:\\Users\\User\\AppData\\Local\\Pub\\Cache", }, { name: "PUB_CACHE is used", pubCacheEnv: "/root/cache", wantDir: "/root/cache", }, + { + name: "PUB_CACHE is used in Windows", + pubCacheEnv: "C:\\Cache", + windowsTest: true, + wantDir: "C:\\Cache", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if runtime.GOOS != "windows" && tt.windowsTest { - t.Skipf("This test is not used for %s", runtime.GOOS) + if runtime.GOOS == "windows" { + if !tt.windowsTest { + t.Skipf("This test is not used for %s", runtime.GOOS) + } + t.Setenv("LOCALAPPDATA", "C:\\Users\\User\\AppData\\Local") + } else { + if tt.windowsTest { + t.Skipf("This test is not used for %s", runtime.GOOS) + } + t.Setenv("HOME", "/root") } - t.Setenv("HOME", "/root") + t.Setenv("PUB_CACHE", tt.pubCacheEnv) dir := cacheDir() From 127f34ba4d56cce82e85af882395abea3a82e0a0 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 13 Oct 2023 15:03:28 +0600 Subject: [PATCH 03/11] docs: add info about dependency tree --- docs/docs/configuration/reporting.md | 2 ++ docs/docs/coverage/language/dart.md | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/docs/configuration/reporting.md b/docs/docs/configuration/reporting.md index 951fd779cd55..4e6149789ec0 100644 --- a/docs/docs/configuration/reporting.md +++ b/docs/docs/configuration/reporting.md @@ -65,6 +65,8 @@ The following packages/languages are currently supported: - Composer - Java - Maven: pom.xml +- Dart + - pub: pubspec.lock This tree is the reverse of the npm list command. However, if you want to resolve a vulnerability in a particular indirect dependency, the reversed tree is useful to know where that dependency comes from and identify which package you actually need to update. diff --git a/docs/docs/coverage/language/dart.md b/docs/docs/coverage/language/dart.md index f7bba0a3951f..0ce6d1cc52b0 100644 --- a/docs/docs/coverage/language/dart.md +++ b/docs/docs/coverage/language/dart.md @@ -13,7 +13,7 @@ The following table provides an outline of the features Trivy offers. | Package manager | File | Transitive dependencies | Dev dependencies | [Dependency graph][dependency-graph] | Position | |-------------------------|--------------|:-----------------------:|:----------------:|:------------------------------------:|:--------:| -| [Dart][dart-repository] | pubspec.lock | ✓ | Included | - | - | +| [Dart][dart-repository] | pubspec.lock | ✓ | Included | ✓ | - | ## Dart In order to detect dependencies, Trivy searches for `pubspec.lock`. @@ -21,6 +21,11 @@ In order to detect dependencies, Trivy searches for `pubspec.lock`. Trivy marks indirect dependencies, but `pubspec.lock` file doesn't have options to separate root and dev transitive dependencies. So Trivy includes all dependencies in report. +To build `dependency tree` Trivy parses [cache directory][cache-directory]. Currently supported default directories and `PUB_CACHE` environment (absolute path only). +!!! note + Make sure the cache directory contains all the dependencies installed in your application. To download missing dependencies, use `dart pub get` command. + [dart]: https://dart.dev/ [dart-repository]: https://pub.dev/ [dependency-graph]: ../../configuration/reporting.md#show-origins-of-vulnerable-dependencies +[cache-directory]: https://dart.dev/tools/pub/glossary#system-cache From c30a8d4979751711f131d89284ec48ba7cf4a0cc Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Mon, 16 Oct 2023 13:55:21 +0600 Subject: [PATCH 04/11] refactor --- .../analyzer/language/dart/pub/pubspec.go | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index bc50f9b4086b..7d79a89f982c 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -47,7 +47,7 @@ func (a pubSpecLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostA var apps []types.Application // get all DependsOn from cache dir - // `lib_ID` -> `lib_names` + // lib ID -> DependsOn names allDependsOn, err := findDependsOn() if err != nil { log.Logger.Warnf("Unable to parse cache dir: %s", err) @@ -67,19 +67,21 @@ func (a pubSpecLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostA return nil } - // Required to search for library versions from DependsOn. - libs := lo.SliceToMap(app.Libraries, func(lib types.Package) (string, string) { - return lib.Name, lib.ID - }) - - for i, lib := range app.Libraries { - var dependsOn []string - for _, depName := range allDependsOn[lib.ID] { - if depID, ok := libs[depName]; ok { - dependsOn = append(dependsOn, depID) + if allDependsOn != nil { + // Required to search for library versions for DependsOn. + libs := lo.SliceToMap(app.Libraries, func(lib types.Package) (string, string) { + return lib.Name, lib.ID + }) + + for i, lib := range app.Libraries { + var dependsOn []string + for _, depName := range allDependsOn[lib.ID] { + if depID, ok := libs[depName]; ok { + dependsOn = append(dependsOn, depID) + } } + app.Libraries[i].DependsOn = dependsOn } - app.Libraries[i].DependsOn = dependsOn } sort.Sort(app.Libraries) @@ -116,7 +118,8 @@ func findDependsOn() (map[string][]string, error) { id, dependsOn, err := parsePubSpecYaml(p) if err != nil { - return xerrors.Errorf("unable to parse %q: %s", p, err) + log.Logger.Debugf("unable to parse %q: %s", p, err) + return nil } if id != "" { deps[id] = dependsOn @@ -145,9 +148,9 @@ func cacheDir() string { } type pubSpecYaml struct { - Name string `yaml:"name"` - Version string `yaml:"version"` - Dependencies map[string]string `yaml:"dependencies,omitempty"` + Name string `yaml:"name"` + Version string `yaml:"version"` + Dependencies map[string]interface{} `yaml:"dependencies,omitempty"` } func parsePubSpecYaml(filePath string) (string, []string, error) { @@ -164,7 +167,7 @@ func parsePubSpecYaml(filePath string) (string, []string, error) { if len(spec.Dependencies) > 0 { // pubspec.yaml uses version ranges // save only dependencies names - dependsOn := lo.MapToSlice(spec.Dependencies, func(key string, _ string) string { + dependsOn := lo.MapToSlice(spec.Dependencies, func(key string, _ interface{}) string { return key }) return utils.PackageID(spec.Name, spec.Version), dependsOn, nil From f5bbf4b55ed621341c63b3b293b343739e17e08d Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 19 Oct 2023 12:11:03 +0600 Subject: [PATCH 05/11] docs: add table for languages/packages supported by dependency tree --- docs/docs/configuration/reporting.md | 63 ++++++++++++++++------------ 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/docs/docs/configuration/reporting.md b/docs/docs/configuration/reporting.md index 4e6149789ec0..3af7e53ff1d9 100644 --- a/docs/docs/configuration/reporting.md +++ b/docs/docs/configuration/reporting.md @@ -41,32 +41,29 @@ In some cases, vulnerable dependencies are not linked directly, and it requires To make this task simpler Trivy can show a dependency origin tree with the `--dependency-tree` flag. This flag is only available with the `--format table` flag. -The following packages/languages are currently supported: - -- OS packages - - apk - - dpkg - - rpm -- Node.js - - npm: package-lock.json - - pnpm: pnpm-lock.yaml - - yarn: yarn.lock -- .NET - - NuGet: packages.lock.json -- Python - - Poetry: poetry.lock -- Ruby - - Bundler: Gemfile.lock -- Rust - - Binaries built with [cargo-auditable][cargo-auditable] -- Go - - Modules: go.mod -- PHP - - Composer -- Java - - Maven: pom.xml -- Dart - - pub: pubspec.lock +The following package managers are currently supported: + +| Package Managers | +|------------------| +| apk | +| dpkg | +| rpm | + +The following languages are currently supported: + +| Language | File | +|----------|--------------------------------------------| +| Node.js | [package-lock.json][nodejs-package-lock] | +| | [pnpm-lock.yaml][pnpm-lock] | +| | [yarn.lock][yarn-lock] | +| .NET | [packages.lock.json][dotnet-packages-lock] | +| Python | [poetry.lock][poetry-lock] | +| Ruby | [Gemfile.lock][gemfile-lock] | +| Rust | [cargo-auditable binaries][cargo-binaries] | +| Go | [go.mod][go-mod] | +| PHP | [composer.lock][composer-lock] | +| Java | [pom.xml][pom-xml] | +| Dart | [pubspec.lock][pubspec-lock] | This tree is the reverse of the npm list command. However, if you want to resolve a vulnerability in a particular indirect dependency, the reversed tree is useful to know where that dependency comes from and identify which package you actually need to update. @@ -410,4 +407,16 @@ $ trivy convert --format table --severity CRITICAL result.json [github-sbom-submit]: https://docs.github.com/en/rest/dependency-graph/dependency-submission?apiVersion=2022-11-28#create-a-snapshot-of-dependencies-for-a-repository [os_packages]: ../scanner/vulnerability.md#os-packages -[language_packages]: ../scanner/vulnerability.md#language-specific-packages \ No newline at end of file +[language_packages]: ../scanner/vulnerability.md#language-specific-packages + +[nodejs-package-lock]: ../coverage/language/nodejs.md#npm +[pnpm-lock]: ../coverage/language/nodejs.md#pnpm +[yarn-lock]: ../coverage/language/nodejs.md#yarn +[dotnet-packages-lock]: ../coverage/language/dotnet.md#packageslockjson +[poetry-lock]: ../coverage/language/python.md#poetry +[gemfile-lock]: ../coverage/language/ruby.md#bundler +[go-mod]: ../coverage/language/golang.md#go-modules +[composer-lock]: ../coverage/language/php.md#composer +[pom-xml]: ../coverage/language/java.md#pomxml +[pubspec-lock]: ../coverage/language/dart.md#dart +[cargo-binaries]: ../coverage/language/rust.md#binaries \ No newline at end of file From ffe7308f7258d614b98ec9752b01889944feb628 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Thu, 19 Oct 2023 12:28:38 +0600 Subject: [PATCH 06/11] refactor --- pkg/fanal/analyzer/language/dart/pub/pubspec.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index 7d79a89f982c..b7589c7305df 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -164,15 +164,15 @@ func parsePubSpecYaml(filePath string) (string, []string, error) { if err = yaml.NewDecoder(f).Decode(&spec); err != nil { return "", nil, xerrors.Errorf("unable to decode %q: %w", filePath, err) } - if len(spec.Dependencies) > 0 { - // pubspec.yaml uses version ranges - // save only dependencies names - dependsOn := lo.MapToSlice(spec.Dependencies, func(key string, _ interface{}) string { - return key - }) - return utils.PackageID(spec.Name, spec.Version), dependsOn, nil + if len(spec.Dependencies) == 0 { + return "", nil, nil } - return "", nil, nil + // pubspec.yaml uses version ranges + // save only dependencies names + dependsOn := lo.MapToSlice(spec.Dependencies, func(key string, _ interface{}) string { + return key + }) + return utils.PackageID(spec.Name, spec.Version), dependsOn, nil } func (a pubSpecLockAnalyzer) Required(filePath string, _ os.FileInfo) bool { From 3abdc473a0e6dc558969ea67c85348c03cbd7910 Mon Sep 17 00:00:00 2001 From: DmitriyLewen Date: Fri, 20 Oct 2023 11:20:51 +0600 Subject: [PATCH 07/11] refactor: use fsutils.WalkDir --- .../analyzer/language/dart/pub/pubspec.go | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index b7589c7305df..24ff94cb520b 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -104,21 +104,15 @@ func findDependsOn() (map[string][]string, error) { return nil, nil } - deps := make(map[string][]string) - if err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { - if err != nil { - return err - } else if !d.Type().IsRegular() { - return nil - } - // parse only `pubspec.yaml` files - if filepath.Base(p) != pubSpecYamlFileName { - return nil - } + required := func(path string, d fs.DirEntry) bool { + return filepath.Base(path) == pubSpecYamlFileName + } - id, dependsOn, err := parsePubSpecYaml(p) + deps := make(map[string][]string) + if err := fsutils.WalkDir(os.DirFS(dir), ".", required, func(path string, d fs.DirEntry, r io.Reader) error { + id, dependsOn, err := parsePubSpecYaml(r) if err != nil { - log.Logger.Debugf("unable to parse %q: %s", p, err) + log.Logger.Debugf("unable to parse %q: %s", path, err) return nil } if id != "" { @@ -149,21 +143,24 @@ func cacheDir() string { type pubSpecYaml struct { Name string `yaml:"name"` - Version string `yaml:"version"` + Version string `yaml:"version,omitempty"` Dependencies map[string]interface{} `yaml:"dependencies,omitempty"` } -func parsePubSpecYaml(filePath string) (string, []string, error) { - f, err := os.Open(filePath) - if err != nil { - return "", nil, xerrors.Errorf("unable to open %q to get list of direct deps: %w", filePath, err) +func parsePubSpecYaml(r io.Reader) (string, []string, error) { + var spec pubSpecYaml + if err := yaml.NewDecoder(r).Decode(&spec); err != nil { + return "", nil, xerrors.Errorf("unable to decode: %w", err) } - defer func() { _ = f.Close() }() - var spec pubSpecYaml - if err = yaml.NewDecoder(f).Decode(&spec); err != nil { - return "", nil, xerrors.Errorf("unable to decode %q: %w", filePath, err) + // Version is a required field only for packages from pub.dev: + // https://dart.dev/tools/pub/pubspec#version + // We can skip packages without version, + // because we compare packages by ID (name+version) + if spec.Version == "" { + return "", nil, nil } + if len(spec.Dependencies) == 0 { return "", nil, nil } From 6543732b5d01c5ca6060b512ca004cf11c6ba753 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 20 Oct 2023 17:33:21 +0900 Subject: [PATCH 08/11] refactor: use maps.Keys Signed-off-by: knqyf263 --- pkg/fanal/analyzer/language/dart/pub/pubspec.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index 24ff94cb520b..5747465ef745 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -10,6 +10,7 @@ import ( "sort" "github.com/samber/lo" + "golang.org/x/exp/maps" "golang.org/x/xerrors" "gopkg.in/yaml.v3" @@ -166,9 +167,8 @@ func parsePubSpecYaml(r io.Reader) (string, []string, error) { } // pubspec.yaml uses version ranges // save only dependencies names - dependsOn := lo.MapToSlice(spec.Dependencies, func(key string, _ interface{}) string { - return key - }) + dependsOn := maps.Keys(spec.Dependencies) + return utils.PackageID(spec.Name, spec.Version), dependsOn, nil } From 8a6335588529ae0cc0ca18b7f2680407148ee8ef Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 20 Oct 2023 17:33:42 +0900 Subject: [PATCH 09/11] refactor: capitalize a debug message Signed-off-by: knqyf263 --- pkg/fanal/analyzer/language/dart/pub/pubspec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index 5747465ef745..6a2780e86784 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -113,7 +113,7 @@ func findDependsOn() (map[string][]string, error) { if err := fsutils.WalkDir(os.DirFS(dir), ".", required, func(path string, d fs.DirEntry, r io.Reader) error { id, dependsOn, err := parsePubSpecYaml(r) if err != nil { - log.Logger.Debugf("unable to parse %q: %s", path, err) + log.Logger.Debugf("Unable to parse %q: %s", path, err) return nil } if id != "" { From 1f58071b0d507095efeb9760301abe8e908099ee Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 20 Oct 2023 17:34:10 +0900 Subject: [PATCH 10/11] refactor: aggregate if statements Signed-off-by: knqyf263 --- pkg/fanal/analyzer/language/dart/pub/pubspec.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/fanal/analyzer/language/dart/pub/pubspec.go b/pkg/fanal/analyzer/language/dart/pub/pubspec.go index 6a2780e86784..221fe73e6b1a 100644 --- a/pkg/fanal/analyzer/language/dart/pub/pubspec.go +++ b/pkg/fanal/analyzer/language/dart/pub/pubspec.go @@ -158,13 +158,10 @@ func parsePubSpecYaml(r io.Reader) (string, []string, error) { // https://dart.dev/tools/pub/pubspec#version // We can skip packages without version, // because we compare packages by ID (name+version) - if spec.Version == "" { + if spec.Version == "" || len(spec.Dependencies) == 0 { return "", nil, nil } - if len(spec.Dependencies) == 0 { - return "", nil, nil - } // pubspec.yaml uses version ranges // save only dependencies names dependsOn := maps.Keys(spec.Dependencies) From 9c258ade95eb0669d998b12631cab2805fccf186 Mon Sep 17 00:00:00 2001 From: knqyf263 Date: Fri, 20 Oct 2023 17:36:01 +0900 Subject: [PATCH 11/11] docs: some tweaks Signed-off-by: knqyf263 --- docs/docs/configuration/reporting.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/docs/configuration/reporting.md b/docs/docs/configuration/reporting.md index 3af7e53ff1d9..17aa6fd973ec 100644 --- a/docs/docs/configuration/reporting.md +++ b/docs/docs/configuration/reporting.md @@ -41,13 +41,13 @@ In some cases, vulnerable dependencies are not linked directly, and it requires To make this task simpler Trivy can show a dependency origin tree with the `--dependency-tree` flag. This flag is only available with the `--format table` flag. -The following package managers are currently supported: +The following OS package managers are currently supported: -| Package Managers | -|------------------| -| apk | -| dpkg | -| rpm | +| OS Package Managers | +|---------------------| +| apk | +| dpkg | +| rpm | The following languages are currently supported: @@ -65,7 +65,7 @@ The following languages are currently supported: | Java | [pom.xml][pom-xml] | | Dart | [pubspec.lock][pubspec-lock] | -This tree is the reverse of the npm list command. +This tree is the reverse of the dependency graph. However, if you want to resolve a vulnerability in a particular indirect dependency, the reversed tree is useful to know where that dependency comes from and identify which package you actually need to update. In table output, it looks like: