From d8a80e11d28f9eb5099e101c0c73c2dcfaddec55 Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Mon, 25 Sep 2023 18:43:39 +0530 Subject: [PATCH 01/21] Adding pkg:conan to the constants --- src/LCT.APICommunications/ApiConstant.cs | 1 + src/LCT.Common/Constants/Dataconstant.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/LCT.APICommunications/ApiConstant.cs b/src/LCT.APICommunications/ApiConstant.cs index b896eaa2..f6332227 100644 --- a/src/LCT.APICommunications/ApiConstant.cs +++ b/src/LCT.APICommunications/ApiConstant.cs @@ -29,6 +29,7 @@ public static class ApiConstant public const string ComponentNameUrl = "?name="; public const string NPMExternalID = "pkg:npm/"; public const string NugetExternalID = "pkg:nuget/"; + public const string ConanExternalID = "pkg:conan/"; public const string NpmExtension = ".tgz"; public const string NugetExtension = ".nupkg"; public const string MavenExtension = "-sources.jar"; diff --git a/src/LCT.Common/Constants/Dataconstant.cs b/src/LCT.Common/Constants/Dataconstant.cs index 3f8d7504..0b30c82d 100644 --- a/src/LCT.Common/Constants/Dataconstant.cs +++ b/src/LCT.Common/Constants/Dataconstant.cs @@ -22,6 +22,7 @@ public static class Dataconstant {"DEBIAN", "pkg:deb/debian"}, {"MAVEN", "pkg:maven"}, {"PYTHON", "pkg:pypi"}, + {"CONAN", "pkg:conan"}, }; //Identified types From ae86ab1fda481f710e2815166bc059320ade5d94 Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Mon, 25 Sep 2023 18:44:07 +0530 Subject: [PATCH 02/21] Adding Conan to the configuration --- src/LCT.Common/CommonAppSettings.cs | 1 + src/LCT.Common/Model/Config.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/LCT.Common/CommonAppSettings.cs b/src/LCT.Common/CommonAppSettings.cs index 7acb4cb1..c1d10b46 100644 --- a/src/LCT.Common/CommonAppSettings.cs +++ b/src/LCT.Common/CommonAppSettings.cs @@ -65,6 +65,7 @@ public CommonAppSettings(IFolderAction iFolderAction) public Config Maven { get; set; } public Config Debian { get; set; } public Config Python { get; set; } + public Config Conan { get; set; } public string CaVersion { get; set; } public string CycloneDxSBomTemplatePath { get; set; } public string[] InternalRepoList { get; set; } diff --git a/src/LCT.Common/Model/Config.cs b/src/LCT.Common/Model/Config.cs index 9b0f4ca3..6bc38c08 100644 --- a/src/LCT.Common/Model/Config.cs +++ b/src/LCT.Common/Model/Config.cs @@ -22,6 +22,7 @@ public class Config public string[] JfrogNugetRepoList { get; set; } public string[] JfrogMavenRepoList { get; set; } public string[] JfrogPythonRepoList { get; set; } + public string[] JfrogConanRepoList { get; set; } public string[] DevDependentScopeList { get; set; } } From 4b61c7e10bf73f23d0276ed5d3ea1b83ec4c8d82 Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Mon, 25 Sep 2023 18:44:47 +0530 Subject: [PATCH 03/21] Creating a conan processor to read and parse the conan.lock file --- src/LCT.PackageIdentifier/ConanProcessor.cs | 351 ++++++++++++++++++++ 1 file changed, 351 insertions(+) create mode 100644 src/LCT.PackageIdentifier/ConanProcessor.cs diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs new file mode 100644 index 00000000..bf6bf145 --- /dev/null +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -0,0 +1,351 @@ +// -------------------------------------------------------------------------------------------------------------------- +// SPDX-FileCopyrightText: 2023 Siemens AG +// +// SPDX-License-Identifier: MIT +// -------------------------------------------------------------------------------------------------------------------- + +using CycloneDX.Models; +using LCT.APICommunications; +using LCT.APICommunications.Model.AQL; +using LCT.Common; +using LCT.Common.Constants; +using LCT.PackageIdentifier.Interface; +using LCT.PackageIdentifier.Model; +using LCT.Services.Interface; +using log4net; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Security; +using System.Threading.Tasks; + + +namespace LCT.PackageIdentifier +{ + + /// + /// Parses the Conan Packages + /// + public class ConanProcessor : CycloneDXBomParser, IParser + { + #region fields + static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + readonly CycloneDXBomParser cycloneDXBomParser; + private const string NotFoundInRepo = "Not Found in JFrogRepo"; + #endregion + + #region constructor + public ConanProcessor() + { + if (cycloneDXBomParser == null) + { + cycloneDXBomParser = new CycloneDXBomParser(); + } + } + #endregion + + #region public methods + public Bom ParsePackageFile(CommonAppSettings appSettings) + { + List componentsForBOM = new List(); + Bom bom = new Bom(); + int totalComponentsIdentified = 0; + + ParsingInputFileForBOM(appSettings, ref componentsForBOM); + totalComponentsIdentified = componentsForBOM.Count; + + componentsForBOM = GetExcludedComponentsList(componentsForBOM); + componentsForBOM = componentsForBOM.Distinct(new ComponentEqualityComparer()).ToList(); + + BomCreator.bomKpiData.DuplicateComponents = totalComponentsIdentified - componentsForBOM.Count; + var componentsWithMultipleVersions = componentsForBOM.GroupBy(s => s.Name) + .Where(g => g.Count() > 1).SelectMany(g => g).ToList(); + + if (componentsWithMultipleVersions.Count != 0) + { + Logger.Warn($"Multiple versions detected :\n"); + foreach (var item in componentsWithMultipleVersions) + { + Logger.Warn($"Component Name : {item.Name}\nComponent Version : {item.Version}\nPackage Found in : {item.Description}\n"); + } + } + bom.Components = componentsForBOM; + Logger.Debug($"ParsePackageFile():End"); + return bom; + } + + public async Task IdentificationOfInternalComponents(ComponentIdentification componentData, CommonAppSettings appSettings, + IJFrogService jFrogService, IBomHelper bomhelper) + { + // get the component list from Jfrog for given repository + List aqlResultList = + await bomhelper.GetListOfComponentsFromRepo(appSettings.InternalRepoList, jFrogService); + + // find the components in the list of internal components + List internalComponents = new List(); + var internalComponentStatusUpdatedList = new List(); + var inputIterationList = componentData.comparisonBOMData; + + foreach (Component component in inputIterationList) + { + var currentIterationItem = component; + bool isTrue = IsInternalConanComponent(aqlResultList, currentIterationItem); + if (currentIterationItem.Properties?.Count == null || currentIterationItem.Properties?.Count <= 0) + { + currentIterationItem.Properties = new List(); + } + + Property isInternal = new() { Name = Dataconstant.Cdx_IsInternal, Value = "false" }; + if (isTrue) + { + internalComponents.Add(currentIterationItem); + isInternal.Value = "true"; + } + else + { + isInternal.Value = "false"; + } + + currentIterationItem.Properties.Add(isInternal); + internalComponentStatusUpdatedList.Add(currentIterationItem); + } + + // update the comparison BOM data + componentData.comparisonBOMData = internalComponentStatusUpdatedList; + componentData.internalComponents = internalComponents; + + return componentData; + } + + + public async Task> GetJfrogRepoDetailsOfAComponent(List componentsForBOM, + CommonAppSettings appSettings, IJFrogService jFrogService, IBomHelper bomhelper) + { + // get the component list from Jfrog for given repository + List aqlResultList = await bomhelper.GetListOfComponentsFromRepo(appSettings.Conan?.JfrogConanRepoList, jFrogService); + Property projectType = new() { Name = Dataconstant.Cdx_ProjectType, Value = appSettings.ProjectType }; + List modifiedBOM = new List(); + + foreach (var component in componentsForBOM) + { + string repoName = GetArtifactoryRepoName(aqlResultList, component); + Property artifactoryrepo = new() { Name = Dataconstant.Cdx_ArtifactoryRepoUrl, Value = repoName }; + Component componentVal = component; + + if (componentVal.Properties?.Count == null || componentVal.Properties?.Count <= 0) + { + componentVal.Properties = new List(); + } + componentVal.Properties.Add(artifactoryrepo); + componentVal.Properties.Add(projectType); + componentVal.Description = string.Empty; + + modifiedBOM.Add(componentVal); + } + + return modifiedBOM; + } + + #endregion + + #region private methods + private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List componentsForBOM) + { + List configFiles; + configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Conan); + + foreach (string filepath in configFiles) + { + Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); + var components = ParsePackageLockJson(filepath, appSettings); + AddingIdentifierType(components, "PackageFile"); + componentsForBOM.AddRange(components); + } + } + + private List ParsePackageLockJson(string filepath, CommonAppSettings appSettings) + { + List lstComponentForBOM = new List(); + int noOfDevDependent = 0; + int noOfExcludedComponents = 0; + try + { + string jsonContent = File.ReadAllText(filepath); + + var jsonDeserialized = JObject.Parse(jsonContent); + var nodes = jsonDeserialized["graph_lock"]["nodes"]; + + List nodePackages = new List(); + foreach (var node in nodes) + { + string nodeId = ((JProperty)node).Name; + var conanPackage = JsonConvert.DeserializeObject(((JProperty)node).Value.ToString()); + conanPackage.Id = nodeId; + nodePackages.Add(conanPackage); + } + + GetPackagesForBom(ref lstComponentForBOM, ref noOfDevDependent, nodePackages); + + if (appSettings.Conan.ExcludedComponents != null) + { + lstComponentForBOM = CommonHelper.RemoveExcludedComponents(lstComponentForBOM, appSettings.Conan.ExcludedComponents, ref noOfExcludedComponents); + BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; + + } + + BomCreator.bomKpiData.DevDependentComponents += noOfDevDependent; + } + catch (JsonReaderException ex) + { + Environment.ExitCode = -1; + Logger.Error($"ParsePackageFile():", ex); + } + catch (IOException ex) + { + Environment.ExitCode = -1; + Logger.Error($"ParsePackageFile():", ex); + } + catch (SecurityException ex) + { + Environment.ExitCode = -1; + Logger.Error($"ParsePackageFile():", ex); + } + + return lstComponentForBOM; + } + + private static void GetPackagesForBom(ref List lstComponentForBOM, ref int noOfDevDependent, List nodePackages) + { + var rootNode = nodePackages.FirstOrDefault(); + if (!rootNode.Dependencies.Any() || rootNode.Dependencies == null) + { + throw new ArgumentNullException(nameof(nodePackages), "Dependency(requires) node name details not present in the root node."); + } + + // Ignoring the root node as it is the package information node. + foreach (var component in nodePackages.Skip(1)) + { + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile += 1; + Property isdev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + + if (string.IsNullOrEmpty(component.Reference)) + { + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile--; + continue; + } + + Component components = new Component(); + + // dev components are not ignored and added as a part of SBOM + if (IsDevDependency(component, rootNode, ref noOfDevDependent)) + { + isdev.Value = "true"; + } + + string packageName = Convert.ToString(component.Reference); + + if (packageName.Contains('/')) + { + components.Name = packageName.Split(new char[] { '/', '@' })[0]; + components.Version = packageName.Split(new char[] { '/', '@' })[1]; + } + else + { + components.Name = packageName; + } + + components.Purl = $"{ApiConstant.ConanExternalID}{components.Name}@{components.Version}"; + components.BomRef = $"{ApiConstant.ConanExternalID}{components.Name}@{components.Version}"; + + components.Properties = new List(); + components.Properties.Add(isdev); + lstComponentForBOM.Add(components); + } + } + + private static bool IsDevDependency(ConanPackage component, ConanPackage rootNode, ref int noOfDevDependent) + { + var isDev = false; + if (rootNode.DevDependencies != null && rootNode.DevDependencies.Contains(component.Id)) + { + isDev = true; + noOfDevDependent++; + } + + return isDev; + } + + private static bool IsInternalConanComponent(List aqlResultList, Component component) + { + string jfrogcomponentPath = $"{component.Name}/{component.Version}"; + if (aqlResultList.Exists( + x => x.Path.Contains(jfrogcomponentPath, StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + return false; + } + + private static string GetArtifactoryRepoName(List aqlResultList, Component component) + { + string jfrogcomponentPath = $"{component.Name}/{component.Version}"; + + string repoName = aqlResultList.Find(x => x.Path.Contains( + jfrogcomponentPath, StringComparison.OrdinalIgnoreCase))?.Repo ?? NotFoundInRepo; + + return repoName; + } + + private static List GetExcludedComponentsList(List componentsForBOM) + { + List components = new List(); + foreach (Component componentsInfo in componentsForBOM) + { + if (!string.IsNullOrEmpty(componentsInfo.Name) && !string.IsNullOrEmpty(componentsInfo.Version) && !string.IsNullOrEmpty(componentsInfo.Purl) && componentsInfo.Purl.Contains(Dataconstant.PurlCheck()["CONAN"])) + { + components.Add(componentsInfo); + Logger.Debug($"GetExcludedComponentsList():ValidComponent For CONAN : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}"); + } + else + { + BomCreator.bomKpiData.ComponentsExcluded++; + Logger.Debug($"GetExcludedComponentsList():InvalidComponent For CONAN : Component Details : {componentsInfo?.Name} @ {componentsInfo?.Version} @ {componentsInfo?.Purl}"); + } + } + return components; + } + + private static void AddingIdentifierType(List components, string identifiedBy) + { + foreach (var component in components) + { + if (component.Properties == null) + { + component.Properties = new List(); + } + + Property isDev; + Property identifierType; + if (identifiedBy == "PackageFile") + { + identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.Discovered }; + component.Properties.Add(identifierType); + } + else + { + isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.ManullayAdded }; + component.Properties.Add(isDev); + component.Properties.Add(identifierType); + } + } + } + + #endregion + } +} From 08764b743049437ae4101e94fba8247cd458ac9c Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Mon, 25 Sep 2023 18:45:28 +0530 Subject: [PATCH 04/21] Updating the BOM creator to have type CONAN --- src/LCT.PackageIdentifier/BomCreator.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/LCT.PackageIdentifier/BomCreator.cs b/src/LCT.PackageIdentifier/BomCreator.cs index ea26b94b..14e083f7 100644 --- a/src/LCT.PackageIdentifier/BomCreator.cs +++ b/src/LCT.PackageIdentifier/BomCreator.cs @@ -119,6 +119,9 @@ private async Task CallPackageParser(CommonAppSettings appSettings) case "PYTHON": parser = new PythonProcessor(); return await ComponentIdentification(appSettings, parser); + case "CONAN": + parser = new ConanProcessor(); + return await ComponentIdentification(appSettings, parser); default: Logger.Error($"GenerateBom():Invalid ProjectType - {appSettings.ProjectType}"); break; From af33c509b1189bb0974cfcab53f4f051d41540ff Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Mon, 25 Sep 2023 18:45:42 +0530 Subject: [PATCH 05/21] New model for conan packages --- .../Model/ConanPackage.cs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/LCT.PackageIdentifier/Model/ConanPackage.cs diff --git a/src/LCT.PackageIdentifier/Model/ConanPackage.cs b/src/LCT.PackageIdentifier/Model/ConanPackage.cs new file mode 100644 index 00000000..843dd6f0 --- /dev/null +++ b/src/LCT.PackageIdentifier/Model/ConanPackage.cs @@ -0,0 +1,26 @@ + +// -------------------------------------------------------------------------------------------------------------------- +// SPDX-FileCopyrightText: 2023 Siemens AG +// +// SPDX-License-Identifier: MIT + +// -------------------------------------------------------------------------------------------------------------------- + +using Newtonsoft.Json; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace LCT.PackageIdentifier.Model +{ + [ExcludeFromCodeCoverage] + public class ConanPackage + { + public string Id { get; set; } + [JsonProperty("ref")] + public string Reference { get; set; } + [JsonProperty("requires")] + public List Dependencies { get; set; } + [JsonProperty("build_requires")] + public List DevDependencies { get; set; } + } +} From 06b761ad62118eb19434bd6cee99fb9dbf903540 Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Tue, 26 Sep 2023 14:01:08 +0530 Subject: [PATCH 06/21] Adding conan package identifier unit tests and the test conan lock file --- .../ConanParserTests.cs | 247 ++++++++++++++++++ .../PackageIdentifierUTTestFiles/conan.lock | 154 +++++++++++ 2 files changed, 401 insertions(+) create mode 100644 src/LCT.PackageIdentifier.UTest/ConanParserTests.cs create mode 100644 src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/conan.lock diff --git a/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs new file mode 100644 index 00000000..9104925e --- /dev/null +++ b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs @@ -0,0 +1,247 @@ +// -------------------------------------------------------------------------------------------------------------------- +// SPDX-FileCopyrightText: 2023 Siemens AG +// +// SPDX-License-Identifier: MIT +// -------------------------------------------------------------------------------------------------------------------- + +using CycloneDX.Models; +using LCT.APICommunications.Model.AQL; +using LCT.Common; +using LCT.Common.Model; +using LCT.PackageIdentifier; +using LCT.PackageIdentifier.Interface; +using LCT.PackageIdentifier.Model; +using LCT.Services.Interface; +using Moq; +using NUnit.Framework; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace PackageIdentifier.UTest +{ + [TestFixture] + public class ConanParserTests + { + [TestCase] + public void ParseLockFile_GivenAInputFilePath_ReturnsSuccess() + { + //Arrange + int expectednoofcomponents = 17; + string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; + string outFolder = Path.GetDirectoryName(exePath); + string packagefilepath = outFolder + @"\PackageIdentifierUTTestFiles"; + + string[] Includes = { "conan.lock" }; + Config config = new Config() + { + Include = Includes + }; + + CommonAppSettings appSettings = new CommonAppSettings() + { + PackageFilePath = packagefilepath, + Conan = config + }; + + //Act + Bom listofcomponents = new ConanProcessor().ParsePackageFile(appSettings); + + //Assert + Assert.That(expectednoofcomponents, Is.EqualTo(listofcomponents.Components.Count), "Checks for no of components"); + + } + + [TestCase] + public void ParseLockFile_GivenAInputFilePath_ReturnDevDependentComp() + { + //Arrange + string IsDev = "true"; + string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; + string outFolder = Path.GetDirectoryName(exePath); + string packagefilepath = outFolder + @"\PackageIdentifierUTTestFiles"; + + string[] Includes = { "conan.lock" }; + Config config = new Config() + { + Include = Includes + }; + + CommonAppSettings appSettings = new CommonAppSettings() + { + PackageFilePath = packagefilepath, + Conan = config + }; + + //Act + Bom listofcomponents = new ConanProcessor().ParsePackageFile(appSettings); + + var IsDevDependency = listofcomponents.Components.Find(a => a.Name == "googletest").Properties[0].Value; + + //Assert + Assert.That(IsDev, Is.EqualTo(IsDevDependency), "Checks if Dev Dependency Component or not"); + + } + + [TestCase] + public void ParseLockFile_GivenAInputFilePathExcludeComponent_ReturnComponentCount() + { + //Arrange + int totalComponentsAfterExclusion = 15; + string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; + string outFolder = Path.GetDirectoryName(exePath); + string packagefilepath = outFolder + @"\PackageIdentifierUTTestFiles"; + + string[] Includes = { "conan.lock" }; + Config config = new Config() + { + Include = Includes, + ExcludedComponents = new List { "openldap:2.6.4-shared-ossl3.1", "libcurl:7.87.0-shared-ossl3.1" } + }; + + CommonAppSettings appSettings = new CommonAppSettings() + { + PackageFilePath = packagefilepath, + Conan = config + }; + + //Act + Bom listofcomponents = new ConanProcessor().ParsePackageFile(appSettings); + + //Assert + Assert.That(totalComponentsAfterExclusion, Is.EqualTo(listofcomponents.Components.Count), "Checks if the excluded components have been removed"); + } + + [TestCase] + public void IsDevDependent_GivenListOfDevComponents_ReturnsSuccess() + { + //Arrange + var conanPackage = new ConanPackage() {Id = "10"}; + var rootNode = new ConanPackage() {DevDependencies = new List { "10", "11", "12" }}; + var noOfDevDependent = 0; + //Act + bool actual = ConanProcessor.IsDevDependency(conanPackage, rootNode, ref noOfDevDependent); + + //Assert + Assert.That(true, Is.EqualTo(actual), "Component is a dev dependent"); + } + + [Test] + public async Task IdentificationOfInternalComponents_ReturnsComponentData_Successfully() + { + // Arrange + Component component = new Component() + { + Name = "securitycommunicationmanager", + Description = string.Empty, + Version = "2.6.5", + Purl = "pkg:conan/securitycommunicationmanager@2.6.5" + }; + + var components = new List() { component }; + ComponentIdentification componentIdentification = new() { comparisonBOMData = components }; + string[] repoList = { "internalrepo1", "internalrepo2" }; + CommonAppSettings appSettings = new() { InternalRepoList = repoList }; + + AqlResult aqlResult = new() + { + Name = "index.json", + Path = "siemens-energy/securitycommunicationmanager/2.7.1/stable", + Repo = "internalrepo1" + }; + + List results = new List() { aqlResult }; + Mock mockJfrogService = new Mock(); + Mock mockBomHelper = new Mock(); + mockBomHelper.Setup(m => m.GetListOfComponentsFromRepo(It.IsAny(), It.IsAny())) + .ReturnsAsync(results); + + // Act + ConanProcessor conanProcessor = new ConanProcessor(); + var actual = await conanProcessor.IdentificationOfInternalComponents(componentIdentification, appSettings, mockJfrogService.Object, mockBomHelper.Object); + + // Assert + Assert.That(actual, Is.Not.Null); + } + + [Test] + public async Task GetJfrogRepoDetailsOfAComponent_ReturnsWithData_SuccessFully() + { + // Arrange + Component component = new Component() + { + Name = "securitycommunicationmanager", + Description = string.Empty, + Version = "2.6.5", + Purl = "pkg:conan/securitycommunicationmanager@2.6.5" + }; + var components = new List() { component }; + string[] repoList = { "internalrepo1", "internalrepo2" }; + CommonAppSettings appSettings = new(); + appSettings.Conan = new LCT.Common.Model.Config() { JfrogConanRepoList = repoList }; + AqlResult aqlResult = new() + { + Name = "index.json", + Path = "siemens-energy/securitycommunicationmanager/2.6.5/stable", + Repo = "internalrepo1" + }; + + List results = new List() { aqlResult }; + + Mock mockJfrogService = new Mock(); + Mock mockBomHelper = new Mock(); + mockBomHelper.Setup(m => m.GetListOfComponentsFromRepo(It.IsAny(), It.IsAny())) + .ReturnsAsync(results); + + // Act + ConanProcessor conanProcessor = new ConanProcessor(); + var actual = await conanProcessor.GetJfrogRepoDetailsOfAComponent( + components, appSettings, mockJfrogService.Object, mockBomHelper.Object); + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + + // Assert + Assert.That(actual, Is.Not.Null); + Assert.That(aqlResult.Repo, Is.EqualTo(reponameActual)); + } + + [Test] + public async Task GetArtifactoryRepoName_Conan_ReturnsNotFound_ReturnsFailure() + { + // Arrange + Component component = new Component() + { + Name = "securitycommunicationmanager", + Description = string.Empty, + Version = "2.6.5", + Purl = "pkg:conan/securitycommunicationmanager@2.6.5" + }; + var components = new List() { component }; + string[] repoList = { "internalrepo1", "internalrepo2" }; + CommonAppSettings appSettings = new(); + appSettings.Conan = new LCT.Common.Model.Config() { JfrogConanRepoList = repoList }; + AqlResult aqlResult = new() + { + Name = "index.json", + Path = "siemens-energy/securitycommunicationmanager/2.7.1/stable", + Repo = "internalrepo1" + }; + + List results = new() { aqlResult }; + + Mock mockJfrogService = new Mock(); + Mock mockBomHelper = new Mock(); + mockBomHelper.Setup(m => m.GetListOfComponentsFromRepo(It.IsAny(), It.IsAny())) + .ReturnsAsync(results); + + // Act + ConanProcessor conanProcessor = new ConanProcessor(); + var actual = await conanProcessor.GetJfrogRepoDetailsOfAComponent( + components, appSettings, mockJfrogService.Object, mockBomHelper.Object); + + var reponameActual = actual.First(x => x.Properties[0].Name == "internal:siemens:clearing:repo-url").Properties[0].Value; + + Assert.That("Not Found in JFrogRepo", Is.EqualTo(reponameActual)); + } + } +} diff --git a/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/conan.lock b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/conan.lock new file mode 100644 index 00000000..2a45747b --- /dev/null +++ b/src/LCT.PackageIdentifier.UTest/PackageIdentifierUTTestFiles/conan.lock @@ -0,0 +1,154 @@ +{ + "graph_lock": { + "nodes": { + "0": { + "options": "libcurl:static=None\nopenssl:static=False", + "requires": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16" + ], + "build_requires": [ + "17" + ], + "path": "conan\\conanfile\\linux-x86_64", + "context": "host" + }, + "1": { + "ref": "rapidjson/1.1.0-csc-01@siemens-energy/stable#6d624490b731387491675eebeff7ab66", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "85ca839500f017c06cd5c39b4ad50c5e", + "context": "host" + }, + "2": { + "ref": "libest/3.2.0-shared.5@siemens-energy/stable#0", + "options": "", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "3": { + "ref": "openssl/3.0.9-shared.3@siemens-energy/stable#0", + "options": "static=False", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "4": { + "ref": "sqlite/3.37.0-shared.1@siemens-energy/stable#0", + "options": "", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "5": { + "ref": "oss_mbedtls/2.28.2-shared@siemens-energy/stable#0", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "0", + "context": "host" + }, + "6": { + "ref": "mongoose/v7.11-csc-01@siemens-energy/stable#7084912b9de8ac5f93c2e8aa16c259a8", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "c4b1dedbd2bd5223337ce892732c693e", + "context": "host" + }, + "7": { + "ref": "basics/1.0.8@siemens-energy/stable#0", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "0", + "context": "host" + }, + "8": { + "ref": "osal/1.0.30@siemens-energy/stable#21e224648b08c0c356e2e60d0f143ae7", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "f5db57253fc37159485da5d9d37d4844", + "context": "host" + }, + "9": { + "ref": "SecurityBasics/2.10.2@siemens-energy/stable#0", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "0", + "context": "host" + }, + "10": { + "ref": "securityaccessmanager/2.2.6@siemens-energy/stable#7522ada1783555e22afd338d9bd03d60", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "493721829e0ba6c4cccb9304acc9ad71", + "context": "host" + }, + "11": { + "ref": "SecurityEventLogger/2.0.24@siemens-energy/stable#ddd584e3d5551e3ba568f07b95a9e6af", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "f32d263a0ba627319365f9ee41781589", + "context": "host" + }, + "12": { + "ref": "securitypkimanager/2.6.3@siemens-energy/stable#a8d80c7af932e2062513e94fd2001077", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "33612b4203a7c283c1dd56aa795863f6", + "context": "host" + }, + "13": { + "ref": "SecurityStorageManager/2.11.2@siemens-energy/stable#6c98465724cb00ab842f16a6a17c64b6", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "8f87b12afc830478491b566d2b7bf7e1", + "context": "host" + }, + "14": { + "ref": "securitycommunicationmanager/2.6.5@siemens-energy/stable#fbacb77f419f7c1dc2af769841266fba", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "dcd1c2048a844e785cb79e918452d56e", + "context": "host" + }, + "15": { + "ref": "openldap/2.6.4-shared-ossl3.1@siemens-energy/stable#0", + "options": "", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "16": { + "ref": "libcurl/7.87.0-shared-ossl3.1@siemens-energy/stable#0", + "options": "static=None", + "package_id": "49e7f59961e8ed9b7d1d33caa2e6d613b00b72a5", + "prev": "0", + "context": "host" + }, + "17": { + "ref": "googletest/1.8.0@siemens-energy/stable#0", + "options": "", + "package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", + "prev": "0", + "context": "host" + } + }, + "revisions_enabled": true + }, + "version": "0.4", + "profile_host": "[settings]\narch=x86_64\narch_build=x86_64\nbuild_type=Release\ncompiler=gcc\ncompiler.libcxx=libstdc++11\ncompiler.version=9.3\nos=Linux\nos_build=Linux\nswsign:os=Windows\nswsign:os_build=Windows\nswsign:compiler=Visual Studio\nswsign:compiler.version=15\nswsign:build_type=Release\nswsign:compiler.runtime=MD\nmbedtls_csc:os=Windows\nmbedtls_csc:os_build=Windows\nmbedtls_csc:compiler=Visual Studio\nmbedtls_csc:compiler.version=15\nmbedtls_csc:build_type=Release\nmbedtls_csc:compiler.runtime=MD\n[options]\nopenssl:static=False\n[build_requires]\n[env]\n" +} \ No newline at end of file From 204027ecae716d24933fdc7d5ee95a20e0c8c23b Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Wed, 27 Sep 2023 09:51:21 +0530 Subject: [PATCH 07/21] Updating the access modifier to public for the IsDevDependency method. --- src/LCT.PackageIdentifier/ConanProcessor.cs | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs index bf6bf145..0fc79b5d 100644 --- a/src/LCT.PackageIdentifier/ConanProcessor.cs +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -150,6 +150,18 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List lstComponentForBOM, re } } - private static bool IsDevDependency(ConanPackage component, ConanPackage rootNode, ref int noOfDevDependent) - { - var isDev = false; - if (rootNode.DevDependencies != null && rootNode.DevDependencies.Contains(component.Id)) - { - isDev = true; - noOfDevDependent++; - } - - return isDev; - } - private static bool IsInternalConanComponent(List aqlResultList, Component component) { string jfrogcomponentPath = $"{component.Name}/{component.Version}"; From a4f812c0e8874bddfa3b131bf4100608ed36a751 Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Wed, 27 Sep 2023 09:52:39 +0530 Subject: [PATCH 08/21] change the variable name to standard --- src/LCT.PackageIdentifier.UTest/ConanParserTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs index 9104925e..3607db2f 100644 --- a/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs +++ b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs @@ -28,7 +28,7 @@ public class ConanParserTests public void ParseLockFile_GivenAInputFilePath_ReturnsSuccess() { //Arrange - int expectednoofcomponents = 17; + int expectedNoOfcomponents = 17; string exePath = System.Reflection.Assembly.GetExecutingAssembly().Location; string outFolder = Path.GetDirectoryName(exePath); string packagefilepath = outFolder + @"\PackageIdentifierUTTestFiles"; @@ -49,7 +49,7 @@ public void ParseLockFile_GivenAInputFilePath_ReturnsSuccess() Bom listofcomponents = new ConanProcessor().ParsePackageFile(appSettings); //Assert - Assert.That(expectednoofcomponents, Is.EqualTo(listofcomponents.Components.Count), "Checks for no of components"); + Assert.That(expectedNoOfcomponents, Is.EqualTo(listofcomponents.Components.Count), "Checks for no of components"); } @@ -76,8 +76,8 @@ public void ParseLockFile_GivenAInputFilePath_ReturnDevDependentComp() //Act Bom listofcomponents = new ConanProcessor().ParsePackageFile(appSettings); - - var IsDevDependency = listofcomponents.Components.Find(a => a.Name == "googletest").Properties[0].Value; + var IsDevDependency = listofcomponents.Components.Find(a => a.Name == "googletest") + .Properties.First(x => x.Name == "internal:siemens:clearing:development").Value; //Assert Assert.That(IsDev, Is.EqualTo(IsDevDependency), "Checks if Dev Dependency Component or not"); From bf50e20baf50b20ee4ec964bb96f2f9f55f25188 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Fri, 29 Sep 2023 18:04:27 +0530 Subject: [PATCH 09/21] Update --- .../LCT.PackageIdentifier.UTest.csproj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj b/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj index 651788d3..e2a08c85 100644 --- a/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj +++ b/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj @@ -55,6 +55,9 @@ + + Always + Always From cd9a023ed781a1ad9a80ca882c0cfb0ae92144d9 Mon Sep 17 00:00:00 2001 From: Aditya Narayan <57411194+adityanarayanp@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:35:17 +0530 Subject: [PATCH 10/21] Adding the Conan settings in the configuration file --- src/LCT.Common/appSettings.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/LCT.Common/appSettings.json b/src/LCT.Common/appSettings.json index e00c4e32..57ef12f3 100644 --- a/src/LCT.Common/appSettings.json +++ b/src/LCT.Common/appSettings.json @@ -74,5 +74,14 @@ "" //This should be the release pypi in JFrog ], "ExcludedComponents": [] + }, + "Conan": { + "Include": [ "conan.lock" ], + "Exclude": [], + "JfrogConanRepoList": [ + "", //This is a mirror repo for conan in JFrog + "" //This should be the release repo in JFrog + ], + "ExcludedComponents": [] } } From 77cee4575a62b507fd3999b9a6f7520abd2199e3 Mon Sep 17 00:00:00 2001 From: karthika Date: Fri, 6 Oct 2023 10:48:28 +0530 Subject: [PATCH 11/21] Documentation Changes --- CA.nuspec | 4 ++-- doc/UsageDoc/CA_UsageDocument.md | 32 ++++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/CA.nuspec b/CA.nuspec index c6c712e3..41415181 100644 --- a/CA.nuspec +++ b/CA.nuspec @@ -4,7 +4,7 @@ continuous-clearing - 4.0.0 + 6.0.0 Siemens AG continuous-clearing contributors https://github.com/siemens/continuous-clearing @@ -13,7 +13,7 @@ false The License clearing tool helps the Project Manager/Developer, to reduce the manual effort and enable the faster license clearing process, - by automatically identifying the third party oss components used in their project(i.e., npm, nuget, maven and Debian type) and it creates them in the sw360 and fossology + by automatically identifying the third party oss components used in their project(i.e., npm, nuget, maven, python, conan and Debian type) and it creates them in the sw360 and fossology for clearing license diff --git a/doc/UsageDoc/CA_UsageDocument.md b/doc/UsageDoc/CA_UsageDocument.md index 7280c9e4..434ea3c2 100644 --- a/doc/UsageDoc/CA_UsageDocument.md +++ b/doc/UsageDoc/CA_UsageDocument.md @@ -46,20 +46,20 @@ # Introduction -The Continuous Clearing Tool helps the Project Manager/Developer to automate the sw360 clearing process of 3rd party components. This tool scans and identifies the third-party components used in a NPM, NUGET, MAVEN and Debian projects and makes an entry in SW360, if it is not present. Continuous Clearing Tool links the components to the respective project and creates job for code scan in FOSSology.The output is an SBOM file which has a nested description of software artifact components and metadata. +The Continuous Clearing Tool helps the Project Manager/Developer to automate the sw360 clearing process of 3rd party components. This tool scans and identifies the third-party components used in a NPM, NUGET, MAVEN, PYTHON, CONAN and Debian projects and makes an entry in SW360, if it is not present. Continuous Clearing Tool links the components to the respective project and creates job for code scan in FOSSology.The output is an SBOM file which has a nested description of software artifact components and metadata. Continuous Clearing Tool reduces the effort in creating components in SW360 and identifying the matching source codes from the public repository. Tool eliminates the manual error while creating component and identifying correct version of source code from public repository. Continuous Clearing Tool harmonize the creation of 3P components in SW360 by filling necessary information. # Continuous Clearing Tool workflow diagram - Package Identifier - - [NPM/NUGET/MAVEN](../usagedocimg/packageIdentifiernpmnuget.PNG) + - [NPM/NUGET/MAVEN/PYTHON/CONAN](../usagedocimg/packageIdentifiernpmnuget.PNG) - [Debian](../usagedocimg/packageIdentifierdebian.PNG) - SW360 Package Creator - - [NPM/NUGET/MAVEN](../usagedocimg/packageCreatirnpmnuget.PNG) + - [NPM/NUGET/MAVEN/PYTHON/CONAN](../usagedocimg/packageCreatirnpmnuget.PNG) - [Debian](../usagedocimg/packagecreatordebian.PNG) - Artifactory Uploader - - [NPM/NUGET/MAVEN](../usagedocimg/artifactoryuploader.PNG) + - [NPM/NUGET/MAVEN/PYTHON/CONAN](../usagedocimg/artifactoryuploader.PNG) # Prerequisite @@ -158,13 +158,19 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and mvn clean install -DskipTests=true - - **Project Type :** **Python** + - **Project Type :** **Python** * Input file repository should contain **poetry.lock** file. `Note : Python package support in clearing tool is currently only for SBOM discovery and classification.Component Creation and Source code identification is not supported currently` + + - **Project Type :** **Conan** + + * Input file repository should contain **conan.lock** file. + + `Note : Conan package support in clearing tool is currently only for SBOM discovery and classification.Component Creation and Source code identification is not supported currently` - - **Project Type :** **Debian** + - **Project Type :** **Debian** **Note** : below steps is required only if you have `tar` file to process , otherwise you can keep `CycloneDx.json` file in the InputDirectory. * Create `InputImage` directory for keeping `tar` images and `InputDirectory` for resulted file storing . @@ -258,8 +264,22 @@ Continuous Clearing Tool reduces the effort in creating components in SW360 and "Python": { "Include": [ "poetry.lock", "*.cdx.json" ], "Exclude": [], + "JfrogPythonRepoList": [ + "", + "",//This should be the release repo in JFrog + ], + "ExcludedComponents": [] + }, + "Conan": { + "Include": [ "conan.lock"], + "Exclude": [], + "JfrogConanRepoList": [ + "", + "", + ], "ExcludedComponents": [] } + } ``` From dddca76bd1eccbecd9474fc9a702085a2f8a912b Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Tue, 24 Oct 2023 19:55:24 +0530 Subject: [PATCH 12/21] Updating to handle transitive packages in all nodes in conan --- .../ConanParserTests.cs | 4 ++-- .../LCT.PackageIdentifier.UTest.csproj | 3 +++ src/LCT.PackageIdentifier/ConanProcessor.cs | 19 ++++++++++++++----- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs index 3607db2f..295755ef 100644 --- a/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs +++ b/src/LCT.PackageIdentifier.UTest/ConanParserTests.cs @@ -118,10 +118,10 @@ public void IsDevDependent_GivenListOfDevComponents_ReturnsSuccess() { //Arrange var conanPackage = new ConanPackage() {Id = "10"}; - var rootNode = new ConanPackage() {DevDependencies = new List { "10", "11", "12" }}; + var buildNodeIds = new List { "10", "11", "12" }; var noOfDevDependent = 0; //Act - bool actual = ConanProcessor.IsDevDependency(conanPackage, rootNode, ref noOfDevDependent); + bool actual = ConanProcessor.IsDevDependency(conanPackage, buildNodeIds, ref noOfDevDependent); //Assert Assert.That(true, Is.EqualTo(actual), "Component is a dev dependent"); diff --git a/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj b/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj index 651788d3..e2a08c85 100644 --- a/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj +++ b/src/LCT.PackageIdentifier.UTest/LCT.PackageIdentifier.UTest.csproj @@ -55,6 +55,9 @@ + + Always + Always diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs index 0fc79b5d..d4f87126 100644 --- a/src/LCT.PackageIdentifier/ConanProcessor.cs +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -23,7 +23,6 @@ using System.Security; using System.Threading.Tasks; - namespace LCT.PackageIdentifier { @@ -150,10 +149,10 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List buildNodeIds, ref int noOfDevDependent) { var isDev = false; - if (rootNode.DevDependencies != null && rootNode.DevDependencies.Contains(component.Id)) + if (buildNodeIds!= null && buildNodeIds.Contains(component.Id)) { isDev = true; noOfDevDependent++; @@ -238,7 +237,8 @@ private static void GetPackagesForBom(ref List lstComponentForBOM, re throw new ArgumentNullException(nameof(nodePackages), "Dependency(requires) node name details not present in the root node."); } - // Ignoring the root node as it is the package information node. + // Ignoring the root node as it is the package information node and we are anyways considering all + // nodes in the lock file. foreach (var component in nodePackages.Skip(1)) { BomCreator.bomKpiData.ComponentsinPackageLockJsonFile += 1; @@ -253,7 +253,8 @@ private static void GetPackagesForBom(ref List lstComponentForBOM, re Component components = new Component(); // dev components are not ignored and added as a part of SBOM - if (IsDevDependency(component, rootNode, ref noOfDevDependent)) + var buildNodeIds = GetBuildNodeIds(nodePackages); + if (IsDevDependency(component, buildNodeIds, ref noOfDevDependent)) { isdev.Value = "true"; } @@ -279,6 +280,14 @@ private static void GetPackagesForBom(ref List lstComponentForBOM, re } } + private static List GetBuildNodeIds(List nodePackages) + { + return nodePackages + .Where(y => y.DevDependencies != null) + .SelectMany(y => y.DevDependencies) + .ToList(); + } + private static bool IsInternalConanComponent(List aqlResultList, Component component) { string jfrogcomponentPath = $"{component.Name}/{component.Version}"; From 12395ab53ad6a5adf5facb93421578d59c249fa5 Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Thu, 26 Oct 2023 10:52:52 +0530 Subject: [PATCH 13/21] SbOm Reading changes --- src/LCT.Common/CommonHelper.cs | 14 ++++ src/LCT.Common/CycloneDXBomParser.cs | 4 +- src/LCT.PackageIdentifier/ConanProcessor.cs | 74 +++++++++++++++++--- src/LCT.PackageIdentifier/NugetProcessor.cs | 15 +--- src/LCT.PackageIdentifier/PythonProcessor.cs | 2 +- 5 files changed, 81 insertions(+), 28 deletions(-) diff --git a/src/LCT.Common/CommonHelper.cs b/src/LCT.Common/CommonHelper.cs index 914dda8b..06b161b4 100644 --- a/src/LCT.Common/CommonHelper.cs +++ b/src/LCT.Common/CommonHelper.cs @@ -5,6 +5,7 @@ // -------------------------------------------------------------------------------------------------------------------- using CycloneDX.Models; +using LCT.Common.Constants; using LCT.Common.Model; using log4net; using log4net.Core; @@ -228,5 +229,18 @@ public static bool ComponentPropertyCheck(Component component, string constant) } return component.Properties.Exists(x => x.Name == constant); } + + public static void GetDetailsforManuallyAdded(List componentsForBOM, List listComponentForBOM) + { + foreach (var component in componentsForBOM) + { + component.Properties = new List(); + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.ManullayAdded }; + component.Properties.Add(isDev); + component.Properties.Add(identifierType); + listComponentForBOM.Add(component); + } + } } } diff --git a/src/LCT.Common/CycloneDXBomParser.cs b/src/LCT.Common/CycloneDXBomParser.cs index 7cb0bada..7c143bd4 100644 --- a/src/LCT.Common/CycloneDXBomParser.cs +++ b/src/LCT.Common/CycloneDXBomParser.cs @@ -72,8 +72,8 @@ public static Bom ExtractSBOMDetailsFromTemplate(Bom template) } //Taking SBOM Template Metadata - bom.Metadata = template?.Metadata; - bom.Dependencies = template?.Dependencies; + bom.Metadata = template.Metadata; + bom.Dependencies = template.Dependencies; return bom; } diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs index d4f87126..b35a7d3b 100644 --- a/src/LCT.PackageIdentifier/ConanProcessor.cs +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -54,9 +54,9 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) Bom bom = new Bom(); int totalComponentsIdentified = 0; - ParsingInputFileForBOM(appSettings, ref componentsForBOM); - totalComponentsIdentified = componentsForBOM.Count; + ParsingInputFileForBOM(appSettings, ref componentsForBOM, ref bom); + totalComponentsIdentified = componentsForBOM.Count; componentsForBOM = GetExcludedComponentsList(componentsForBOM); componentsForBOM = componentsForBOM.Distinct(new ComponentEqualityComparer()).ToList(); @@ -149,7 +149,7 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List buildNodeIds, ref int noOfDevDependent) + public static bool IsDevDependency(ConanPackage component, ConanPackage rootNode, ref int noOfDevDependent) { var isDev = false; if (buildNodeIds!= null && buildNodeIds.Contains(component.Id)) @@ -164,18 +164,48 @@ public static bool IsDevDependency(ConanPackage component, List buildNod #endregion #region private methods - private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List componentsForBOM) + private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List listComponentForBOM, ref Bom bom) { List configFiles; + List componentsForBOM = new List(); configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Conan); foreach (string filepath in configFiles) { - Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); - var components = ParsePackageLockJson(filepath, appSettings); - AddingIdentifierType(components, "PackageFile"); - componentsForBOM.AddRange(components); + if (filepath.ToLower().EndsWith("conan.lock")) + { + Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); + var components = ParsePackageLockJson(filepath, appSettings); + AddingIdentifierType(components, "PackageFile"); + componentsForBOM.AddRange(components); + } + else if (filepath.EndsWith(FileConstant.CycloneDXFileExtension) && !filepath.EndsWith(FileConstant.SBOMTemplateFileExtension)) + { + Logger.Debug($"ParsingInputFileForBOM():Found as CycloneDXFile"); + bom = cycloneDXBomParser.ParseCycloneDXBom(filepath); + CheckValidComponentsForProjectType(bom.Components, appSettings.ProjectType); + componentsForBOM.AddRange(bom.Components); + CommonHelper.GetDetailsforManuallyAdded(componentsForBOM, listComponentForBOM); + } + } + + int initialCount = componentsForBOM.Count; + GetDistinctComponentList(ref componentsForBOM); + BomCreator.bomKpiData.DuplicateComponents = initialCount - listComponentForBOM.Count; + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = listComponentForBOM.Count; + BomCreator.bomKpiData.DevDependentComponents = listComponentForBOM.Count(s => s.Properties[0].Value == "true"); + bom.Components = listComponentForBOM; + + if (File.Exists(appSettings.CycloneDxSBomTemplatePath) && appSettings.CycloneDxSBomTemplatePath.EndsWith(FileConstant.SBOMTemplateFileExtension)) + { + //Adding Template Component Details + Bom templateDetails; + templateDetails = ExtractSBOMDetailsFromTemplate(cycloneDXBomParser.ParseCycloneDXBom(appSettings.CycloneDxSBomTemplatePath)); + CheckValidComponentsForProjectType(templateDetails.Components, appSettings.ProjectType); + SbomTemplate.AddComponentDetails(bom.Components, templateDetails); } + + bom = RemoveExcludedComponents(appSettings, bom); } private List ParsePackageLockJson(string filepath, CommonAppSettings appSettings) @@ -232,11 +262,11 @@ private List ParsePackageLockJson(string filepath, CommonAppSettings private static void GetPackagesForBom(ref List lstComponentForBOM, ref int noOfDevDependent, List nodePackages) { var rootNode = nodePackages.FirstOrDefault(); - if (!rootNode.Dependencies.Any() || rootNode.Dependencies == null) + if (rootNode == null || !rootNode.Dependencies.Any() || rootNode.Dependencies == null) { throw new ArgumentNullException(nameof(nodePackages), "Dependency(requires) node name details not present in the root node."); } - + // Ignoring the root node as it is the package information node and we are anyways considering all // nodes in the lock file. foreach (var component in nodePackages.Skip(1)) @@ -323,7 +353,7 @@ private static List GetExcludedComponentsList(List compone else { BomCreator.bomKpiData.ComponentsExcluded++; - Logger.Debug($"GetExcludedComponentsList():InvalidComponent For CONAN : Component Details : {componentsInfo?.Name} @ {componentsInfo?.Version} @ {componentsInfo?.Purl}"); + Logger.Debug($"GetExcludedComponentsList():InvalidComponent For CONAN : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}"); } } return components; @@ -355,6 +385,28 @@ private static void AddingIdentifierType(List components, string iden } } + private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) + { + List componentForBOM = cycloneDXBOM.Components.ToList(); + int noOfExcludedComponents = 0; + if (appSettings.Conan.ExcludedComponents != null) + { + componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Conan.ExcludedComponents, ref noOfExcludedComponents); + BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; + } + cycloneDXBOM.Components = componentForBOM; + return cycloneDXBOM; + } + + private static void GetDistinctComponentList(ref List listofComponents) + { + int initialCount = listofComponents.Count; + listofComponents = listofComponents.GroupBy(x => new { x.Name, x.Version, x.Purl }).Select(y => y.First()).ToList(); + + if (listofComponents.Count != initialCount) + BomCreator.bomKpiData.DuplicateComponents = initialCount - listofComponents.Count; + } + #endregion } } diff --git a/src/LCT.PackageIdentifier/NugetProcessor.cs b/src/LCT.PackageIdentifier/NugetProcessor.cs index 9c70f2e8..e5451da0 100644 --- a/src/LCT.PackageIdentifier/NugetProcessor.cs +++ b/src/LCT.PackageIdentifier/NugetProcessor.cs @@ -383,7 +383,7 @@ private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List componentsForBOM, List listComponentForBOM) - { - foreach (var component in componentsForBOM) - { - component.Properties = new List(); - Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; - Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.ManullayAdded }; - component.Properties.Add(isDev); - component.Properties.Add(identifierType); - listComponentForBOM.Add(component); - } - } - private static void ConvertToCycloneDXModel(List listComponentForBOM, List listofComponents, List dependencies) { diff --git a/src/LCT.PackageIdentifier/PythonProcessor.cs b/src/LCT.PackageIdentifier/PythonProcessor.cs index 6abc2d9f..7df741c9 100644 --- a/src/LCT.PackageIdentifier/PythonProcessor.cs +++ b/src/LCT.PackageIdentifier/PythonProcessor.cs @@ -48,7 +48,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) foreach (string config in configFiles) { - if (config.EndsWith("poetry.lock")) + if (config.ToLower().EndsWith("poetry.lock")) { listofComponents.AddRange(ExtractDetailsForPoetryLockfile(config, dependencies)); } From 3c1ac4fc37c8abb91fa3f69588abf018ed27fc55 Mon Sep 17 00:00:00 2001 From: Aditya Narayan Date: Thu, 26 Oct 2023 15:04:18 +0530 Subject: [PATCH 14/21] Adding dependencies to the SBOM --- src/LCT.PackageIdentifier/ConanProcessor.cs | 111 ++++++++---------- .../Model/ConanPackage.cs | 2 +- 2 files changed, 47 insertions(+), 66 deletions(-) diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs index b35a7d3b..1d3ed906 100644 --- a/src/LCT.PackageIdentifier/ConanProcessor.cs +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -52,11 +52,12 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) { List componentsForBOM = new List(); Bom bom = new Bom(); + List dependencies = new List(); int totalComponentsIdentified = 0; - ParsingInputFileForBOM(appSettings, ref componentsForBOM, ref bom); - + ParsingInputFileForBOM(appSettings, ref componentsForBOM, ref dependencies); totalComponentsIdentified = componentsForBOM.Count; + componentsForBOM = GetExcludedComponentsList(componentsForBOM); componentsForBOM = componentsForBOM.Distinct(new ComponentEqualityComparer()).ToList(); @@ -73,6 +74,7 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) } } bom.Components = componentsForBOM; + bom.Dependencies = dependencies; Logger.Debug($"ParsePackageFile():End"); return bom; } @@ -149,10 +151,10 @@ public async Task> GetJfrogRepoDetailsOfAComponent(List buildNodeIds, ref int noOfDevDependent) { var isDev = false; - if (buildNodeIds!= null && buildNodeIds.Contains(component.Id)) + if (buildNodeIds != null && buildNodeIds.Contains(component.Id)) { isDev = true; noOfDevDependent++; @@ -164,51 +166,23 @@ public static bool IsDevDependency(ConanPackage component, ConanPackage rootNode #endregion #region private methods - private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List listComponentForBOM, ref Bom bom) + private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List componentsForBOM, ref List dependenciesForBom) { List configFiles; - List componentsForBOM = new List(); + List dependencies = new List(); configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Conan); foreach (string filepath in configFiles) { - if (filepath.ToLower().EndsWith("conan.lock")) - { - Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); - var components = ParsePackageLockJson(filepath, appSettings); - AddingIdentifierType(components, "PackageFile"); - componentsForBOM.AddRange(components); - } - else if (filepath.EndsWith(FileConstant.CycloneDXFileExtension) && !filepath.EndsWith(FileConstant.SBOMTemplateFileExtension)) - { - Logger.Debug($"ParsingInputFileForBOM():Found as CycloneDXFile"); - bom = cycloneDXBomParser.ParseCycloneDXBom(filepath); - CheckValidComponentsForProjectType(bom.Components, appSettings.ProjectType); - componentsForBOM.AddRange(bom.Components); - CommonHelper.GetDetailsforManuallyAdded(componentsForBOM, listComponentForBOM); - } + Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); + var components = ParsePackageLockJson(filepath, appSettings, ref dependencies); + AddingIdentifierType(components, "PackageFile"); + componentsForBOM.AddRange(components); + dependenciesForBom.AddRange(dependencies); } - - int initialCount = componentsForBOM.Count; - GetDistinctComponentList(ref componentsForBOM); - BomCreator.bomKpiData.DuplicateComponents = initialCount - listComponentForBOM.Count; - BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = listComponentForBOM.Count; - BomCreator.bomKpiData.DevDependentComponents = listComponentForBOM.Count(s => s.Properties[0].Value == "true"); - bom.Components = listComponentForBOM; - - if (File.Exists(appSettings.CycloneDxSBomTemplatePath) && appSettings.CycloneDxSBomTemplatePath.EndsWith(FileConstant.SBOMTemplateFileExtension)) - { - //Adding Template Component Details - Bom templateDetails; - templateDetails = ExtractSBOMDetailsFromTemplate(cycloneDXBomParser.ParseCycloneDXBom(appSettings.CycloneDxSBomTemplatePath)); - CheckValidComponentsForProjectType(templateDetails.Components, appSettings.ProjectType); - SbomTemplate.AddComponentDetails(bom.Components, templateDetails); - } - - bom = RemoveExcludedComponents(appSettings, bom); } - private List ParsePackageLockJson(string filepath, CommonAppSettings appSettings) + private List ParsePackageLockJson(string filepath, CommonAppSettings appSettings, ref List dependencies) { List lstComponentForBOM = new List(); int noOfDevDependent = 0; @@ -238,6 +212,8 @@ private List ParsePackageLockJson(string filepath, CommonAppSettings } + GetDependecyDetails(lstComponentForBOM, nodePackages, dependencies); + BomCreator.bomKpiData.DevDependentComponents += noOfDevDependent; } catch (JsonReaderException ex) @@ -259,10 +235,38 @@ private List ParsePackageLockJson(string filepath, CommonAppSettings return lstComponentForBOM; } + private static void GetDependecyDetails(List componentsForBOM, List nodePackages, List dependencies) + { + foreach (Component component in componentsForBOM) + { + var node = nodePackages.Find(x => x.Reference.Contains($"{component.Name}/{component.Version}")); + var dependencyNodes = new List(); + if (node.Dependencies != null && node.Dependencies.Count > 0) + { + dependencyNodes.AddRange(nodePackages.Where(x => node.Dependencies.Contains(x.Id)).ToList()); + } + if (node.DevDependencies != null && node.DevDependencies.Count > 0) + { + dependencyNodes.AddRange(nodePackages.Where(x => node.DevDependencies.Contains(x.Id)).ToList()); + } + var dependency = new Dependency(); + var subDependencies = componentsForBOM.Where(x => dependencyNodes.Exists(y => y.Reference.Contains($"{x.Name}/{x.Version}"))) + .Select(x => new Dependency { Ref = x.Purl }).ToList(); + + dependency.Ref = component.Purl; + dependency.Dependencies = subDependencies; + + if(subDependencies.Count > 0) + { + dependencies.Add(dependency); + } + } + } + private static void GetPackagesForBom(ref List lstComponentForBOM, ref int noOfDevDependent, List nodePackages) { var rootNode = nodePackages.FirstOrDefault(); - if (rootNode == null || !rootNode.Dependencies.Any() || rootNode.Dependencies == null) + if (!rootNode.Dependencies.Any() || rootNode.Dependencies == null) { throw new ArgumentNullException(nameof(nodePackages), "Dependency(requires) node name details not present in the root node."); } @@ -303,7 +307,6 @@ private static void GetPackagesForBom(ref List lstComponentForBOM, re components.Purl = $"{ApiConstant.ConanExternalID}{components.Name}@{components.Version}"; components.BomRef = $"{ApiConstant.ConanExternalID}{components.Name}@{components.Version}"; - components.Properties = new List(); components.Properties.Add(isdev); lstComponentForBOM.Add(components); @@ -353,7 +356,7 @@ private static List GetExcludedComponentsList(List compone else { BomCreator.bomKpiData.ComponentsExcluded++; - Logger.Debug($"GetExcludedComponentsList():InvalidComponent For CONAN : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}"); + Logger.Debug($"GetExcludedComponentsList():InvalidComponent For CONAN : Component Details : {componentsInfo?.Name} @ {componentsInfo?.Version} @ {componentsInfo?.Purl}"); } } return components; @@ -385,28 +388,6 @@ private static void AddingIdentifierType(List components, string iden } } - private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) - { - List componentForBOM = cycloneDXBOM.Components.ToList(); - int noOfExcludedComponents = 0; - if (appSettings.Conan.ExcludedComponents != null) - { - componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Conan.ExcludedComponents, ref noOfExcludedComponents); - BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; - } - cycloneDXBOM.Components = componentForBOM; - return cycloneDXBOM; - } - - private static void GetDistinctComponentList(ref List listofComponents) - { - int initialCount = listofComponents.Count; - listofComponents = listofComponents.GroupBy(x => new { x.Name, x.Version, x.Purl }).Select(y => y.First()).ToList(); - - if (listofComponents.Count != initialCount) - BomCreator.bomKpiData.DuplicateComponents = initialCount - listofComponents.Count; - } - #endregion } } diff --git a/src/LCT.PackageIdentifier/Model/ConanPackage.cs b/src/LCT.PackageIdentifier/Model/ConanPackage.cs index 843dd6f0..1498dfb7 100644 --- a/src/LCT.PackageIdentifier/Model/ConanPackage.cs +++ b/src/LCT.PackageIdentifier/Model/ConanPackage.cs @@ -17,7 +17,7 @@ public class ConanPackage { public string Id { get; set; } [JsonProperty("ref")] - public string Reference { get; set; } + public string Reference { get; set; } = string.Empty; [JsonProperty("requires")] public List Dependencies { get; set; } [JsonProperty("build_requires")] From eda934c2e4da64ed392993e66d762e52390355b2 Mon Sep 17 00:00:00 2001 From: sumanthkb44 <84563853+sumanthkb44@users.noreply.github.com> Date: Tue, 31 Oct 2023 11:31:03 +0530 Subject: [PATCH 15/21] Sbom reading changes (#100) * Sbom reading changes * Update --------- Co-authored-by: Sumanth K B --- src/LCT.PackageIdentifier/ConanProcessor.cs | 116 +++++++++++++++----- src/LCT.Services/Sw360Service.cs | 2 +- 2 files changed, 89 insertions(+), 29 deletions(-) diff --git a/src/LCT.PackageIdentifier/ConanProcessor.cs b/src/LCT.PackageIdentifier/ConanProcessor.cs index 1d3ed906..04065bfd 100644 --- a/src/LCT.PackageIdentifier/ConanProcessor.cs +++ b/src/LCT.PackageIdentifier/ConanProcessor.cs @@ -50,18 +50,15 @@ public ConanProcessor() #region public methods public Bom ParsePackageFile(CommonAppSettings appSettings) { - List componentsForBOM = new List(); + List componentsForBOM; Bom bom = new Bom(); - List dependencies = new List(); - int totalComponentsIdentified = 0; - ParsingInputFileForBOM(appSettings, ref componentsForBOM, ref dependencies); - totalComponentsIdentified = componentsForBOM.Count; + ParsingInputFileForBOM(appSettings, ref bom); + componentsForBOM = bom.Components; componentsForBOM = GetExcludedComponentsList(componentsForBOM); componentsForBOM = componentsForBOM.Distinct(new ComponentEqualityComparer()).ToList(); - BomCreator.bomKpiData.DuplicateComponents = totalComponentsIdentified - componentsForBOM.Count; var componentsWithMultipleVersions = componentsForBOM.GroupBy(s => s.Name) .Where(g => g.Count() > 1).SelectMany(g => g).ToList(); @@ -73,8 +70,8 @@ public Bom ParsePackageFile(CommonAppSettings appSettings) Logger.Warn($"Component Name : {item.Name}\nComponent Version : {item.Version}\nPackage Found in : {item.Description}\n"); } } + bom.Components = componentsForBOM; - bom.Dependencies = dependencies; Logger.Debug($"ParsePackageFile():End"); return bom; } @@ -166,31 +163,67 @@ public static bool IsDevDependency(ConanPackage component, List buildNod #endregion #region private methods - private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref List componentsForBOM, ref List dependenciesForBom) + private void ParsingInputFileForBOM(CommonAppSettings appSettings, ref Bom bom) { List configFiles; List dependencies = new List(); + List componentsForBOM = new List(); configFiles = FolderScanner.FileScanner(appSettings.PackageFilePath, appSettings.Conan); foreach (string filepath in configFiles) { - Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); - var components = ParsePackageLockJson(filepath, appSettings, ref dependencies); - AddingIdentifierType(components, "PackageFile"); - componentsForBOM.AddRange(components); - dependenciesForBom.AddRange(dependencies); + if (filepath.ToLower().EndsWith("conan.lock")) + { + Logger.Debug($"ParsingInputFileForBOM():FileName: " + filepath); + var components = ParsePackageLockJson(filepath, ref dependencies); + AddingIdentifierType(components, "PackageFile"); + componentsForBOM.AddRange(components); + } + else if (filepath.EndsWith(FileConstant.CycloneDXFileExtension) && !filepath.EndsWith(FileConstant.SBOMTemplateFileExtension)) + { + Logger.Debug($"ParsingInputFileForBOM():Found as CycloneDXFile"); + bom = cycloneDXBomParser.ParseCycloneDXBom(filepath); + CheckValidComponentsForProjectType(bom.Components, appSettings.ProjectType); + componentsForBOM.AddRange(bom.Components); + GetDetailsforManuallyAddedComp(componentsForBOM); + } + } + + int initialCount = componentsForBOM.Count; + GetDistinctComponentList(ref componentsForBOM); + BomCreator.bomKpiData.DuplicateComponents = initialCount - componentsForBOM.Count; + BomCreator.bomKpiData.ComponentsinPackageLockJsonFile = componentsForBOM.Count; + bom.Components = componentsForBOM; + + if (bom.Dependencies != null) + { + bom.Dependencies.AddRange(dependencies); + } + else + { + bom.Dependencies = dependencies; } + + if (File.Exists(appSettings.CycloneDxSBomTemplatePath) && appSettings.CycloneDxSBomTemplatePath.EndsWith(FileConstant.SBOMTemplateFileExtension)) + { + //Adding Template Component Details + Bom templateDetails; + templateDetails = ExtractSBOMDetailsFromTemplate(cycloneDXBomParser.ParseCycloneDXBom(appSettings.CycloneDxSBomTemplatePath)); + CheckValidComponentsForProjectType(templateDetails.Components, appSettings.ProjectType); + SbomTemplate.AddComponentDetails(bom.Components, templateDetails); + } + + bom = RemoveExcludedComponents(appSettings, bom); } - private List ParsePackageLockJson(string filepath, CommonAppSettings appSettings, ref List dependencies) + private static List ParsePackageLockJson(string filepath, ref List dependencies) { List lstComponentForBOM = new List(); int noOfDevDependent = 0; - int noOfExcludedComponents = 0; + try { string jsonContent = File.ReadAllText(filepath); - var jsonDeserialized = JObject.Parse(jsonContent); var nodes = jsonDeserialized["graph_lock"]["nodes"]; @@ -205,13 +238,6 @@ private List ParsePackageLockJson(string filepath, CommonAppSettings GetPackagesForBom(ref lstComponentForBOM, ref noOfDevDependent, nodePackages); - if (appSettings.Conan.ExcludedComponents != null) - { - lstComponentForBOM = CommonHelper.RemoveExcludedComponents(lstComponentForBOM, appSettings.Conan.ExcludedComponents, ref noOfExcludedComponents); - BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; - - } - GetDependecyDetails(lstComponentForBOM, nodePackages, dependencies); BomCreator.bomKpiData.DevDependentComponents += noOfDevDependent; @@ -238,7 +264,7 @@ private List ParsePackageLockJson(string filepath, CommonAppSettings private static void GetDependecyDetails(List componentsForBOM, List nodePackages, List dependencies) { foreach (Component component in componentsForBOM) - { + { var node = nodePackages.Find(x => x.Reference.Contains($"{component.Name}/{component.Version}")); var dependencyNodes = new List(); if (node.Dependencies != null && node.Dependencies.Count > 0) @@ -255,8 +281,8 @@ private static void GetDependecyDetails(List componentsForBOM, List 0) + + if (subDependencies.Count > 0) { dependencies.Add(dependency); } @@ -356,7 +382,7 @@ private static List GetExcludedComponentsList(List compone else { BomCreator.bomKpiData.ComponentsExcluded++; - Logger.Debug($"GetExcludedComponentsList():InvalidComponent For CONAN : Component Details : {componentsInfo?.Name} @ {componentsInfo?.Version} @ {componentsInfo?.Purl}"); + Logger.Debug($"GetExcludedComponentsList():InvalidComponent For CONAN : Component Details : {componentsInfo.Name} @ {componentsInfo.Version} @ {componentsInfo.Purl}"); } } return components; @@ -388,6 +414,40 @@ private static void AddingIdentifierType(List components, string iden } } + private static void GetDistinctComponentList(ref List listofComponents) + { + int initialCount = listofComponents.Count; + listofComponents = listofComponents.GroupBy(x => new { x.Name, x.Version, x.Purl }).Select(y => y.First()).ToList(); + + if (listofComponents.Count != initialCount) + BomCreator.bomKpiData.DuplicateComponents = initialCount - listofComponents.Count; + } + + private static Bom RemoveExcludedComponents(CommonAppSettings appSettings, Bom cycloneDXBOM) + { + List componentForBOM = cycloneDXBOM.Components.ToList(); + int noOfExcludedComponents = 0; + if (appSettings.Conan.ExcludedComponents != null) + { + componentForBOM = CommonHelper.RemoveExcludedComponents(componentForBOM, appSettings.Conan.ExcludedComponents, ref noOfExcludedComponents); + BomCreator.bomKpiData.ComponentsExcluded += noOfExcludedComponents; + } + cycloneDXBOM.Components = componentForBOM; + return cycloneDXBOM; + } + + private static void GetDetailsforManuallyAddedComp(List componentsForBOM) + { + foreach (var component in componentsForBOM) + { + component.Properties = new List(); + Property isDev = new() { Name = Dataconstant.Cdx_IsDevelopment, Value = "false" }; + Property identifierType = new() { Name = Dataconstant.Cdx_IdentifierType, Value = Dataconstant.ManullayAdded }; + component.Properties.Add(isDev); + component.Properties.Add(identifierType); + } + } + #endregion } -} +} \ No newline at end of file diff --git a/src/LCT.Services/Sw360Service.cs b/src/LCT.Services/Sw360Service.cs index 22a448e3..7bb77a9d 100644 --- a/src/LCT.Services/Sw360Service.cs +++ b/src/LCT.Services/Sw360Service.cs @@ -67,7 +67,7 @@ public async Task> GetAvailableReleasesInSw360(List if (modelMappedObject != null) { - availableComponentsList = await GetAvailableComponenentsList(modelMappedObject?.Embedded?.Sw360Releases, listOfComponentsToBom); + availableComponentsList = await GetAvailableComponenentsList(modelMappedObject.Embedded?.Sw360Releases, listOfComponentsToBom); } } catch (HttpRequestException ex) From 2bdf6d4cf1d9abe273a37b2f3cf49c1a056acfea Mon Sep 17 00:00:00 2001 From: Aditya Narayan <57411194+adityanarayanp@users.noreply.github.com> Date: Thu, 2 Nov 2023 19:25:03 +0530 Subject: [PATCH 16/21] setting version in CA.nuspec to 5.1.0 --- CA.nuspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CA.nuspec b/CA.nuspec index 41415181..438a520f 100644 --- a/CA.nuspec +++ b/CA.nuspec @@ -4,7 +4,7 @@ continuous-clearing - 6.0.0 + 5.1.0 Siemens AG continuous-clearing contributors https://github.com/siemens/continuous-clearing From c6c788bbeb8adaab1d2da81a7417080d342cbced Mon Sep 17 00:00:00 2001 From: Sumanth K B Date: Thu, 9 Nov 2023 13:02:20 +0530 Subject: [PATCH 17/21] Nuget error handling --- .../NugetDevDependencyParser.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs b/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs index 386ffb68..dc43d67c 100644 --- a/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs +++ b/src/LCT.PackageIdentifier/NugetDevDependencyParser.cs @@ -82,7 +82,13 @@ private static bool IsTestProject(string projectPath) catch (InvalidProjectFileException ex) { Logger.Debug($"IsTestProject(): Failed to read project file : " + projectPath, ex); - Logger.Warn($"IsTestProject: Failed to read project file : " + projectPath); + Logger.Warn($"IsTestProject: Failed to read project file, evaluation fails for : " + projectPath); + return false; + } + catch (InvalidOperationException ex) + { + Logger.Debug($"IsTestProject(): Failed to read project file : " + projectPath, ex); + Logger.Warn($"IsTestProject: Failed to read project file, Maybe there is already an equivalent project loaded in the project collection " + projectPath); return false; } catch (MissingFieldException ex) @@ -97,6 +103,12 @@ private static bool IsTestProject(string projectPath) Logger.Warn($"IsTestProject: Failed to read project file : " + projectPath); return false; } + catch (IOException ex) + { + Logger.Debug($"IsTestProject(): Failed to read project file : " + projectPath, ex); + Logger.Warn($"IsTestProject: Failed to read project file : " + projectPath); + return false; + } foreach (ProjectItem item in csProj.Items) { From 1380aa129fb519c952245f8852f0d565fa798f66 Mon Sep 17 00:00:00 2001 From: karthika Date: Fri, 10 Nov 2023 10:16:14 +0530 Subject: [PATCH 18/21] Read meOSS Updated --- ReadmeOSS_continuous-clearing_DockerImage.html | 2 +- ReadmeOSS_continuous-clearing_nupkg.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ReadmeOSS_continuous-clearing_DockerImage.html b/ReadmeOSS_continuous-clearing_DockerImage.html index 6e7c7637..67fb626d 100644 --- a/ReadmeOSS_continuous-clearing_DockerImage.html +++ b/ReadmeOSS_continuous-clearing_DockerImage.html @@ -77,7 +77,7 @@ -

Clearing Automation Docker Image V5.0.0

+

Clearing Automation Docker Image V5.1.0

Open Source Software

English / English
diff --git a/ReadmeOSS_continuous-clearing_nupkg.html b/ReadmeOSS_continuous-clearing_nupkg.html index 2a70086c..52304c2c 100644 --- a/ReadmeOSS_continuous-clearing_nupkg.html +++ b/ReadmeOSS_continuous-clearing_nupkg.html @@ -77,7 +77,7 @@ -

continuous-clearing V5.0.0

+

continuous-clearing V5.1.0

Open Source Software

English / English
From 4ea8138155e6b6dd97bfb9c8b3bf33185009d29c Mon Sep 17 00:00:00 2001 From: prasenjeetnath <147582543+prasenjeetnath@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:25:57 +0530 Subject: [PATCH 19/21] Update compile.yml --- .github/workflows/compile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 920a2a61..84d8821c 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -195,9 +195,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: v5.0.0 + tag_name: v5.1.0 #tag_name: ${{ needs.build.outputs.semver }} - release_name: Release v5.0.0 + release_name: Release v5.1.0 body: | ${{ github.event.head_commit.message }} draft: true From 865c5c9cbd29e921b54aac7b96d9a268ac2e2cd2 Mon Sep 17 00:00:00 2001 From: prasenjeetnath <147582543+prasenjeetnath@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:27:48 +0530 Subject: [PATCH 20/21] Update compile.yml --- .github/workflows/compile.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 84d8821c..2c712611 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -94,7 +94,7 @@ jobs: $sourceFolder = Join-Path $env:GITHUB_WORKSPACE "out" | Join-Path -ChildPath "*" $outFolder = Join-Path $env:GITHUB_WORKSPACE "out" | Join-Path -ChildPath "continuous-clearing" New-Item -ItemType Directory -Force -Path $outFolder - $fileName = "continuous-clearing-5.0.0.zip" + $fileName = "continuous-clearing-5.1.0.zip" Write-Host "Filename: '$fileName'" Write-Host "sourceFolder: '$sourceFolder'" Write-Host "Outfolder: '$outFolder'" @@ -138,8 +138,8 @@ jobs: - name: Create Nuget Packages id: createNupkg run: | - nuget pack CA.nuspec -Version 5.0.0 - Write-Host "::set-output name=nupkg-LicenseClearingTool::continuous-clearing.5.0.0.nupkg" + nuget pack CA.nuspec -Version 5.1.0 + Write-Host "::set-output name=nupkg-LicenseClearingTool::continuous-clearing.5.1.0.nupkg" - name: Archive NuGet Packages uses: actions/upload-artifact@v2 @@ -152,9 +152,9 @@ jobs: id: builddocker #if: ${{ false }} # disable for now run: | - docker build . --file Dockerfile --tag ${{ github.repository }}:continuous-clearing-v5.0.0 - docker save ${{ github.repository }}:continuous-clearing-v5.0.0 -o continuous-clearing-v5.0.0.tar - Write-Host "::set-output name=docker-LicenseClearingTool::continuous-clearing-v5.0.0.tar" + docker build . --file Dockerfile --tag ${{ github.repository }}:continuous-clearing-v5.1.0 + docker save ${{ github.repository }}:continuous-clearing-v5.0.0 -o continuous-clearing-v5.1.0.tar + Write-Host "::set-output name=docker-LicenseClearingTool::continuous-clearing-v5.1.0.tar" - name: Archive docker image #if: ${{ false }} # disable for now From 8a31811d1be29e741ed09e5b3ae521cf0ad2d318 Mon Sep 17 00:00:00 2001 From: prasenjeetnath <147582543+prasenjeetnath@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:29:07 +0530 Subject: [PATCH 21/21] Update compile.yml updated to 5.1.0 --- .github/workflows/compile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 2c712611..f2479f43 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -153,7 +153,7 @@ jobs: #if: ${{ false }} # disable for now run: | docker build . --file Dockerfile --tag ${{ github.repository }}:continuous-clearing-v5.1.0 - docker save ${{ github.repository }}:continuous-clearing-v5.0.0 -o continuous-clearing-v5.1.0.tar + docker save ${{ github.repository }}:continuous-clearing-v5.1.0 -o continuous-clearing-v5.1.0.tar Write-Host "::set-output name=docker-LicenseClearingTool::continuous-clearing-v5.1.0.tar" - name: Archive docker image