Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Npm and Python bug fixes #224

Merged
merged 14 commits into from
Jan 10, 2025
Merged

Npm and Python bug fixes #224

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
b40e6ad
Npm and Python bug fixes
ragavareddychalapala Jan 2, 2025
51e069c
removed unnessary name
ragavareddychalapala Jan 2, 2025
70e0aa0
Updated review comments
ragavareddychalapala Jan 2, 2025
15ffb9d
removed unnessary space
ragavareddychalapala Jan 2, 2025
5d05078
Updated as per review comments and fix test cases
ragavareddychalapala Jan 3, 2025
8c60d62
Updated code changes
ragavareddychalapala Jan 4, 2025
d6a99a4
removed commented code
ragavareddychalapala Jan 4, 2025
beb6769
Removed Unnessasary usings
ragavareddychalapala Jan 4, 2025
b461694
refactored code
ragavareddychalapala Jan 4, 2025
a03c1c1
unnessasary
ragavareddychalapala Jan 4, 2025
a79e8b0
unit test case fix
ragavareddychalapala Jan 4, 2025
8b24886
reverted
ragavareddychalapala Jan 6, 2025
8601964
Updated for requested changes
ragavareddychalapala Jan 6, 2025
cceeb67
Reverted
ragavareddychalapala Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions src/ArtifactoryUploader/PackageUploadHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -951,7 +951,7 @@ public async static Task<AqlResult> GetSrcRepoDetailsForPyPiOrConanPackages(Comp
if (item.Purl.Contains("pypi", StringComparison.OrdinalIgnoreCase))
{
// get the component list from Jfrog for given repo
aqlResultList = await GetListOfComponentsFromRepo(new string[] { item.Properties.Find(x => x.Name == Dataconstant.Cdx_ArtifactoryRepoName)?.Value }, jFrogService);
aqlResultList = await GetPypiListOfComponentsFromRepo(new string[] { item.Properties.Find(x => x.Name == Dataconstant.Cdx_ArtifactoryRepoName)?.Value }, jFrogService);
if (aqlResultList.Count > 0)
{
return GetArtifactoryRepoName(aqlResultList, item);
Expand Down Expand Up @@ -1166,15 +1166,27 @@ public static async Task<List<AqlResult>> GetListOfComponentsFromRepo(string[] r

return aqlResultList;
}
public static async Task<List<AqlResult>> GetPypiListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService)
{
if (repoList != null && repoList.Length > 0)
{
foreach (var repo in repoList)
{
var test = await jFrogService.GetPypiInternalComponentDataByRepo(repo) ?? new List<AqlResult>();
adityanarayanp marked this conversation as resolved.
Show resolved Hide resolved
aqlResultList.AddRange(test);
}
}

return aqlResultList;
}

private static AqlResult GetArtifactoryRepoName(List<AqlResult> aqlResultList, Component component)
{
string jfrogcomponentName = $"{component.Name}-{component.Version}";

AqlResult repoName = aqlResultList.Find(x => x.Name.Contains(jfrogcomponentName, StringComparison.OrdinalIgnoreCase));
AqlResult repoName = aqlResultList.Find(x => x.properties.Any(p => p.key == "pypi.normalized.name" && p.value == component.Name) && x.properties.Any(p => p.key == "pypi.version" && p.value == component.Version));

return repoName;
}


private static AqlResult GetArtifactoryRepoNameForConan(List<AqlResult> aqlResultList, Component component)
{
Expand Down
15 changes: 15 additions & 0 deletions src/LCT.APICommunications/Interfaces/IJfrogAqlApiCommunication.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
// SPDX-License-Identifier: MIT
// --------------------------------------------------------------------------------------------------------------------

using LCT.Common;
using System.ComponentModel;
using System.Net.Http;
using System.Threading.Tasks;

Expand All @@ -20,6 +22,19 @@ public interface IJfrogAqlApiCommunication
/// <param name="repoName">repoName</param>
/// <returns>HttpResponseMessage</returns>
Task<HttpResponseMessage> GetInternalComponentDataByRepo(string repoName);
/// <summary>
/// Gets the internal component data based on repo name
/// </summary>
/// <param name="repoName">repoName</param>
/// <returns>HttpResponseMessage</returns>
Task<HttpResponseMessage> GetNpmInternalComponentDataByRepo(string repoName);
/// <summary>
/// Gets the internal component data based on repo name
/// </summary>
/// <param name="repoName">repoName</param>
/// <returns>HttpResponseMessage</returns>
Task<HttpResponseMessage> GetPypiInternalComponentDataByRepo(string repoName);


/// <summary>
/// Gets the package information in the repo, via the name or path
Expand Down
33 changes: 33 additions & 0 deletions src/LCT.APICommunications/JfrogAqlApiCommunication.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

using LCT.APICommunications.Interfaces;
using LCT.APICommunications.Model;
using LCT.Common;
using System;
using System.Collections.Generic;
using System.Net.Http;
Expand Down Expand Up @@ -65,7 +66,39 @@ public async Task<HttpResponseMessage> GetInternalComponentDataByRepo(string rep
HttpContent httpContent = new StringContent(aqlQueryToBody);
return await httpClient.PostAsync(uri, httpContent);
}
public async Task<HttpResponseMessage> GetNpmInternalComponentDataByRepo(string repoName)
{
HttpClient httpClient = GetHttpClient(ArtifactoryCredentials);
TimeSpan timeOutInSec = TimeSpan.FromSeconds(TimeoutInSec);
httpClient.Timeout = timeOutInSec;

StringBuilder query = new();
query.Append("items.find({\"repo\":\"");
query.Append($"{repoName}");
query.Append("\"}).include(\"repo\", \"path\", \"name\",\"@npm.name\",\"@npm.version\", \"actual_sha1\",\"actual_md5\",\"sha256\")");

string aqlQueryToBody = query.ToString();
string uri = $"{DomainName}{ApiConstant.JfrogArtifactoryApiSearchAql}";
HttpContent httpContent = new StringContent(aqlQueryToBody);
return await httpClient.PostAsync(uri, httpContent);
}
public async Task<HttpResponseMessage> GetPypiInternalComponentDataByRepo(string repoName)
{
HttpClient httpClient = GetHttpClient(ArtifactoryCredentials);
TimeSpan timeOutInSec = TimeSpan.FromSeconds(TimeoutInSec);
httpClient.Timeout = timeOutInSec;

StringBuilder query = new();
query.Append("items.find({\"repo\":\"");
query.Append($"{repoName}");
query.Append("\"}).include(\"repo\", \"path\", \"name\",\"@pypi.normalized.name\",\"@pypi.version\", \"actual_sha1\",\"actual_md5\",\"sha256\")");

string aqlQueryToBody = query.ToString();
string uri = $"{DomainName}{ApiConstant.JfrogArtifactoryApiSearchAql}";
HttpContent httpContent = new StringContent(aqlQueryToBody);
return await httpClient.PostAsync(uri, httpContent);
}

/// <summary>
/// Gets the package information in the repo, via the name or path
/// </summary>
Expand Down
10 changes: 10 additions & 0 deletions src/LCT.APICommunications/Model/AQL/AqlResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
// --------------------------------------------------------------------------------------------------------------------

using Newtonsoft.Json;
using System.Collections.Generic;

namespace LCT.APICommunications.Model.AQL
{
Expand All @@ -31,5 +32,14 @@ public class AqlResult

[JsonProperty("sha256")]
public string SHA256 { get; set; }

public List<AqlProperty> properties { get; set; }

}
public class AqlProperty
{
public string key { get; set; }
public string value { get; set; }
}

}
14 changes: 14 additions & 0 deletions src/LCT.Facade/Interfaces/IJfrogAqlApiCommunicationFacade.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
// SPDX-License-Identifier: MIT
// --------------------------------------------------------------------------------------------------------------------

using LCT.Common;
using System.Net.Http;
using System.Threading.Tasks;

Expand All @@ -20,6 +21,19 @@ public interface IJfrogAqlApiCommunicationFacade
/// <param name="repoName">repoName</param>
/// <returns>HttpResponseMessage</returns>
Task<HttpResponseMessage> GetInternalComponentDataByRepo(string repoName);
/// <summary>
/// Gets the Internal Component Data By Repo Name
/// </summary>
/// <param name="repoName">repoName</param>
/// <returns>HttpResponseMessage</returns>
Task<HttpResponseMessage> GetNpmInternalComponentDataByRepo(string repoName);
/// <summary>
/// Gets the Internal Component Data By Repo Name
/// </summary>
/// <param name="repoName">repoName</param>
/// <returns>HttpResponseMessage</returns>
Task<HttpResponseMessage> GetPypiInternalComponentDataByRepo(string repoName);


/// <summary>
/// Gets the package information in the repo, via the name or path
Expand Down
23 changes: 21 additions & 2 deletions src/LCT.Facade/JfrogAqlApiCommunicationFacade.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
// --------------------------------------------------------------------------------------------------------------------

using LCT.APICommunications.Interfaces;
using LCT.Common;
using LCT.Facade.Interfaces;
using System.Net.Http;
using System.Threading.Tasks;
Expand Down Expand Up @@ -43,9 +44,27 @@ public Task<HttpResponseMessage> CheckConnection()
/// <returns>HttpResponseMessage</returns>
public async Task<HttpResponseMessage> GetInternalComponentDataByRepo(string repoName)
{
return await m_jfrogAqlApiCommunication.GetInternalComponentDataByRepo(repoName);
return await m_jfrogAqlApiCommunication.GetInternalComponentDataByRepo(repoName);
}

/// <summary>
/// Gets the Internal Component Data By Repo Name
/// </summary>
/// <param name="repoName">repoName</param>
/// <returns>HttpResponseMessage</returns>
public async Task<HttpResponseMessage> GetNpmInternalComponentDataByRepo(string repoName)
{
return await m_jfrogAqlApiCommunication.GetNpmInternalComponentDataByRepo(repoName);
}
/// <summary>
/// Gets the Internal Component Data By Repo Name
/// </summary>
/// <param name="repoName">repoName</param>
/// <returns>HttpResponseMessage</returns>
public async Task<HttpResponseMessage> GetPypiInternalComponentDataByRepo(string repoName)
{
return await m_jfrogAqlApiCommunication.GetPypiInternalComponentDataByRepo(repoName);
}

/// <summary>
/// Gets the package information in the repo, via the name or path
/// </summary>
Expand Down
28 changes: 28 additions & 0 deletions src/LCT.PackageIdentifier/BomHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,34 @@ public async Task<List<AqlResult>> GetListOfComponentsFromRepo(string[] repoList

return aqlResultList;
}
public async Task<List<AqlResult>> GetNpmListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService)
{
List<AqlResult> aqlResultList = new();
if (repoList != null && repoList.Length > 0)
{
foreach (var repo in repoList)
{
var test = await jFrogService.GetNpmInternalComponentDataByRepo(repo) ?? new List<AqlResult>();
aqlResultList.AddRange(test);
}
}

return aqlResultList;
}
public async Task<List<AqlResult>> GetPypiListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService)
{
List<AqlResult> aqlResultList = new();
if (repoList != null && repoList.Length > 0)
{
foreach (var repo in repoList)
{
var test = await jFrogService.GetPypiInternalComponentDataByRepo(repo) ?? new List<AqlResult>();
aqlResultList.AddRange(test);
}
}

return aqlResultList;
}

#endregion
}
Expand Down
4 changes: 4 additions & 0 deletions src/LCT.PackageIdentifier/Interface/IBomHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

using CycloneDX.Models;
using LCT.APICommunications.Model.AQL;
using LCT.Common;
using LCT.PackageIdentifier.Model;
using LCT.Services.Interface;
using System.Collections.Generic;
Expand All @@ -23,5 +24,8 @@ public interface IBomHelper
public string GetProjectSummaryLink(string projectId, string sw360Url);
public string GetFullNameOfComponent(Component item);
public Task<List<AqlResult>> GetListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService);
public Task<List<AqlResult>> GetNpmListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService);
public Task<List<AqlResult>> GetPypiListOfComponentsFromRepo(string[] repoList, IJFrogService jFrogService);

}
}
44 changes: 21 additions & 23 deletions src/LCT.PackageIdentifier/NpmProcessor.cs
View file
Open in desktop
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the method IsInternalNpmComponent still checks by the old way, it also needs to be updated to use the new way

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

UT does not look to be updated

Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public class NpmProcessor : CycloneDXBomParser, IParser
private const string Version = "version";
private const string NotFoundInRepo = "Not Found in JFrogRepo";
private const string Requires = "requires";
private const string Name = "name";
public NpmProcessor(ICycloneDXBomParser cycloneDXBomParser)
{
_cycloneDXBomParser = cycloneDXBomParser;
Expand Down Expand Up @@ -227,7 +228,16 @@ private static void GetPackagesForBom(string filepath, ref List<BundledComponent
}

string folderPath = CommonHelper.TrimEndOfString(filepath, $"\\{FileConstant.PackageLockFileName}");
string packageName = CommonHelper.GetSubstringOfLastOccurance(prop.Name, $"node_modules/");
string packageName=string.Empty;
if (properties[Name] != null)
{
packageName = Convert.ToString(properties[Name]);
}
else
{
packageName = CommonHelper.GetSubstringOfLastOccurance(prop.Name, $"node_modules/");
}

string componentName = packageName.StartsWith('@') ? packageName.Replace("@", "%40") : packageName;

if (packageName.Contains('@'))
Expand All @@ -238,7 +248,7 @@ private static void GetPackagesForBom(string filepath, ref List<BundledComponent
else
{
components.Name = packageName;
}
}

components.Type = Component.Classification.Library;
components.Description = folderPath;
Expand Down Expand Up @@ -353,7 +363,7 @@ public async Task<ComponentIdentification> IdentificationOfInternalComponents(
{
// get the component list from Jfrog for given repo
List<AqlResult> aqlResultList =
await bomhelper.GetListOfComponentsFromRepo(appSettings.InternalRepoList, jFrogService);
await bomhelper.GetNpmListOfComponentsFromRepo(appSettings.InternalRepoList, jFrogService);

// find the components in the list of internal components
List<Component> internalComponents = new List<Component>();
Expand Down Expand Up @@ -396,15 +406,17 @@ public async Task<List<Component>> GetJfrogRepoDetailsOfAComponent(List<Componen
{
// get the component list from Jfrog for given repo + internal repo
string[] repoList = appSettings.InternalRepoList.Concat(appSettings.Npm?.JfrogNpmRepoList).ToArray();
List<AqlResult> aqlResultList = await bomhelper.GetListOfComponentsFromRepo(repoList, jFrogService);
List<AqlResult> aqlResultList = await bomhelper.GetNpmListOfComponentsFromRepo(repoList, jFrogService);

Property projectType = new() { Name = Dataconstant.Cdx_ProjectType, Value = appSettings.ProjectType };
List<Component> modifiedBOM = new List<Component>();

foreach (var component in componentsForBOM)
{
string jfrogpackageName = $"{component.Name}-{component.Version}{ApiConstant.NpmExtension}";
var hashes = aqlResultList.FirstOrDefault(x => x.Name == jfrogpackageName);


string jfrogpackageName= bomhelper.GetFullNameOfComponent(component);

var hashes = aqlResultList.FirstOrDefault(x => x.properties.Any(p => p.key == "npm.name" && p.value == jfrogpackageName) && x.properties.Any(p => p.key == "npm.version" && p.value == component.Version));
string jfrogRepoPath = string.Empty;
AqlResult finalRepoData = GetJfrogArtifactoryRepoDetials(aqlResultList, component, bomhelper, out jfrogRepoPath);
Property artifactoryrepo = new() { Name = Dataconstant.Cdx_ArtifactoryRepoName, Value = finalRepoData.Repo };
Expand Down Expand Up @@ -633,26 +645,12 @@ public AqlResult GetJfrogArtifactoryRepoDetials(List<AqlResult> aqlResultList,
{
AqlResult aqlResult = new AqlResult();
jfrogRepoPath = Dataconstant.JfrogRepoPathNotFound;
string jfrogcomponentName = $"{component.Name}-{component.Version}.tgz";
string jfrogpackageName = bomHelper.GetFullNameOfComponent(component);

var aqlResults = aqlResultList.FindAll(x => x.Name.Equals(
jfrogcomponentName, StringComparison.OrdinalIgnoreCase));
var aqlResults = aqlResultList.FindAll(x => x.properties.Any(p => p.key == "npm.name" && p.value == jfrogpackageName) && x.properties.Any(p => p.key == "npm.version" && p.value == component.Version));

string repoName = CommonIdentiferHelper.GetRepodetailsFromPerticularOrder(aqlResults);

if (repoName.Equals(NotFoundInRepo, StringComparison.OrdinalIgnoreCase))
{
string fullName = bomHelper.GetFullNameOfComponent(component);
string fullNameVersion = $"{fullName}-{component.Version}.tgz";
if (!fullNameVersion.Equals(jfrogcomponentName, StringComparison.OrdinalIgnoreCase))
{
aqlResults = aqlResultList.FindAll(x => x.Name.Equals(
fullNameVersion, StringComparison.OrdinalIgnoreCase));

repoName = CommonIdentiferHelper.GetRepodetailsFromPerticularOrder(aqlResults);
}
}

// Forming Jfrog repo Path
if (!repoName.Equals(NotFoundInRepo, StringComparison.OrdinalIgnoreCase))
{
Expand Down
Loading
Loading