From e57598fd2655405ee0bc51397f1e3972026c202d Mon Sep 17 00:00:00 2001 From: Manith Date: Mon, 12 Aug 2024 06:16:58 +0000 Subject: [PATCH] Add support for git repo as upstream source eext currently supports 'srpm' and 'tarball' as upstream sources for packages. Some users wanted to use an upstream git repo directly as source for security purposes. Hence we add support for eext to use an upstream git repo as an upstream source. The git upstream source requires 'url' and 'revision' to be specified. 'url': Web url to the upstream git repo 'revision': Commit hash or release tag of the git repo Users can verify their upstream git repo at the revision provided, by specifying the corresponding public key. Note that this verification only works if the commit/tag are signed, since we use 'git-verify' to validate. For unsigned commits/tags, user need to enable 'skip-check' in the signature field (not recommended though, since this may introduce security vulnerabilities). Another rule users need to adhere to while using git repo as upstream, is that the spec file in 'spec/' folder should be named '.spec' where 'pkgName' is the same as mentioned in eext.yaml. And within the spec file, please ensure that the 'Source0:' field points to 'Source0.tar.gz' in eext.yaml. --- cmd/create_srpm_test.go | 35 -- impl/create_srpm.go | 104 ++---- impl/create_srpm_for_git.go | 303 ++++++++++++++++++ impl/create_srpm_for_others.go | 193 +++++++++++ impl/create_srpm_from_git_test.go | 254 +++++++++++++++ impl/create_srpm_from_others_test.go | 44 +++ impl/testData/upstream-git-repo-1/eext.yaml | 14 + .../upstream-git-repo-1/spec/libpcap.spec | 174 ++++++++++ manifest/manifest.go | 70 ++-- manifest/manifest_test.go | 22 +- manifest/testData/sampleManifest4.yaml | 30 ++ manifest/testData/sampleManifest5.yaml | 13 + srcconfig/srcconfig.go | 5 +- testutil/testutil.go | 4 +- util/util.go | 126 +------- 15 files changed, 1141 insertions(+), 250 deletions(-) create mode 100644 impl/create_srpm_for_git.go create mode 100644 impl/create_srpm_for_others.go create mode 100644 impl/create_srpm_from_git_test.go create mode 100644 impl/create_srpm_from_others_test.go create mode 100644 impl/testData/upstream-git-repo-1/eext.yaml create mode 100644 impl/testData/upstream-git-repo-1/spec/libpcap.spec create mode 100644 manifest/testData/sampleManifest4.yaml create mode 100644 manifest/testData/sampleManifest5.yaml diff --git a/cmd/create_srpm_test.go b/cmd/create_srpm_test.go index a05aac6b..575858c9 100644 --- a/cmd/create_srpm_test.go +++ b/cmd/create_srpm_test.go @@ -73,41 +73,6 @@ func testCreateSrpm(t *testing.T, } } -func testTarballSig(t *testing.T, folder string) { - curPath, _ := os.Getwd() - workingDir := filepath.Join(curPath, "testData/tarballSig", folder) - tarballPath := map[string]string{ - "checkTarball": filepath.Join(workingDir, "linux.10.4.1.tar.gz"), - "matchTarball": filepath.Join(workingDir, "libpcap-1.10.4.tar.gz"), - } - tarballSigPath := filepath.Join(workingDir, "libpcap-1.10.4.tar.gz.sig") - - switch folder { - case "checkTarball": - ok, _ := util.CheckValidSignature(tarballPath[folder], tarballSigPath) - require.Equal(t, false, ok) - case "matchTarball": - intermediateTarball, err := util.MatchtarballSignCmprsn( - tarballPath[folder], - tarballSigPath, - workingDir, - "TestmatchTarballSignature : ", - ) - os.Remove(intermediateTarball) - require.Equal(t, nil, err) - } -} - -func TestCheckTarballSignature(t *testing.T) { - t.Log("Test tarball Signatue Check") - testTarballSig(t, "checkTarball") -} - -func TestMatchTarballSignature(t *testing.T) { - t.Log("Test tarball Signatue Match") - testTarballSig(t, "matchTarball") -} - func TestCreateSrpmFromSrpm(t *testing.T) { t.Log("Test createSrpm from SRPM") testCreateSrpm(t, diff --git a/impl/create_srpm.go b/impl/create_srpm.go index d9c7344e..ebf4ee02 100644 --- a/impl/create_srpm.go +++ b/impl/create_srpm.go @@ -23,6 +23,7 @@ type upstreamSrcSpec struct { sigFile string pubKeyPath string skipSigCheck bool + gitSpec gitSpec } type srpmBuilder struct { @@ -77,9 +78,6 @@ func (bldr *srpmBuilder) clean() error { // Put them into downloadDir and populate bldr.upstreamSrc func (bldr *srpmBuilder) fetchUpstream() error { bldr.log("starting") - repo := bldr.repo - pkg := bldr.pkgSpec.Name - isPkgSubdirInRepo := bldr.pkgSpec.Subdir // First fetch upstream source downloadDir := getDownloadDir(bldr.pkgSpec.Name) @@ -89,70 +87,18 @@ func (bldr *srpmBuilder) fetchUpstream() error { } for _, upstreamSrcFromManifest := range bldr.pkgSpec.UpstreamSrc { - srcParams, err := srcconfig.GetSrcParams( - bldr.pkgSpec.Name, - upstreamSrcFromManifest.FullURL, - upstreamSrcFromManifest.SourceBundle.Name, - upstreamSrcFromManifest.Signature.DetachedSignature.FullURL, - upstreamSrcFromManifest.SourceBundle.SrcRepoParamsOverride, - upstreamSrcFromManifest.Signature.DetachedSignature.OnUncompressed, - bldr.srcConfig, - bldr.errPrefix) - if err != nil { - return fmt.Errorf("%sUnable to get source params for %s", - err, upstreamSrcFromManifest.SourceBundle.Name) - } - - var downloadErr error - upstreamSrc := upstreamSrcSpec{} - - bldr.log("downloading %s", srcParams.SrcURL) - // Download source - if upstreamSrc.sourceFile, downloadErr = download( - srcParams.SrcURL, - downloadDir, - repo, pkg, isPkgSubdirInRepo, - bldr.errPrefix); downloadErr != nil { - return downloadErr + upstreamSrcType := bldr.pkgSpec.Type + var upstreamSrc *upstreamSrcSpec + var err error + if upstreamSrcType == "git-upstream" { + upstreamSrc, err = bldr.getUpstreamSourceForGit(upstreamSrcFromManifest, downloadDir) + } else { + upstreamSrc, err = bldr.getUpstreamSourceForOthers(upstreamSrcFromManifest, downloadDir) } - bldr.log("downloaded") - - upstreamSrc.skipSigCheck = upstreamSrcFromManifest.Signature.SkipCheck - pubKey := upstreamSrcFromManifest.Signature.DetachedSignature.PubKey - - if bldr.pkgSpec.Type == "tarball" && !upstreamSrc.skipSigCheck { - if srcParams.SignatureURL == "" || pubKey == "" { - return fmt.Errorf("%sNo detached-signature/public-key specified for upstream-sources entry %s", - bldr.errPrefix, srcParams.SrcURL) - } - if upstreamSrc.sigFile, downloadErr = download( - srcParams.SignatureURL, - downloadDir, - repo, pkg, isPkgSubdirInRepo, - bldr.errPrefix); downloadErr != nil { - return downloadErr - } - - pubKeyPath := filepath.Join(getDetachedSigDir(), pubKey) - if pathErr := util.CheckPath(pubKeyPath, false, false); pathErr != nil { - return fmt.Errorf("%sCannot find public-key at path %s", - bldr.errPrefix, pubKeyPath) - } - upstreamSrc.pubKeyPath = pubKeyPath - } else if bldr.pkgSpec.Type == "srpm" || bldr.pkgSpec.Type == "unmodified-srpm" { - // We don't expect SRPMs to have detached signature or - // to be validated with a public-key specified in manifest. - if srcParams.SignatureURL != "" { - return fmt.Errorf("%sUnexpected detached-sig specified for SRPM", - bldr.errPrefix) - } - if pubKey != "" { - return fmt.Errorf("%sUnexpected public-key specified for SRPM", - bldr.errPrefix) - } + if err != nil { + return err } - - bldr.upstreamSrc = append(bldr.upstreamSrc, upstreamSrc) + bldr.upstreamSrc = append(bldr.upstreamSrc, *upstreamSrc) } bldr.log("successful") @@ -201,7 +147,7 @@ func (bldr *srpmBuilder) verifyUpstreamSrpm() error { } if !upstreamSrc.skipSigCheck { - if err := util.VerifyRpmSignature(upstreamSrpmFilePath, bldr.errPrefix); err != nil { + if err := verifyRpmSignature(upstreamSrpmFilePath, bldr.errPrefix); err != nil { return err } } @@ -216,6 +162,15 @@ func (bldr *srpmBuilder) verifyUpstream() error { if err := bldr.verifyUpstreamSrpm(); err != nil { return err } + } else if bldr.pkgSpec.Type == "git-upstream" { + for _, upstreamSrc := range bldr.upstreamSrc { + if !upstreamSrc.skipSigCheck { + err := verifyGitSignature(upstreamSrc.pubKeyPath, upstreamSrc.gitSpec, bldr.errPrefix) + if err != nil { + return err + } + } + } } else { downloadDir := getDownloadDir(bldr.pkgSpec.Name) @@ -223,17 +178,17 @@ func (bldr *srpmBuilder) verifyUpstream() error { if !upstreamSrc.skipSigCheck { upstreamSourceFilePath := filepath.Join(downloadDir, upstreamSrc.sourceFile) upstreamSigFilePath := filepath.Join(downloadDir, upstreamSrc.sigFile) - uncompressedTarball, err := util.MatchtarballSignCmprsn( + uncompressedTarballPath, err := matchTarballSignCmprsn( upstreamSourceFilePath, upstreamSigFilePath, downloadDir, bldr.errPrefix) if err != nil { return err } - if uncompressedTarball != "" { - upstreamSourceFilePath = uncompressedTarball - defer os.Remove(uncompressedTarball) + if uncompressedTarballPath != "" { + upstreamSourceFilePath = uncompressedTarballPath + defer os.Remove(uncompressedTarballPath) } - if err := util.VerifyTarballSignature( + if err := verifyTarballSignature( upstreamSourceFilePath, upstreamSigFilePath, upstreamSrc.pubKeyPath, @@ -281,7 +236,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeSrpm() error { // also checks tarball signature func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error { - supportedTypes := []string{"tarball", "standalone"} + supportedTypes := []string{"tarball", "standalone", "git-upstream"} if !slices.Contains(supportedTypes, bldr.pkgSpec.Type) { panic(fmt.Sprintf("%ssetupRpmbuildTreeNonSrpm called for unsupported type %s", bldr.errPrefix, bldr.pkgSpec.Type)) @@ -294,7 +249,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error { return err } - if bldr.pkgSpec.Type == "tarball" { + if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "git-upstream" { downloadDir := getDownloadDir(bldr.pkgSpec.Name) for _, upstreamSrc := range bldr.upstreamSrc { upstreamSourceFilePath := filepath.Join(downloadDir, upstreamSrc.sourceFile) @@ -389,7 +344,8 @@ func (bldr *srpmBuilder) setupRpmbuildTree() error { if err := bldr.setupRpmbuildTreeSrpm(); err != nil { return err } - } else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" { + } else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" || + bldr.pkgSpec.Type == "git-upstream" { if err := bldr.setupRpmbuildTreeNonSrpm(); err != nil { return err } diff --git a/impl/create_srpm_for_git.go b/impl/create_srpm_for_git.go new file mode 100644 index 00000000..367b7551 --- /dev/null +++ b/impl/create_srpm_for_git.go @@ -0,0 +1,303 @@ +// Copyright (c) 2022 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +package impl + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "code.arista.io/eos/tools/eext/manifest" + "code.arista.io/eos/tools/eext/srcconfig" + "code.arista.io/eos/tools/eext/util" +) + +type gitSpec struct { + SrcUrl string + Revision string + ClonedDir string +} + +type GitRevisionType int + +const ( + UNDEFINED GitRevisionType = 0 + COMMIT GitRevisionType = 1 + TAG GitRevisionType = 2 +) + +func (spec *gitSpec) typeOfGitRevisionFromRemote() (GitRevisionType, error) { + srcURL := spec.SrcUrl + revision := spec.Revision + cmd := exec.Command("git", "ls-remote", srcURL, revision) + var out bytes.Buffer + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return UNDEFINED, fmt.Errorf("git ls-remote of repo %s failed: %s", srcURL, err) + } + if len(out.String()) == 0 { + return COMMIT, nil + } else if strings.Contains(out.String(), revision) { + return TAG, nil + } else { + return UNDEFINED, fmt.Errorf("revision %s not found in repo %s, please provide valid commit/tag", revision, srcURL) + } +} + +func (spec *gitSpec) typeOfGitRevision() (GitRevisionType, error) { + repoPath := spec.ClonedDir + revision := spec.Revision + if err := util.CheckPath(repoPath, true, false); err != nil { + return spec.typeOfGitRevisionFromRemote() + } + + cmd := exec.Command("git", "show", revision) + cmd.Dir = repoPath + var out bytes.Buffer + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return UNDEFINED, fmt.Errorf("git show of repo %s failed %s", repoPath, err) + } + + tagCmd := exec.Command("git", "show-ref", "--tags", revision) + tagCmd.Dir = repoPath + tagCmd.Stdout = &out + if err := tagCmd.Run(); err == nil { + return TAG, nil + } + + topLine := strings.SplitAfterN(out.String(), "\n", 2)[0] + if strings.Contains(topLine, revision) { + return COMMIT, nil + } + + return UNDEFINED, fmt.Errorf("revision %s not found in repo %s, provide valid commit/tag", revision, repoPath) +} + +func getRpmNameFromSpecFile(repo, pkg string, isPkgSubdirInRepo bool) (string, error) { + pkgSpecDirInRepo := getPkgSpecDirInRepo(repo, pkg, isPkgSubdirInRepo) + specFiles, _ := filepath.Glob(filepath.Join(pkgSpecDirInRepo, "*.spec")) + numSpecFiles := len(specFiles) + if numSpecFiles == 0 { + return "", fmt.Errorf("no *.spec files found in %s", pkgSpecDirInRepo) + } + if numSpecFiles > 1 { + return "", fmt.Errorf("multiple *.spec files %s found in %s", strings.Join(specFiles, ","), pkgSpecDirInRepo) + } + specFilePath := specFiles[0] + + cmd := []string{"-q", "--srpm", "--qf", "%{NAME}-%{VERSION}", specFilePath} + rpmName, err := util.CheckOutput("rpmspec", cmd...) + if err != nil { + return "", fmt.Errorf("cannot query spec file %s for %s", specFilePath, pkg) + } + + return rpmName, nil +} + +// We aren't using 'git clone' since it is slow for large repos. +// This method is faster and only pulls necessary changes. +func cloneGitRepo(pkg, srcURL, revision, targetDir string) (string, error) { + // Cloning the git repo to a temporary directory + cloneDir, err := os.MkdirTemp(targetDir, pkg) + if err != nil { + return "", fmt.Errorf("error while creating tempDir for %s, %s", pkg, err) + } + // Init the dir as a git repo + err = util.RunSystemCmdInDir(cloneDir, "git", "init") + if err != nil { + return "", fmt.Errorf("git init at %s failed: %s", cloneDir, err) + } + // Add the srcURL as the origin for the repo + err = util.RunSystemCmdInDir(cloneDir, "git", "remote", "add", "origin", srcURL) + if err != nil { + return "", fmt.Errorf("adding %s as git remote failed: %s", srcURL, err) + } + // Fetch repo tags, for user inputs revision as TAG + err = util.RunSystemCmdInDir(cloneDir, "git", "fetch", "--tags") + if err != nil { + return "", fmt.Errorf("fetching tags failed for %s: %s", pkg, err) + } + // Fetch the code changes for the provided revision + err = util.RunSystemCmdInDir(cloneDir, "git", "fetch", "origin", revision) + if err != nil { + return "", fmt.Errorf("fetching revision %s failed for %s: %s", revision, pkg, err) + } + // Pull code to repo at provided revision + err = util.RunSystemCmdInDir(cloneDir, "git", "reset", "--hard", "FETCH_HEAD") + if err != nil { + return "", fmt.Errorf("fetching HEAD at %s failed: %s", revision, err) + } + + return cloneDir, nil +} + +func generateArchiveFile(targetDir, clonedDir, revision, repo, pkg string, isPkgSubdirInRepo bool, + errPrefix util.ErrPrefix) (string, error) { + // User should ensure the same fileName is specified in .spec file. + // We use Source0.tar.gz as the generated tarball path, + // since this can be extended to support multiple sources in future. + gitArchiveFile := "Source0.tar.gz" + gitArchiveFilePath := filepath.Join(targetDir, gitArchiveFile) + parentFolder, err := getRpmNameFromSpecFile(repo, pkg, isPkgSubdirInRepo) + if err != nil { + return "", err + } + + // Create the tarball from the specified commit/tag revision + archiveCmd := []string{"archive", + "--prefix", parentFolder + "/", + "-o", gitArchiveFilePath, + revision, + } + err = util.RunSystemCmdInDir(clonedDir, "git", archiveCmd...) + if err != nil { + return "", fmt.Errorf("%sgit archive of %s failed: %s %v", errPrefix, pkg, err, archiveCmd) + } + + return gitArchiveFile, nil +} + +// Download the git repo, and create a tarball at the provided commit/tag. +func archiveGitRepo(srcURL, targetDir, revision, repo, pkg string, isPkgSubdirInRepo bool, + errPrefix util.ErrPrefix) (string, string, error) { + cloneDir, err := cloneGitRepo(pkg, srcURL, revision, targetDir) + if err != nil { + return "", "", fmt.Errorf("cloning git repo failed: %s", err) + } + + gitArchiveFile, err := generateArchiveFile(targetDir, cloneDir, revision, repo, pkg, isPkgSubdirInRepo, errPrefix) + if err != nil { + return "", "", fmt.Errorf("generating git archive failed: %s", err) + } + + return gitArchiveFile, cloneDir, nil +} + +func getGitSpecAndSrcFile(srcUrl, revision, downloadDir, repo, pkg string, + isPkgSubdirInRepo bool, errPrefix util.ErrPrefix) (*gitSpec, string, error) { + spec := gitSpec{ + SrcUrl: srcUrl, + Revision: revision, + } + + sourceFile, clonedDir, downloadErr := archiveGitRepo( + srcUrl, + downloadDir, + revision, + repo, pkg, isPkgSubdirInRepo, + errPrefix) + if downloadErr != nil { + return nil, "", downloadErr + } + + spec.ClonedDir = clonedDir + return &spec, sourceFile, nil +} + +func (bldr *srpmBuilder) getUpstreamSourceForGit(upstreamSrcFromManifest manifest.UpstreamSrc, + downloadDir string) (*upstreamSrcSpec, error) { + + repo := bldr.repo + pkg := bldr.pkgSpec.Name + isPkgSubdirInRepo := bldr.pkgSpec.Subdir + + srcParams, err := srcconfig.GetSrcParams( + pkg, + upstreamSrcFromManifest.GitBundle.Url, + upstreamSrcFromManifest.SourceBundle.Name, + upstreamSrcFromManifest.Signature.DetachedSignature.FullURL, + upstreamSrcFromManifest.SourceBundle.SrcRepoParamsOverride, + upstreamSrcFromManifest.Signature.DetachedSignature.OnUncompressed, + bldr.srcConfig, + bldr.errPrefix) + if err != nil { + return nil, fmt.Errorf("%sunable to get source params for %s", + err, upstreamSrcFromManifest.SourceBundle.Name) + } + + upstreamSrc := upstreamSrcSpec{} + + bldr.log("creating tarball for %s from repo %s", pkg, srcParams.SrcURL) + srcUrl := srcParams.SrcURL + revision := upstreamSrcFromManifest.GitBundle.Revision + spec, sourceFile, err := getGitSpecAndSrcFile(srcUrl, revision, downloadDir, + repo, pkg, isPkgSubdirInRepo, bldr.errPrefix) + if err != nil { + return nil, err + } + bldr.log("tarball created") + + upstreamSrc.gitSpec = *spec + upstreamSrc.sourceFile = sourceFile + upstreamSrc.skipSigCheck = upstreamSrcFromManifest.Signature.SkipCheck + pubKey := upstreamSrcFromManifest.Signature.DetachedSignature.PubKey + + if !upstreamSrc.skipSigCheck { + if pubKey == "" { + return nil, fmt.Errorf("%sexpected public-key for %s to verify git repo", + bldr.errPrefix, pkg) + } + pubKeyPath := filepath.Join(getDetachedSigDir(), pubKey) + if pathErr := util.CheckPath(pubKeyPath, false, false); pathErr != nil { + return nil, fmt.Errorf("%sCannot find public-key at path %s", + bldr.errPrefix, pubKeyPath) + } + upstreamSrc.pubKeyPath = pubKeyPath + } + + return &upstreamSrc, nil +} + +// verifyGitSignature verifies that the git repo commit/tag is signed. +func verifyGitSignature(pubKeyPath string, gitSpec gitSpec, errPrefix util.ErrPrefix) error { + tmpDir, mkdtErr := os.MkdirTemp("", "eext-keyring") + if mkdtErr != nil { + return fmt.Errorf("%sError '%s'creating temp dir for keyring", + errPrefix, mkdtErr) + } + defer os.RemoveAll(tmpDir) + + err := os.Setenv("GNUPGHOME", tmpDir) + if err != nil { + return fmt.Errorf("%sunable to set ENV variable GNUPGHOME", errPrefix) + } + defer os.Unsetenv("GNUPGHOME") + + if err := util.RunSystemCmd("gpg", "--fingerprint"); err != nil { + return fmt.Errorf("%sError '%s'creating keyring", + errPrefix, err) + } + + // Import public key + if err := util.RunSystemCmd("gpg", "--import", pubKeyPath); err != nil { + return fmt.Errorf("%sError '%s' importing public-key %s", + errPrefix, err, pubKeyPath) + } + + var verifyRepoCmd []string + revision := gitSpec.Revision + revisionType, err := gitSpec.typeOfGitRevision() + if err != nil { + return fmt.Errorf("%sinvalid revision %s provided, provide either a COMMIT or TAG %s", errPrefix, revision, err) + } + if revisionType == COMMIT { + verifyRepoCmd = []string{"verify-commit", "-v", revision} + } else if revisionType == TAG { + verifyRepoCmd = []string{"verify-tag", "-v", revision} + } else { + return fmt.Errorf("%sinvalid revision %s provided, provide either a COMMIT or TAG", errPrefix, revision) + } + + clonedDir := gitSpec.ClonedDir + err = util.RunSystemCmdInDir(clonedDir, "git", verifyRepoCmd...) + if err != nil { + return fmt.Errorf("%serror during verifying git repo at %s: %s", errPrefix, clonedDir, err) + } + + return nil +} diff --git a/impl/create_srpm_for_others.go b/impl/create_srpm_for_others.go new file mode 100644 index 00000000..8f4dfd1e --- /dev/null +++ b/impl/create_srpm_for_others.go @@ -0,0 +1,193 @@ +// Copyright (c) 2023 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +package impl + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "code.arista.io/eos/tools/eext/manifest" + "code.arista.io/eos/tools/eext/srcconfig" + "code.arista.io/eos/tools/eext/util" +) + +func (bldr *srpmBuilder) getUpstreamSourceForOthers(upstreamSrcFromManifest manifest.UpstreamSrc, + downloadDir string) (*upstreamSrcSpec, error) { + + repo := bldr.repo + pkg := bldr.pkgSpec.Name + isPkgSubdirInRepo := bldr.pkgSpec.Subdir + + srcParams, err := srcconfig.GetSrcParams( + pkg, + upstreamSrcFromManifest.FullURL, + upstreamSrcFromManifest.SourceBundle.Name, + upstreamSrcFromManifest.Signature.DetachedSignature.FullURL, + upstreamSrcFromManifest.SourceBundle.SrcRepoParamsOverride, + upstreamSrcFromManifest.Signature.DetachedSignature.OnUncompressed, + bldr.srcConfig, + bldr.errPrefix) + if err != nil { + return nil, fmt.Errorf("%sUnable to get source params for %s", + err, upstreamSrcFromManifest.SourceBundle.Name) + } + + var downloadErr error + upstreamSrc := upstreamSrcSpec{} + + upstreamSrcType := bldr.pkgSpec.Type + bldr.log("downloading %s", srcParams.SrcURL) + // Download source + if upstreamSrc.sourceFile, downloadErr = download( + srcParams.SrcURL, + downloadDir, + repo, pkg, isPkgSubdirInRepo, + bldr.errPrefix); downloadErr != nil { + return nil, downloadErr + } + bldr.log("downloaded") + + upstreamSrc.skipSigCheck = upstreamSrcFromManifest.Signature.SkipCheck + pubKey := upstreamSrcFromManifest.Signature.DetachedSignature.PubKey + + if upstreamSrcType == "tarball" && !upstreamSrc.skipSigCheck { + if srcParams.SignatureURL == "" || pubKey == "" { + return nil, fmt.Errorf("%sNo detached-signature/public-key specified for upstream-sources entry %s", + bldr.errPrefix, srcParams.SrcURL) + } + if upstreamSrc.sigFile, downloadErr = download( + srcParams.SignatureURL, + downloadDir, + repo, pkg, isPkgSubdirInRepo, + bldr.errPrefix); downloadErr != nil { + return nil, downloadErr + } + + pubKeyPath := filepath.Join(getDetachedSigDir(), pubKey) + if pathErr := util.CheckPath(pubKeyPath, false, false); pathErr != nil { + return nil, fmt.Errorf("%sCannot find public-key at path %s", + bldr.errPrefix, pubKeyPath) + } + upstreamSrc.pubKeyPath = pubKeyPath + } else if upstreamSrcType == "srpm" || upstreamSrcType == "unmodified-srpm" { + // We don't expect SRPMs to have detached signature or + // to be validated with a public-key specified in manifest. + if srcParams.SignatureURL != "" { + return nil, fmt.Errorf("%sUnexpected detached-sig specified for SRPM", + bldr.errPrefix) + } + if pubKey != "" { + return nil, fmt.Errorf("%sUnexpected public-key specified for SRPM", + bldr.errPrefix) + } + } + + return &upstreamSrc, nil +} + +// verifyRpmSignature verifies that the RPM specified at rpmPath +// is signed with a valid key in the key ring and that the signatures +// are valid. +func verifyRpmSignature(rpmPath string, errPrefix util.ErrPrefix) error { + output, err := util.CheckOutput("rpm", "-K", rpmPath) + if err != nil { + return fmt.Errorf("%s:%s", errPrefix, err) + } + if !strings.Contains(output, "digests signatures OK") { + return fmt.Errorf("%sSignature check of %s failed. rpm -K output:\n%s", + errPrefix, rpmPath, output) + } + return nil +} + +// checkValidSignature verifies that tarball anf signature +// correspond to same package +func checkValidSignature(tarballPath, tarballSigPath string) ( + bool, bool) { + lastDotIndex := strings.LastIndex(tarballSigPath, ".") + if lastDotIndex == -1 || !strings.HasPrefix( + tarballPath, tarballSigPath[:lastDotIndex]) { + return false, false + } + decompress := strings.Count(tarballPath[lastDotIndex:], ".") + dcmprsnReqd := (decompress > 0) + return true, dcmprsnReqd +} + +// uncompressTarball decompresses the compression one layer at a time +// to match the tarball with its valid signature +func uncompressTarball(tarballPath string, downloadDir string) (string, error) { + if err := util.RunSystemCmd( + "7za", "x", + "-y", tarballPath, + "-o"+downloadDir); err != nil { + return "", err + } + lastDotIndex := strings.LastIndex(tarballPath, ".") + return tarballPath[:lastDotIndex], nil +} + +// matchTarballSignCmprsn evaluvates and finds correct compressed/uncompressed tarball +// that matches with the sign file. +func matchTarballSignCmprsn(tarballPath string, tarballSigPath string, + downloadDir string, errPrefix util.ErrPrefix) (string, error) { + ok, dcmprsnReqd := checkValidSignature(tarballPath, tarballSigPath) + if !ok { + return "", fmt.Errorf("%sError while matching tarball and signature", + errPrefix) + } + if dcmprsnReqd { + newTarballPath, err := uncompressTarball(tarballPath, downloadDir) + if err != nil { + return "", fmt.Errorf("%sError '%s' while decompressing trarball", + errPrefix, err) + } + return newTarballPath, nil + } + return "", nil +} + +// VerifyTarballSignature verifies that the detached signature of the tarball +// is valid. +func verifyTarballSignature( + tarballPath string, tarballSigPath string, pubKeyPath string, + errPrefix util.ErrPrefix) error { + tmpDir, mkdtErr := os.MkdirTemp("", "eext-keyring") + if mkdtErr != nil { + return fmt.Errorf("%sError '%s'creating temp dir for keyring", + errPrefix, mkdtErr) + } + defer os.RemoveAll(tmpDir) + + keyRingPath := filepath.Join(tmpDir, "eext.gpg") + baseArgs := []string{ + "--homedir", tmpDir, + "--no-default-keyring", "--keyring", keyRingPath} + gpgCmd := "gpg" + + // Create keyring + createKeyRingCmdArgs := append(baseArgs, "--fingerprint") + if err := util.RunSystemCmd(gpgCmd, createKeyRingCmdArgs...); err != nil { + return fmt.Errorf("%sError '%s'creating keyring", + errPrefix, err) + } + + // Import public key + importKeyCmdArgs := append(baseArgs, "--import", pubKeyPath) + if err := util.RunSystemCmd(gpgCmd, importKeyCmdArgs...); err != nil { + return fmt.Errorf("%sError '%s' importing public-key %s", + errPrefix, err, pubKeyPath) + } + + verifySigArgs := append(baseArgs, "--verify", tarballSigPath, tarballPath) + if output, err := util.CheckOutput(gpgCmd, verifySigArgs...); err != nil { + return fmt.Errorf("%sError verifying signature %s for tarball %s with pubkey %s."+ + "\ngpg --verify err: %sstdout:%s", + errPrefix, tarballSigPath, tarballPath, pubKeyPath, err, output) + } + + return nil +} diff --git a/impl/create_srpm_from_git_test.go b/impl/create_srpm_from_git_test.go new file mode 100644 index 00000000..134ab90d --- /dev/null +++ b/impl/create_srpm_from_git_test.go @@ -0,0 +1,254 @@ +// Copyright (c) 2023 Arista Networks, Inc. All rights reserved. +// Arista Networks, Inc. Confidential and Proprietary. + +package impl + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "code.arista.io/eos/tools/eext/util" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +type TestDataType struct { + gitSpec *gitSpec + expectedValue string +} + +// We are currently using a tarball of the libpcap repo, and extracting it in a temp folder. +// This ensures that we mock 'cloneGitRepo' and steps after are tested. +// If we migrate to a remote repo, we can use this function to update the url. +func getSrcURL() string { + url := "https://artifactory.infra.corp.arista.io/artifactory/eext-sources/eext-testData/libpcap.tar" + return url +} + +func downloadTarball(url, targetDir string) (string, error) { + tarBallFilePath := filepath.Join(targetDir, "libpcap.tar") + out, err := os.Create(tarBallFilePath) + if err != nil { + return "", fmt.Errorf("failed to create file: %s", err) + } + defer out.Close() + + resp, err := http.Get(url) + if err != nil { + return "", fmt.Errorf("failed to download file: %s", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("server returned: %s", resp.Status) + } + _, err = io.Copy(out, resp.Body) + if err != nil { + return "", fmt.Errorf("failed to write to file: %s", err) + } + + return tarBallFilePath, nil +} + +// Gets the .tar file from test repo, and untars it into the required git repo. +func cloneRepoFromUrl(url, targetDir string) (string, error) { + tarBallFilePath, err := downloadTarball(url, targetDir) + if err != nil { + return "", fmt.Errorf("failed to download tarball from %s: %s", url, err) + } + + err = util.RunSystemCmdInDir(targetDir, "tar", "-xvf", tarBallFilePath) + if err != nil { + return "", fmt.Errorf("failed to extract tarball %s: %s", tarBallFilePath, err) + } + + clonedDir := filepath.Join(targetDir, "libpcap") + fmt.Println(clonedDir) + + // suppress git error 128 (dubious ownership) + user, err := util.CheckOutput("whoami", []string{}...) + if err != nil { + return "", fmt.Errorf("failed to get current user %s", err) + } + suppressCmdArgs := []string{"chown", "-R", strings.TrimSpace(user), clonedDir} + err = util.RunSystemCmd("sudo", suppressCmdArgs...) + if err != nil { + return "", fmt.Errorf("failed to suppress git warning %s", err) + } + + return clonedDir, nil +} + +// A mock function for cloneGitRepo(). +// Since we do not use remote git repo, this function downloads a tarball from a test repo, +// and expands it to be used as though we have cloned the repo from git. +func cloneGitDir() (string, error) { + srcURL := getSrcURL() + tempDir, err := os.MkdirTemp("", "upstream-git-test") + if err != nil { + return "", fmt.Errorf("failed to create temp dir: %s", err) + } + + clonedDir, err := cloneRepoFromUrl(srcURL, tempDir) + if err != nil { + return "", fmt.Errorf("failed to clone repo dir from source %s at %s: %s", srcURL, tempDir, err) + } + + return clonedDir, nil +} + +func populateTestData(cloneDir string, revisionList, expectedList []string) []*TestDataType { + // Not used in any tests currently, since we mock cloneGitRepo. + // Will be usefull for testing gitSpec.typeOfGitRevisionFromRemote. + srcURL := getSrcURL() + + var dataList []*TestDataType + for i, revision := range revisionList { + gitSpec := &gitSpec{ + SrcUrl: srcURL, + Revision: revision, + ClonedDir: cloneDir, + } + dataType := &TestDataType{ + gitSpec: gitSpec, + expectedValue: expectedList[i], + } + dataList = append(dataList, dataType) + } + + return dataList +} + +func populateTestDataForRevision(cloneDir string) []*TestDataType { + revisionList := []string{"libpcap-1.10.4", "95691eb", "59747a7e74506bd2fbf6cc668e1d66b68ac6eb6d"} + expectedList := []string{"TAG", "COMMIT", "COMMIT"} + + testData := populateTestData(cloneDir, revisionList, expectedList) + + // Required for testing typeOfGitRevisionFromRemote() + // Keep disabled until we start using remote test data. + /*for i, data := range testData { + if i%2 == 0 { + data.gitSpec.ClonedDir = "" + } + }*/ + + return testData +} + +func populateTestDataForGitSignature(cloneDir string) []*TestDataType { + // Yet to verify commit signatures, + // since not many commits signed with public keys are available. + revisionList := []string{"libpcap-1.10.1"} + expectedList := []string{""} + + return populateTestData(cloneDir, revisionList, expectedList) +} + +func TestRevisionType(t *testing.T) { + cloneDir, err := cloneGitDir() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(cloneDir) + + // To reuse populateTestData(), we convert the obtained GitRevisionType to string + resolveGitRevisionTypeToString := []string{"UNDEFINED", "COMMIT", "TAG"} + testDataList := populateTestDataForRevision(cloneDir) + for _, data := range testDataList { + gitSpec := data.gitSpec + expectedType := data.expectedValue + + typeLocalRepo, err := gitSpec.typeOfGitRevision() + if err != nil { + t.Fatal(err) + } + + // Test requires call to remote git repo. + // Enable when we use remote repo for test. + /*typeRemoteRepo, err := gitSpec.typeOfGitRevisionFromRemote() + if err != nil { + t.Fatal(err) + }*/ + + require.Equal(t, expectedType, resolveGitRevisionTypeToString[typeLocalRepo]) + //require.Equal(t, expectedType, resolveGitRevisionTypeToString[typeRemoteRepo]) + } + t.Log("Test typeOfGitRevision PASSED") +} + +func TestRpmNameFromSpecFile(t *testing.T) { + viper.Set("SrcDir", "testData/") + defer viper.Reset() + pkg := "libpcap" + repo := "upstream-git-repo-1" + expectedRpmName := "libpcap-1.10.1" + + gotRpmName, err := getRpmNameFromSpecFile(repo, pkg, false) + if err != nil { + t.Fatal(err) + } + + require.Equal(t, expectedRpmName, gotRpmName) + t.Log("Test rpmNameFromSpecFile PASSED") +} + +func TestVerifyGitSignature(t *testing.T) { + cloneDir, err := cloneGitDir() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(cloneDir) + + viper.Set("PkiPath", "../pki") + defer viper.Reset() + pubKeyPath := filepath.Join(getDetachedSigDir(), "tcpdump/tcpdumpPubKey.pem") + testData := populateTestDataForGitSignature(cloneDir) + for _, data := range testData { + gitSpec := data.gitSpec + + err := verifyGitSignature(pubKeyPath, *gitSpec, "") + if err != nil { + t.Fatal(err) + } + } + t.Log("Test verifyGitRepoSignature PASSED") +} + +func TestGitArchive(t *testing.T) { + clonedDir, err := cloneGitDir() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(clonedDir) + + testWorkingDir, mkdirErr := os.MkdirTemp("", "upstream-git") + if mkdirErr != nil { + t.Fatal(mkdirErr) + } + defer os.RemoveAll(testWorkingDir) + + viper.Set("SrcDir", "testData/") + defer viper.Reset() + pkg := "libpcap" + repo := "upstream-git-repo-1" + revision := "libpcap-1.10.1" + + archiveFile, err := generateArchiveFile(testWorkingDir, clonedDir, revision, repo, pkg, false, "") + if err != nil { + t.Fatal(err) + } + + archivePath := filepath.Join(testWorkingDir, archiveFile) + err = util.CheckPath(archivePath, false, false) + if err != nil { + t.Fatal(err) + } + + t.Log("Test gitArchive PASSED") +} diff --git a/impl/create_srpm_from_others_test.go b/impl/create_srpm_from_others_test.go new file mode 100644 index 00000000..92981df5 --- /dev/null +++ b/impl/create_srpm_from_others_test.go @@ -0,0 +1,44 @@ +package impl + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func testTarballSig(t *testing.T, folder string) { + curPath, _ := os.Getwd() + workingDir := filepath.Join(curPath, "testData/tarballSig", folder) + tarballPath := map[string]string{ + "checkTarball": filepath.Join(workingDir, "linux.10.4.1.tar.gz"), + "matchTarball": filepath.Join(workingDir, "libpcap-1.10.4.tar.gz"), + } + tarballSigPath := filepath.Join(workingDir, "libpcap-1.10.4.tar.gz.sig") + + switch folder { + case "checkTarball": + ok, _ := checkValidSignature(tarballPath[folder], tarballSigPath) + require.Equal(t, false, ok) + case "matchTarball": + intermediateTarball, err := matchTarballSignCmprsn( + tarballPath[folder], + tarballSigPath, + workingDir, + "TestmatchTarballSignature : ", + ) + os.Remove(intermediateTarball) + require.Equal(t, nil, err) + } +} + +func TestCheckTarballSignature(t *testing.T) { + t.Log("Test tarball Signatue Check") + testTarballSig(t, "checkTarball") +} + +func TestMatchTarballSignature(t *testing.T) { + t.Log("Test tarball Signatue Match") + testTarballSig(t, "matchTarball") +} diff --git a/impl/testData/upstream-git-repo-1/eext.yaml b/impl/testData/upstream-git-repo-1/eext.yaml new file mode 100644 index 00000000..e0c6297a --- /dev/null +++ b/impl/testData/upstream-git-repo-1/eext.yaml @@ -0,0 +1,14 @@ +--- +package: + - name: libpcap + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: libpcap-1.10.1 + signature: + detached-sig: + public-key: tcpdump/tcpdumpPubKey.pem + type: git + build: + repo-bundle: + - name: el9 diff --git a/impl/testData/upstream-git-repo-1/spec/libpcap.spec b/impl/testData/upstream-git-repo-1/spec/libpcap.spec new file mode 100644 index 00000000..5120ebdc --- /dev/null +++ b/impl/testData/upstream-git-repo-1/spec/libpcap.spec @@ -0,0 +1,174 @@ +%define libpcap_version 1.10.1 + +Name: libpcap +Epoch: 14 +Version: %{libpcap_version} +Release: 1%{?dist}.Ar.1.%{?eext_release:%{eext_release}}%{!?eext_release:eng} +Summary: A system-independent interface for user-level packet capture +Group: Development/Libraries +License: BSD with advertising +URL: http://www.tcpdump.org +BuildRequires: glibc-kernheaders >= 2.2.0 git bison flex libnl3-devel gcc +Requires: libnl3 +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) + +Source0: libpcap-%libpcap-{libpcap_version}.tar.gz + + + +%description +Libpcap provides a portable framework for low-level network +monitoring. Libpcap can provide network statistics collection, +security monitoring and network debugging. Since almost every system +vendor provides a different interface for packet capture, the libpcap +authors created this system-independent API to ease in porting and to +alleviate the need for several system-dependent packet capture modules +in each application. + +Install libpcap if you need to do low-level network traffic monitoring +on your network. + + +%package devel +Summary: Libraries and header files for the libpcap library +Group: Development/Libraries +Requires: %{name} = %{epoch}:%{version}-%{release} + +%description devel +Libpcap provides a portable framework for low-level network +monitoring. Libpcap can provide network statistics collection, +security monitoring and network debugging. Since almost every system +vendor provides a different interface for packet capture, the libpcap +authors created this system-independent API to ease in porting and to +alleviate the need for several system-dependent packet capture modules +in each application. + +This package provides the libraries, include files, and other +resources needed for developing libpcap applications. + +%prep +%autosetup -S git -n libpcap-%{libpcap_version} + +#sparc needs -fPIC +%ifarch %{sparc} +sed -i -e 's|-fpic|-fPIC|g' configure +%endif + +find . -name '*.c' -o -name '*.h' | xargs chmod 644 + +%build +export CFLAGS="$RPM_OPT_FLAGS -fno-strict-aliasing" +# Explicitly specify each configure flag to avoid dynamically deciding what +# features to include. If we want a feature, ensure that we have the +# supporting libraries listed as BuildRequires/Requires above. +# Read configure.ac to figure out whether it's "--enable/--disable" +# or "--with/--without". +%configure --enable-usb --disable-netmap --without-dpdk --disable-bluetooth \ + --disable-dbus --disable-rdma --without-dag --without-septel --without-snf \ + --without-turbocap +%{?a4_configure:exit 0} +make %{?_smp_mflags} + +%install +rm -rf $RPM_BUILD_ROOT +make install DESTDIR=$RPM_BUILD_ROOT +rm -f $RPM_BUILD_ROOT%{_libdir}/libpcap.a + +%clean +rm -rf $RPM_BUILD_ROOT + +%post -p /sbin/ldconfig + +%postun -p /sbin/ldconfig + + +%files +%defattr(-,root,root) +%doc LICENSE README.md CHANGES CREDITS +%{_libdir}/libpcap.so.* +%{_mandir}/man7/pcap*.7* + +# Listing all include files individually for the benefit of pkgdeps +%files devel +%defattr(-,root,root) +%{_bindir}/pcap-config +%{_includedir}/pcap-bpf.h +%{_includedir}/pcap-namedb.h +%{_includedir}/pcap.h +%{_includedir}/pcap/bluetooth.h +%{_includedir}/pcap/bpf.h +%{_includedir}/pcap/can_socketcan.h +%{_includedir}/pcap/compiler-tests.h +%{_includedir}/pcap/dlt.h +%{_includedir}/pcap/funcattrs.h +%{_includedir}/pcap/ipnet.h +%{_includedir}/pcap/namedb.h +%{_includedir}/pcap/nflog.h +%{_includedir}/pcap/pcap-inttypes.h +%{_includedir}/pcap/pcap.h +%{_includedir}/pcap/sll.h +%{_includedir}/pcap/socket.h +%{_includedir}/pcap/usb.h +%{_includedir}/pcap/vlan.h +%{_libdir}/libpcap.so +%{_libdir}/pkgconfig/libpcap.pc +%{_mandir}/* + + +%changelog +* Fri Apr 22 2011 Miroslav Lichvar 14:1.1.1-3 +- ignore /sys/net/dev files on ENODEV (#693943) +- drop ppp patch +- compile with -fno-strict-aliasing + +* Tue Feb 08 2011 Fedora Release Engineering - 14:1.1.1-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_15_Mass_Rebuild + +* Tue Apr 06 2010 Miroslav Lichvar 14:1.1.1-1 +- update to 1.1.1 + +* Wed Dec 16 2009 Miroslav Lichvar 14:1.0.0-5.20091201git117cb5 +- update to snapshot 20091201git117cb5 + +* Sat Oct 17 2009 Dennis Gilmore 14:1.0.0-4.20090922gite154e2 +- use -fPIC on sparc arches + +* Wed Sep 23 2009 Miroslav Lichvar 14:1.0.0-3.20090922gite154e2 +- update to snapshot 20090922gite154e2 +- drop old soname + +* Fri Jul 24 2009 Fedora Release Engineering - 14:1.0.0-2.20090716git6de2de +- Rebuilt for https://fedoraproject.org/wiki/Fedora_12_Mass_Rebuild + +* Wed Jul 22 2009 Miroslav Lichvar 14:1.0.0-1.20090716git6de2de +- update to 1.0.0, git snapshot 20090716git6de2de + +* Wed Feb 25 2009 Fedora Release Engineering - 14:0.9.8-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_11_Mass_Rebuild + +* Fri Jun 27 2008 Miroslav Lichvar 14:0.9.8-3 +- use CFLAGS when linking (#445682) + +* Tue Feb 19 2008 Fedora Release Engineering - 14:0.9.8-2 +- Autorebuild for GCC 4.3 + +* Wed Oct 24 2007 Miroslav Lichvar 14:0.9.8-1 +- update to 0.9.8 + +* Wed Aug 22 2007 Miroslav Lichvar 14:0.9.7-3 +- update license tag + +* Wed Jul 25 2007 Jesse Keating - 14:0.9.7-2 +- Rebuild for RH #249435 + +* Tue Jul 24 2007 Miroslav Lichvar 14:0.9.7-1 +- update to 0.9.7 + +* Tue Jun 19 2007 Miroslav Lichvar 14:0.9.6-1 +- update to 0.9.6 + +* Tue Nov 28 2006 Miroslav Lichvar 14:0.9.5-1 +- split from tcpdump package (#193657) +- update to 0.9.5 +- don't package static library +- maintain soname \ No newline at end of file diff --git a/manifest/manifest.go b/manifest/manifest.go index afabb5a8..5296447d 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -5,7 +5,7 @@ package manifest import ( "fmt" - "io/ioutil" + "os" "path/filepath" "golang.org/x/exp/slices" @@ -118,12 +118,18 @@ type SourceBundle struct { SrcRepoParamsOverride srcconfig.SrcRepoParamsOverride `yaml:"override"` } +type GitBundle struct { + Url string `yaml:"url"` + Revision string `yaml:"revision"` +} + // UpstreamSrc spec // Lists each source bundle(tarball/srpm) and // detached signature file for tarball. type UpstreamSrc struct { SourceBundle SourceBundle `yaml:"source-bundle"` FullURL string `yaml:"full-url"` + GitBundle GitBundle `yaml:"git"` Signature Signature `yaml:"signature"` } @@ -147,7 +153,7 @@ type Manifest struct { } func (m Manifest) sanityCheck() error { - allowedPkgTypes := []string{"srpm", "unmodified-srpm", "tarball", "standalone"} + allowedPkgTypes := []string{"srpm", "unmodified-srpm", "tarball", "standalone", "git-upstream"} for _, pkgSpec := range m.Package { if pkgSpec.Name == "" { @@ -184,23 +190,49 @@ func (m Manifest) sanityCheck() error { } for _, upStreamSrc := range pkgSpec.UpstreamSrc { - specifiedFullSrcURL := (upStreamSrc.FullURL != "") - specifiedSrcBundle := (upStreamSrc.SourceBundle != SourceBundle{}) - if !specifiedFullSrcURL && !specifiedSrcBundle { - return fmt.Errorf("Specify source for Build in package %s, provide either full-url or source-bundle", - pkgSpec.Name) - } + if pkgSpec.Type == "git-upstream" { + specifiedUrl := (upStreamSrc.GitBundle.Url != "") + specifiedRevision := (upStreamSrc.GitBundle.Revision != "") + if !specifiedUrl { + return fmt.Errorf("please provide the url for git repo of package %s", pkgSpec.Name) + } + if !specifiedRevision { + return fmt.Errorf("please provide a commit/tag to define revision of package %s", pkgSpec.Name) + } - if specifiedFullSrcURL && specifiedSrcBundle { - return fmt.Errorf( - "Conflicting sources for Build in package %s, provide either full-url or source-bundle", - pkgSpec.Name) - } + specifiedSignature := (upStreamSrc.Signature != Signature{}) + if specifiedSignature { + skipSigCheck := (upStreamSrc.Signature.SkipCheck) + specifiedPubKey := (upStreamSrc.Signature.DetachedSignature.PubKey != "") + if !skipSigCheck && !specifiedPubKey { + return fmt.Errorf( + "please provide the public key to verify git repo for package %s, or skip signature check", + pkgSpec.Name) + } + } else { + return fmt.Errorf( + "signature fields not specified for package %s, provide public key or skip signature check", + pkgSpec.Name) + } + } else { + specifiedFullSrcURL := (upStreamSrc.FullURL != "") + specifiedSrcBundle := (upStreamSrc.SourceBundle != SourceBundle{}) + if !specifiedFullSrcURL && !specifiedSrcBundle { + return fmt.Errorf("Specify source for Build in package %s, provide either full-url or source-bundle", + pkgSpec.Name) + } - specifiedFullSigURL := upStreamSrc.Signature.DetachedSignature.FullURL != "" - if specifiedFullSigURL && specifiedSrcBundle { - return fmt.Errorf("Conflicting signatures for Build in package %s, provide full-url or source-bundle", - pkgSpec.Name) + if specifiedFullSrcURL && specifiedSrcBundle { + return fmt.Errorf( + "Conflicting sources for Build in package %s, provide either full-url or source-bundle", + pkgSpec.Name) + } + + specifiedFullSigURL := upStreamSrc.Signature.DetachedSignature.FullURL != "" + if specifiedFullSigURL && specifiedSrcBundle { + return fmt.Errorf("Conflicting signatures for Build in package %s, provide full-url or source-bundle", + pkgSpec.Name) + } } } } @@ -213,9 +245,9 @@ func LoadManifest(repo string) (*Manifest, error) { repoDir := util.GetRepoDir(repo) yamlPath := filepath.Join(repoDir, "eext.yaml") - yamlContents, readErr := ioutil.ReadFile(yamlPath) + yamlContents, readErr := os.ReadFile(yamlPath) if readErr != nil { - return nil, fmt.Errorf("manifest.LoadManifest: ioutil.ReadFile on %s returned %s", yamlPath, readErr) + return nil, fmt.Errorf("manifest.LoadManifest: os.ReadFile on %s returned %s", yamlPath, readErr) } var manifest Manifest diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 45bb33cf..f417a0e9 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -31,12 +31,15 @@ func TestManifest(t *testing.T) { viper.Set("SrcDir", dir) defer viper.Reset() - t.Log("Copy sample manifest to test directory") - testutil.SetupManifest(t, dir, "pkg1", "sampleManifest1.yaml") + testFiles := []string{"sampleManifest1.yaml", "sampleManifest4.yaml"} + for _, testFile := range testFiles { + t.Logf("Copy sample manifest %s to test directory", testFile) + testutil.SetupManifest(t, dir, "pkg1", testFile) - t.Log("Testing Load") - testLoad(t, "pkg1") - t.Log("Load test passed") + t.Log("Testing Load") + testLoad(t, "pkg1") + t.Log("Load test passed") + } } type manifestTestVariant struct { @@ -57,16 +60,21 @@ func TestManifestNegative(t *testing.T) { defer viper.Reset() testCases := map[string]manifestTestVariant{ - "testBundleAndFullURL": manifestTestVariant{ + "testBundleAndFullURL": { TestPkg: "pkg2", ManifestFile: "sampleManifest2.yaml", ExpectedErr: "Conflicting sources for Build in package libpcap, provide either full-url or source-bundle", }, - "testBundleAndSignature": manifestTestVariant{ + "testBundleAndSignature": { TestPkg: "pkg3", ManifestFile: "sampleManifest3.yaml", ExpectedErr: "Conflicting signatures for Build in package tcpdump, provide full-url or source-bundle", }, + "testGitUpstreamWithoutSignature": { + TestPkg: "pkg5", + ManifestFile: "sampleManifest5.yaml", + ExpectedErr: "signature fields not specified for package libpcap, provide public key or skip signature check", + }, } for testName, variant := range testCases { t.Logf("%s: Copy sample manifest to test directory", testName) diff --git a/manifest/testData/sampleManifest4.yaml b/manifest/testData/sampleManifest4.yaml new file mode 100644 index 00000000..d6a3c3bc --- /dev/null +++ b/manifest/testData/sampleManifest4.yaml @@ -0,0 +1,30 @@ +--- +package: + - name: libpcap1 + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: 104271ba4a14de6743e43bcf87536786d8fddea4 + signature: + detached-sig: + public-key: mrtparse/mrtparsePubKey.pem + type: git-upstream + build: + repo-bundle: + - name: foo + version: v1 + - name: bar + + - name: libpcap2 + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: libpcap-1.10.1 + signature: + skip-check: true + type: git-upstream + build: + repo-bundle: + - name: foo + version: v1 + - name: bar diff --git a/manifest/testData/sampleManifest5.yaml b/manifest/testData/sampleManifest5.yaml new file mode 100644 index 00000000..ba38e989 --- /dev/null +++ b/manifest/testData/sampleManifest5.yaml @@ -0,0 +1,13 @@ +--- +package: + - name: libpcap + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: 104271ba4a14de6743e43bcf87536786d8fddea4 + type: git-upstream + build: + repo-bundle: + - name: foo + version: v1 + - name: bar diff --git a/srcconfig/srcconfig.go b/srcconfig/srcconfig.go index f63d3b10..d3d5eb09 100644 --- a/srcconfig/srcconfig.go +++ b/srcconfig/srcconfig.go @@ -6,7 +6,6 @@ package srcconfig import ( "bytes" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -222,9 +221,9 @@ func LoadSrcConfig() (*SrcConfig, error) { cfgPath, statErr) } - yamlContents, readErr := ioutil.ReadFile(cfgPath) + yamlContents, readErr := os.ReadFile(cfgPath) if readErr != nil { - return nil, fmt.Errorf("srcconfig.LoadSrcConfig: ioutil.ReadFile on %s returned %s", + return nil, fmt.Errorf("srcconfig.LoadSrcConfig: os.ReadFile on %s returned %s", cfgPath, readErr) } diff --git a/testutil/testutil.go b/testutil/testutil.go index dde67794..532510a6 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -5,7 +5,7 @@ package testutil import ( "fmt" - "io/ioutil" + "io" "os" "os/exec" "path/filepath" @@ -191,7 +191,7 @@ func setupQuiet() { func checkAndCleanupQuiet(t *testing.T) { w.Close() - out, err := ioutil.ReadAll(r) + out, err := io.ReadAll(r) if err != nil { t.Fatal(err) } diff --git a/util/util.go b/util/util.go index e071dd38..c0318e2e 100644 --- a/util/util.go +++ b/util/util.go @@ -39,6 +39,20 @@ func RunSystemCmd(name string, arg ...string) error { return err } +// Runs the system command from a specified directory +func RunSystemCmdInDir(dir string, name string, arg ...string) error { + cmd := exec.Command(name, arg...) + cmd.Dir = dir + cmd.Stderr = os.Stderr + if !GlobalVar.Quiet { + cmd.Stdout = os.Stdout + } else { + cmd.Stdout = io.Discard + } + err := cmd.Run() + return err +} + // CheckOutput runs a command on the shell and returns stdout if it is successful // else it return the error func CheckOutput(name string, arg ...string) ( @@ -48,11 +62,11 @@ func CheckOutput(name string, arg ...string) ( if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { return string(output), - fmt.Errorf("Running '%s %s': exited with exit-code %d\nstderr:\n%s", + fmt.Errorf("running '%s %s': exited with exit-code %d\nstderr:\n%s", name, strings.Join(arg, " "), exitErr.ExitCode(), exitErr.Stderr) } return string(output), - fmt.Errorf("Running '%s %s' failed with '%s'", + fmt.Errorf("running '%s %s' failed with '%s'", name, strings.Join(arg, " "), err) } return string(output), nil @@ -147,111 +161,3 @@ func GetRepoDir(repo string) string { } return repoDir } - -// VerifyRpmSignature verifies that the RPM specified at rpmPath -// is signed with a valid key in the key ring and that the signatures -// are valid. -func VerifyRpmSignature(rpmPath string, errPrefix ErrPrefix) error { - output, err := CheckOutput("rpm", "-K", rpmPath) - if err != nil { - return fmt.Errorf("%s:%s", errPrefix, err) - } - if !strings.Contains(output, "digests signatures OK") { - return fmt.Errorf("%sSignature check of %s failed. rpm -K output:\n%s", - errPrefix, rpmPath, output) - } - return nil -} - -// CheckValidSignature verifies that tarball anf signature -// correspond to same package -func CheckValidSignature(tarballPath, tarballSigPath string) ( - bool, bool) { - lastDotIndex := strings.LastIndex(tarballSigPath, ".") - if lastDotIndex == -1 || !strings.HasPrefix( - tarballPath, tarballSigPath[:lastDotIndex]) { - return false, false - } - decompress := strings.Count(tarballPath[lastDotIndex:], ".") - dcmprsnReqd := false - if decompress > 0 { - dcmprsnReqd = true - } - return true, dcmprsnReqd -} - -// UncompressTarball decompresses the compression one layer at a time -// to match the tarball with its valid signature -func uncompressTarball(tarballPath string, downloadDir string) (string, error) { - if err := RunSystemCmd( - "7za", "x", - "-y", tarballPath, - "-o"+downloadDir); err != nil { - return "", err - } - lastDotIndex := strings.LastIndex(tarballPath, ".") - return tarballPath[:lastDotIndex], nil -} - -// MatchtarballSignCmprsn evaluvates and finds correct compressed/uncompressed tarball -// that matches with the sign file. -func MatchtarballSignCmprsn(tarballPath string, tarballSigPath string, - downloadDir string, errPrefix ErrPrefix) (string, error) { - uncompressedTarball := "" - ok, dcmprsnReqd := CheckValidSignature(tarballPath, tarballSigPath) - if !ok { - return uncompressedTarball, fmt.Errorf("%sError while matching tarball and signature", - errPrefix) - } - if dcmprsnReqd { - newTarball, err := uncompressTarball(tarballPath, downloadDir) - if err != nil { - return uncompressedTarball, fmt.Errorf("%sError '%s' while decompressing trarball", - errPrefix, err) - } - uncompressedTarball = newTarball - } - return uncompressedTarball, nil -} - -// VerifyTarballSignature verifies that the detached signature of the tarball -// is valid. -func VerifyTarballSignature( - tarballPath string, tarballSigPath string, pubKeyPath string, - errPrefix ErrPrefix) error { - tmpDir, mkdtErr := os.MkdirTemp("", "eext-keyring") - if mkdtErr != nil { - return fmt.Errorf("%sError '%s'creating temp dir for keyring", - errPrefix, mkdtErr) - } - defer os.RemoveAll(tmpDir) - - keyRingPath := filepath.Join(tmpDir, "eext.gpg") - baseArgs := []string{ - "--homedir", tmpDir, - "--no-default-keyring", "--keyring", keyRingPath} - gpgCmd := "gpg" - - // Create keyring - createKeyRingCmdArgs := append(baseArgs, "--fingerprint") - if err := RunSystemCmd(gpgCmd, createKeyRingCmdArgs...); err != nil { - return fmt.Errorf("%sError '%s'creating keyring", - errPrefix, err) - } - - // Import public key - importKeyCmdArgs := append(baseArgs, "--import", pubKeyPath) - if err := RunSystemCmd(gpgCmd, importKeyCmdArgs...); err != nil { - return fmt.Errorf("%sError '%s' importing public-key %s", - errPrefix, err, pubKeyPath) - } - - verifySigArgs := append(baseArgs, "--verify", tarballSigPath, tarballPath) - if output, err := CheckOutput(gpgCmd, verifySigArgs...); err != nil { - return fmt.Errorf("%sError verifying signature %s for tarball %s with pubkey %s."+ - "\ngpg --verify err: %sstdout:%s", - errPrefix, tarballSigPath, tarballPath, pubKeyPath, err, output) - } - - return nil -}