Merge pull request #1 from RyanJarv/add-http-provider

Add HTTP tar provider for github archives

pkg/buildcontext/azureblob.go

@@ -71,7 +71,7 @@ func (b *AzureBlob) UnpackTarFromBuildContext() (string, error) {
 		return parts.Host, err
 	}
 
-	if err := util.UnpackCompressedTar(tarPath, directory); err != nil {
+	if _, err := util.UnpackCompressedTar(tarPath, directory, false); err != nil {
 		return tarPath, err
 	}
 	// Remove the tar so it doesn't interfere with subsequent commands

pkg/buildcontext/buildcontext.go

@@ -52,6 +52,9 @@ func GetBuildContext(srcContext string) (BuildContext, error) {
 		case constants.GitBuildContextPrefix:
 			return &Git{context: context}, nil
 		case constants.HTTPSBuildContextPrefix:
+			if strings.HasPrefix(srcContext, constants.GitHubArchiveContextHost) {
+				return &GitHubArchive{context: context}, nil
+			}
 			if util.ValidAzureBlobStorageHost(srcContext) {
 				return &AzureBlob{context: srcContext}, nil
 			}

pkg/buildcontext/gcs.go

@@ -66,7 +66,7 @@ func unpackTarFromGCSBucket(bucketName, item, directory string) error {
 		return err
 	}
 	logrus.Debug("Unpacking source context tar...")
-	if err := util.UnpackCompressedTar(tarPath, directory); err != nil {
+	if _, err := util.UnpackCompressedTar(tarPath, directory, false); err != nil {
 		return err
 	}
 	// Remove the tar so it doesn't interfere with subsequent commands

pkg/buildcontext/github.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildcontext
+
+import (
+	"fmt"
+	"github.com/GoogleContainerTools/kaniko/pkg/constants"
+	"github.com/GoogleContainerTools/kaniko/pkg/util"
+	"github.com/pkg/errors"
+	"os"
+	"path/filepath"
+)
+
+// Git unifies calls to download and unpack the build context.
+type GitHubArchive struct {
+	context string
+}
+
+// UnpackTarFromBuildContext will provide the directory where Git Repository is Cloned
+func (g *GitHubArchive) UnpackTarFromBuildContext() (string, error) {
+	uri := "https://" + g.context
+	_, err := downloadAndUnpackTar(uri, constants.BuildContextDir)
+	return constants.BuildContextDir, err
+}
+
+func downloadAndUnpackTar(uri, directory string) ([]string, error) {
+	tarPath := filepath.Join(directory, constants.ContextTar)
+	if err := util.DownloadFileToDest(uri, tarPath, int64(os.Getuid()),  int64(os.Getgid())); err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("downloading tar from %s", uri))
+	}
+	return util.UnpackCompressedTar(tarPath, directory, true)
+}

pkg/buildcontext/s3.go

@@ -74,5 +74,6 @@ func (s *S3) UnpackTarFromBuildContext() (string, error) {
 		return directory, err
 	}
 
-	return directory, util.UnpackCompressedTar(tarPath, directory)
+	_, err = util.UnpackCompressedTar(tarPath, directory, false)
+	return directory, err
 }

pkg/buildcontext/tar.go

@@ -57,5 +57,6 @@ func (t *Tar) UnpackTarFromBuildContext() (string, error) {
 		}
 	}
 
-	return directory, util.UnpackCompressedTar(t.context, directory)
+	_, err := util.UnpackCompressedTar(t.context, directory, false)
+	return directory, err
 }

pkg/buildcontext/tar_test.go

@@ -127,7 +127,7 @@ func TestBuildWithLocalTar(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.dockerfile, func(t *testing.T) {
-			err := util.UnpackCompressedTar(filepath.Join(cwd, tt.srcContext), dirUnpack)
+			_, err := util.UnpackCompressedTar(filepath.Join(cwd, tt.srcContext), dirUnpack, false)
 			testutil.CheckError(t, tt.unpackShouldErr, err)
 			srcSHA, err := getSHAFromFilePath(tt.dockerfile)
 			testutil.CheckError(t, tt.srcShaShouldErr, err)

pkg/constants/constants.go

@@ -57,6 +57,7 @@ const (
 	LocalDirBuildContextPrefix = "dir://"
 	GitBuildContextPrefix      = "git://"
 	HTTPSBuildContextPrefix    = "https://"
+	GitHubArchiveContextHost   = "https://codeload.github.com/"
 
 	HOME = "HOME"
 	// DefaultHOMEValue is the default value Docker sets for $HOME

pkg/util/fs_util.go

@@ -236,7 +236,9 @@ func childDirInIgnoreList(path string) bool {
 }
 
 // unTar returns a list of files that have been extracted from the tar archive at r to the path at dest
-func unTar(r io.Reader, dest string) ([]string, error) {
+// If prefixed is set to true all files move up a directory when being extracted.
+func unTar(r io.Reader, dest string, prefixed bool) ([]string, error) {
+	var lastPrefix string
 	var extractedFiles []string
 	tr := tar.NewReader(r)
 	for {
@@ -247,6 +249,29 @@ func unTar(r io.Reader, dest string) ([]string, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		if prefixed {
+			path := strings.Split(filepath.Clean(hdr.Name), string(filepath.Separator))
+
+			switch hdr.Typeflag {
+			case tar.TypeXGlobalHeader, tar.TypeXHeader:
+				continue
+			case tar.TypeDir:
+				// Append an empty item so we can treat directories the same as files
+				path = append(path, "")
+			}
+
+			if len(path) < 2 {
+				return nil, errors.New(fmt.Sprintf("%s exists above tar extraction prefixed %s", hdr.Name, lastPrefix))
+			}
+			if lastPrefix == "" {
+				lastPrefix = path[0]
+			} else if lastPrefix != path[0] {
+				return nil, errors.New(fmt.Sprintf("%s exists outside tar extraction prefixed %s", hdr.Name, lastPrefix))
+			}
+			hdr.Name = filepath.Join(path[1:]...)
+		}
+
 		if err := ExtractFile(dest, hdr, tr); err != nil {
 			return nil, err
 		}

pkg/util/fs_util_test.go

@@ -616,9 +616,11 @@ func Test_unTar(t *testing.T) {
 		setupTarContents map[string]string
 		tarFileName      string
 		destination      string
+		prefix      	 bool
 		expectedFileList []string
 		errorExpected    bool
 	}{
+		// Prefix = 0 covers util.unTar
 		{
 			name: "multfile tar",
 			setupTarContents: map[string]string{
@@ -629,6 +631,7 @@ func Test_unTar(t *testing.T) {
 			},
 			tarFileName:      "test.tar",
 			destination:      "/",
+			prefix:      	  false,
 			expectedFileList: []string{"foo/file1", "bar/file1", "bar/file2", "file1"},
 			errorExpected:    false,
 		},
@@ -637,9 +640,56 @@ func Test_unTar(t *testing.T) {
 			setupTarContents: map[string]string{},
 			tarFileName:      "test.tar",
 			destination:      "/",
+			prefix:      	  false,
 			expectedFileList: nil,
 			errorExpected:    false,
 		},
+		{
+			name: "removing one prefix dir from a multifile tar",
+			setupTarContents: map[string]string{
+				"prefix/bar/file1": "hello World",
+				"prefix/bar/file2": "hello World",
+				"prefix/file1":     "hello World",
+			},
+			tarFileName:      "test.tar",
+			destination:      "/",
+			prefix:      	  true,
+			expectedFileList: []string{"bar/file1", "bar/file2", "file1"},
+			errorExpected:    false,
+		},
+		{
+			name:             "removing one prefix dir from an empty tar",
+			setupTarContents: map[string]string{},
+			tarFileName:      "test.tar",
+			destination:      "/",
+			prefix:      	  true,
+			expectedFileList: nil,
+			errorExpected:    false,
+		},
+		{
+			name: "removing one prefix dir from a tar with an un-prefixed file",
+			setupTarContents: map[string]string{
+				"bar/file1": "hello World",
+				"bar/file2": "hello World",
+				"should_fail":     "hello World",
+			},
+			tarFileName:      "test.tar",
+			destination:      "/",
+			prefix:      	  true,
+			errorExpected:    true,
+		},
+		{
+			name: "removing one prefix dir from an tar with multiple prefixes",
+			setupTarContents: map[string]string{
+				"foo/file1": "hello World",
+				"foo/file2": "hello World",
+				"bar/should_fail": "hello World",
+			},
+			tarFileName:      "test.tar",
+			destination:      "/",
+			prefix:      	  true,
+			errorExpected:    true,
+		},
 	}
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
@@ -648,15 +698,18 @@ func Test_unTar(t *testing.T) {
 				t.Fatal(err)
 			}
 			defer os.RemoveAll(testDir)
+
+			// update config.RootDir so the files in the tar don't include the testDir path
+			config.RootDir = testDir
 			if err := createUncompressedTar(tc.setupTarContents, tc.tarFileName, testDir); err != nil {
 				t.Fatal(err)
 			}
 			file, err := os.Open(filepath.Join(testDir, tc.tarFileName))
 			if err != nil {
 				t.Fatal(err)
 			}
-			fileList, err := unTar(file, tc.destination)
-			if err != nil {
+			fileList, err := unTar(file, filepath.Join(testDir, tc.destination), tc.prefix)
+			if !tc.errorExpected && err != nil {
 				t.Fatal(err)
 			}
 			// update expectedFileList to take into factor temp directory

pkg/util/tar_util.go

@@ -173,10 +173,10 @@ func UnpackLocalTarArchive(path, dest string) ([]string, error) {
 		}
 		defer file.Close()
 		if compressionLevel == archive.Gzip {
-			return nil, UnpackCompressedTar(path, dest)
+			return UnpackCompressedTar(path, dest, false)
 		} else if compressionLevel == archive.Bzip2 {
 			bzr := bzip2.NewReader(file)
-			return unTar(bzr, dest)
+			return unTar(bzr, dest, false)
 		}
 	}
 	if fileIsUncompressedTar(path) {
@@ -185,7 +185,7 @@ func UnpackLocalTarArchive(path, dest string) ([]string, error) {
 			return nil, err
 		}
 		defer file.Close()
-		return unTar(file, dest)
+		return unTar(file, dest, false)
 	}
 	return nil, errors.New("path does not lead to local tar archive")
 }
@@ -233,17 +233,17 @@ func fileIsUncompressedTar(src string) bool {
 }
 
 // UnpackCompressedTar unpacks the compressed tar at path to dir
-func UnpackCompressedTar(path, dir string) error {
+// If prefixed is set all files move up a directory.
+func UnpackCompressedTar(path, dir string, prefixed bool) ([]string, error) {
 	file, err := os.Open(path)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer file.Close()
 	gzr, err := gzip.NewReader(file)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	defer gzr.Close()
-	_, err = unTar(gzr, dir)
-	return err
+	return unTar(gzr, dir, prefixed)
 }

testutil/util.go

@@ -83,3 +83,4 @@ func checkErr(shouldErr bool, err error) error {
 	}
 	return nil
 }
+