From db918f12ad352f61427edda2551cf012ea8d7e8f Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 2 Nov 2016 15:33:53 -0700 Subject: [PATCH] First working cut --- builddir.go | 256 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ docker_import.go | 58 +++++++++++++ main.go | 79 +++++++++++++++++ sha.go | 43 ++++++++++ 4 files changed, 436 insertions(+) create mode 100644 builddir.go create mode 100644 docker_import.go create mode 100644 main.go create mode 100644 sha.go diff --git a/builddir.go b/builddir.go new file mode 100644 index 0000000..23b4fc3 --- /dev/null +++ b/builddir.go @@ -0,0 +1,256 @@ +package main + +import ( + "compress/gzip" + "crypto/sha256" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/containers/image/copy" + "github.com/containers/image/docker" + "github.com/containers/image/signature" + "github.com/containers/image/transports" + "github.com/containers/image/types" +) + +type BuildDir struct { + dir string + manifest manifestSchema2 + image image +} + +func NewBuildDir() (*BuildDir, error) { + t, err := ioutil.TempDir("", "tarpoon") + if err != nil { + return nil, err + } + d := &BuildDir{ + dir: t, + } + return d, nil +} + +func (t *BuildDir) LoadBase(base string) error { + if base == "scratch" { + return t.createEmptyBase() + } + pc, err := getPolicyContext() + if err != nil { + return err + } + defer pc.Destroy() + src, err := docker.ParseReference(base) + if err != nil { + return err + } + dest, err := transports.ParseImageName("dir:" + t.dir) + if err != nil { + return err + } + // TODO: sigs + err = copy.Image(systemContext(), pc, dest, src, ©.Options{ + RemoveSignatures: false, + SignBy: "", + ReportWriter: os.Stdout, + }) + if err != nil { + return err + } + manifest, err := os.Open(filepath.Join(t.dir, "manifest.json")) + if err != nil { + return err + } + defer manifest.Close() + dec := json.NewDecoder(manifest) + err = dec.Decode(&t.manifest) + if err != nil { + return err + } + configfile := t.manifest.ConfigDescriptor.Digest[7:] + ".tar" + image, err := os.Open(filepath.Join(t.dir, configfile)) + if err != nil { + return err + } + defer image.Close() + dec = json.NewDecoder(image) + err = dec.Decode(&t.image) + if err != nil { + return err + } + return nil +} + +func systemContext() *types.SystemContext { + tlsVerify := true + return &types.SystemContext{ + RegistriesDirPath: "", + DockerCertPath: "", + DockerInsecureSkipTLSVerify: !tlsVerify, + } +} + +func (t *BuildDir) createEmptyBase() error { + t.manifest = manifestSchema2{ + SchemaVersion: 2, + MediaType: "application/vnd.docker.distribution.manifest.v2+json", + } + t.image = image{ + v1Image: v1Image{ + Created: time.Now(), + ContainerConfig: &config{ + Cmd: []string{"/bin/sh", "-c"}, + Env: []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, + }, + Architecture: "amd64", + OS: "linux", + Author: "tarpoon", + }, + RootFS: &rootFS{ + Type: "layers", + }, + } + return nil +} + +func (t *BuildDir) DumpImageConfig() error { + path := filepath.Join(t.dir, "_config.json") + configf, err := os.Create(path) + if err != nil { + return err + } + enc := json.NewEncoder(configf) + err = enc.Encode(t.image) + if err != nil { + configf.Close() + return err + } + configf.Sync() + configf.Close() + sha := sha256File(path) + t.manifest.ConfigDescriptor = descriptor{ + Size: sha.Size(), + Digest: sha.String(), + MediaType: "application/vnd.docker.container.image.v1+json", + } + return os.Rename(path, filepath.Join(t.dir, sha.Name()+".tar")) +} + +func (t *BuildDir) WriteManifest() error { + path := filepath.Join(t.dir, "manifest.json") + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + enc.SetIndent("", "\t") + return enc.Encode(t.manifest) +} + +func (t *BuildDir) AddTarLayer(tarfile string) error { + at := time.Now() + sha, err := t.gzipLayer(tarfile) + if err != nil { + return err + } + gzpath := filepath.Join(t.dir, "_layer.gz") + gzsha := sha256File(gzpath) + t.image.RootFS.DiffIDs = append(t.image.RootFS.DiffIDs, sha.String()) + t.image.History = append(t.image.History, imageHistory{ + Created: at, + CreatedBy: "#(nop) TARPOON", + Comment: "Created by tarpoon", + }) + t.image.Created = at + t.manifest.LayersDescriptors = append(t.manifest.LayersDescriptors, descriptor{ + MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + Size: gzsha.Size(), + Digest: gzsha.String(), + }) + return os.Rename(gzpath, filepath.Join(t.dir, gzsha.Name()+".tar")) +} + +func (t *BuildDir) gzipLayer(tarfile string) (fileSha, error) { + targetpath := filepath.Join(t.dir, "_layer.gz") + tar, err := os.Open(tarfile) + if err != nil { + return fileSha{}, err + } + defer tar.Close() + tarsha := sha256.New() + tr := io.TeeReader(tar, tarsha) + gz, err := os.Create(targetpath) + if err != nil { + return fileSha{}, err + } + defer gz.Close() + gw := gzip.NewWriter(gz) + io.Copy(gw, tr) + gw.Flush() + gw.Close() + return fileSha{tarsha.Sum(nil), 0}, nil +} + +func (t *BuildDir) Push(dockerTarget string) error { + pc, err := getPolicyContext() + if err != nil { + return err + } + defer pc.Destroy() + src, err := transports.ParseImageName("dir:" + t.dir) + if err != nil { + return err + } + dest, err := docker.ParseReference(dockerTarget) + if err != nil { + return err + } + // TODO: sigs + err = copy.Image(systemContext(), pc, dest, src, ©.Options{ + RemoveSignatures: false, + SignBy: "", + ReportWriter: os.Stdout, + }) + return err +} + +func (t *BuildDir) Close() error { + return os.RemoveAll(t.dir) +} + +func getPolicyContext() (*signature.PolicyContext, error) { + policy, err := signature.DefaultPolicy(nil) + if err == nil { + return signature.NewPolicyContext(policy) + } + // Okay, so. I know DefaultPolicy explicitly says not to do this. + // But until /etc/containers/policy.json (a RedHat-ism) becomes either + // the standard or there's a better way, let's go ahead and use the default policy + // that skopeo installs anyway + policy, err = signature.NewPolicyFromBytes([]byte(rhDefaultPolicy)) + if err != nil { + return nil, err + } + return signature.NewPolicyContext(policy) +} + +const rhDefaultPolicy = ` +{ + "default": [ + { + "type": "insecureAcceptAnything" + } + ], + "transports": + { + "docker-daemon": + { + "": [{"type":"insecureAcceptAnything"}] + } + } +} +` diff --git a/docker_import.go b/docker_import.go new file mode 100644 index 0000000..a808212 --- /dev/null +++ b/docker_import.go @@ -0,0 +1,58 @@ +package main + +import "time" + +type manifestSchema2 struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + ConfigDescriptor descriptor `json:"config"` + LayersDescriptors []descriptor `json:"layers"` +} + +type descriptor struct { + MediaType string `json:"mediaType"` + Size int64 `json:"size"` + Digest string `json:"digest"` +} + +type config struct { + Cmd []string + Env []string + Labels map[string]string +} + +type v1Image struct { + ID string `json:"id,omitempty"` + Parent string `json:"parent,omitempty"` + Comment string `json:"comment,omitempty"` + Created time.Time `json:"created"` + ContainerConfig *config `json:"container_config,omitempty"` + DockerVersion string `json:"docker_version,omitempty"` + Author string `json:"author,omitempty"` + // Config is the configuration of the container received from the client + Config *config `json:"config,omitempty"` + // Architecture is the hardware that the image is build and runs on + Architecture string `json:"architecture,omitempty"` + // OS is the operating system used to build and run the image + OS string `json:"os,omitempty"` +} + +type image struct { + v1Image + History []imageHistory `json:"history,omitempty"` + RootFS *rootFS `json:"rootfs,omitempty"` +} + +type imageHistory struct { + Created time.Time `json:"created"` + Author string `json:"author,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Comment string `json:"comment,omitempty"` + EmptyLayer bool `json:"empty_layer,omitempty"` +} + +type rootFS struct { + Type string `json:"type"` + DiffIDs []string `json:"diff_ids,omitempty"` + BaseLayer string `json:"base_layer,omitempty"` +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..b96ecdd --- /dev/null +++ b/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "tarpoon", + Short: "Build Docker images", + Long: "`docker build && docker push` without the `docker`. Push a tarball to a registry for distribution", + Run: rootRun, +} + +var ( + baseImage string + workDir string + exec []string +) + +func init() { + rootCmd.PersistentFlags().StringVarP(&baseImage, "base-image", "b", "scratch", "Base image to append the tar to") + rootCmd.PersistentFlags().StringVarP(&workDir, "workdir", "w", "/", "Working directory of the new container") + rootCmd.PersistentFlags().StringSliceVarP(&exec, "exec", "e", []string{"/bin/sh", "-c"}, "What to exec in the newly built container") +} + +func main() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(-1) + } +} + +func rootRun(cmd *cobra.Command, args []string) { + if len(args) < 2 { + fmt.Println("tarpoon TARFILE DOCKER_PATH") + os.Exit(1) + } + fmt.Printf("** Creating build dir\n") + dir, err := NewBuildDir() + defer dir.Close() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Printf("** Build dir is %s\n", dir.dir) + fmt.Printf("** Loading base image %s\n", baseImage) + err = dir.LoadBase(baseImage) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Printf("** Applying tarball %s\n", args[0]) + err = dir.AddTarLayer(args[0]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Printf("** Preparing image\n") + err = dir.DumpImageConfig() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + err = dir.WriteManifest() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Printf("** Pushing image\n") + err = dir.Push(args[1]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + +} diff --git a/sha.go b/sha.go new file mode 100644 index 0000000..5a89d63 --- /dev/null +++ b/sha.go @@ -0,0 +1,43 @@ +package main + +import ( + "crypto/sha256" + "fmt" + "io" + "os" +) + +type fileSha struct { + bytes []byte + size int64 +} + +func sha256File(path string) fileSha { + h := sha256.New() + f, err := os.Open(path) + if err != nil { + panic(err) + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + panic(err) + } + io.Copy(h, f) + return fileSha{ + bytes: h.Sum(nil), + size: fi.Size(), + } +} + +func (f fileSha) String() string { + return fmt.Sprintf("sha256:%x", f.bytes) +} + +func (f fileSha) Name() string { + return fmt.Sprintf("%x", f.bytes) +} + +func (f fileSha) Size() int64 { + return f.size +}