From a585fd5cb89f54a0807ba5518dd764668f832f15 Mon Sep 17 00:00:00 2001 From: Ryan Sundberg Date: Fri, 10 May 2024 23:01:48 -0700 Subject: [PATCH] TaskConfig: Add option for `file_limit` to set RLIMIT_NOFILE Depending on the workload, the default resource limit (ulimit) for the max number of file descriptors per process may need to be raised, such as for database servers. Add a `file_limit` option to the task config which will allow the limit to be set per task. Signed-off-by: Ryan Sundberg --- README.md | 1 + containerd/containerd.go | 27 +++++++++++++++++++++++++++ containerd/driver.go | 2 ++ 3 files changed, 30 insertions(+) diff --git a/README.md b/README.md index 594cbf9..c419cf1 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ To interact with `images` and `containers` directly, you can use [`nerdctl`](htt | **privileged** | bool | no | Run container in privileged mode. Your container will have all linux capabilities when running in privileged mode. | | **pids_limit** | int64 | no | An integer value that specifies the pid limit for the container. Defaults to unlimited. | | **pid_mode** | string | no | `host` or not set (default). Set to `host` to share the PID namespace with the host. | +| **file_limit** | int64 | no | An integer value that specifies the file descriptor ulimit for the container. Defaults to 1024 by containerd. | | **hostname** | string | no | The hostname to assign to the container. When launching more than one of a task (using `count`) with this option set, every container the task starts will have the same hostname. | | **host_dns** | bool | no | Default (`true`). By default, a container launched using `containerd-driver` will use host `/etc/resolv.conf`. This is similar to [`docker behavior`](https://docs.docker.com/config/containers/container-networking/#dns-services). However, if you don't want to use host DNS, you can turn off this flag by setting `host_dns=false`. | | **seccomp** | bool | no | Enable default seccomp profile. List of [`allowed syscalls`](https://github.com/containerd/containerd/blob/master/contrib/seccomp/seccomp_default.go#L51-L395). | diff --git a/containerd/containerd.go b/containerd/containerd.go index 07231ff..1520f3e 100644 --- a/containerd/containerd.go +++ b/containerd/containerd.go @@ -25,6 +25,7 @@ import ( etchosts "github.com/Roblox/nomad-driver-containerd/etchosts" "github.com/containerd/containerd" + "github.com/containerd/containerd/containers" "github.com/containerd/containerd/cio" "github.com/containerd/containerd/contrib/seccomp" "github.com/containerd/containerd/oci" @@ -93,6 +94,27 @@ func withResolver(creds CredentialsOpt) containerd.RemoteOpt { return containerd.WithResolver(resolver) } +func withFileLimit(maxOpenFiles uint64) oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, spec *oci.Spec) error { + newRlimits := []specs.POSIXRlimit{{ + Type: "RLIMIT_NOFILE", + Hard: maxOpenFiles, + Soft: maxOpenFiles, + }} + + // Copy existing rlimits excluding previous RLIMIT_NOFILE + for _, rlimit := range spec.Process.Rlimits { + if rlimit.Type != "RLIMIT_NOFILE" { + newRlimits = append(newRlimits, rlimit) + } + } + + spec.Process.Rlimits = newRlimits + + return nil + } +} + func (d *Driver) pullImage(imageName, imagePullTimeout string, auth *RegistryAuth) (containerd.Image, error) { pullTimeout, err := time.ParseDuration(imagePullTimeout) if err != nil { @@ -167,6 +189,11 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC } } + // Set the resource limit for open file descriptors + if config.FileLimit > 0 { + opts = append(opts, withFileLimit(uint64(config.FileLimit))) + } + // Size of /dev/shm if len(config.ShmSize) > 0 { shmBytes, err := units.RAMInBytes(config.ShmSize) diff --git a/containerd/driver.go b/containerd/driver.go index 14e5b12..2e4b654 100644 --- a/containerd/driver.go +++ b/containerd/driver.go @@ -106,6 +106,7 @@ var ( "privileged": hclspec.NewAttr("privileged", "bool", false), "pids_limit": hclspec.NewAttr("pids_limit", "number", false), "pid_mode": hclspec.NewAttr("pid_mode", "string", false), + "file_limit": hclspec.NewAttr("file_limit", "number", false), "hostname": hclspec.NewAttr("hostname", "string", false), "host_dns": hclspec.NewDefault( hclspec.NewAttr("host_dns", "bool", false), @@ -190,6 +191,7 @@ type TaskConfig struct { Privileged bool `codec:"privileged"` PidsLimit int64 `codec:"pids_limit"` PidMode string `codec:"pid_mode"` + FileLimit int64 `codec:"file_limit"` Hostname string `codec:"hostname"` HostDNS bool `codec:"host_dns"` ImagePullTimeout string `codec:"image_pull_timeout"`