Feature add docker/volume and volume/secret support (apache#58)

* update docs for using docker/volume and volume/secret * implement scheduler changes for docker/volume and volume/secret * implement python thrift changes for docker/volume and volume secret * update python thrift tests for docker/volume and volume/secret * update aurora client schema for docker/volume and secret/volume * create/update structs for docker/volume and volume/secret isolators
JustinVenus · Jun 21, 2019 · 20540a7 · 20540a7
1 parent 6ec953f
commit 20540a7
Show file tree

Hide file tree

Showing 7 changed files with 360 additions and 41 deletions.
diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift b/api/src/main/thrift/org/apache/aurora/gen/api.thrift
@@ -169,6 +169,48 @@ enum Mode {
   RO = 2
 }
 
+/** Describes a parameter passed to docker cli */
+struct DockerParameter {
+  /** a parameter to pass to docker. (e.g. volume) */
+  1: string name
+  /** the value to pass to a parameter (e.g. /src/webapp:/opt/webapp) */
+  2: string value
+}
+
+struct DockerVolume {
+  /** The driver name on the host facilitate mounting the resource. */
+  1: string driver
+  /** The resource name that will serve as the source for the mount. */
+  2: string name
+  /** The specific options to pass on to the driver. */
+  3: optional list<DockerParameter> options
+}
+
+struct FileSecret {
+  /** The name field is used to lookup the secret source */
+  1: string name
+  /** The key field can be used to reference a single value within a secret containing arbitrary key-value pairs. */
+  2: optional string key
+}
+
+/** The type of volume mount */
+enum VolumeType {
+  UNKNOWN = 0
+  /** Represent container volume type 'HOST_PATH' requires isolator 'filesystem/linux' */
+  HOST_PATH = 1
+  /** Represent container volume type 'DOCKER_VOLUME' requires isolator 'docker/volume' */
+  DOCKER_VOLUME = 2
+  /** Represent container volume type 'FILE_SECRET' requires isolator 'volume/secret' */
+  FILE_SECRET = 3
+}
+
+union VolumeSource {
+  1: string hostPath
+  2: DockerVolume docker
+  3: FileSecret fileSecret
+}
+
+
 /** A volume mount point within a container */
 struct Volume {
   /** The path inside the container where the mount will be created. */
@@ -177,6 +219,10 @@ struct Volume {
   2: string hostPath
   /** The access mode */
   3: Mode mode
+  /** The volume type - set optional for backwards compatibility */
+  4: optional VolumeType volumeType = VolumeType.UNKNOWN
+  /** The volume source - set optional for backwards compatibility */
+  5: optional VolumeSource source
 }
 
 /** Describes an image for use with the Mesos unified containerizer in the Docker format */
@@ -208,15 +254,6 @@ struct MesosContainer {
   /** the optional list of volumes to mount into the task. */
   2: optional list<Volume> volumes
 }
-
-/** Describes a parameter passed to docker cli */
-struct DockerParameter {
-  /** a parameter to pass to docker. (e.g. volume) */
-  1: string name
-  /** the value to pass to a parameter (e.g. /src/webapp:/opt/webapp) */
-  2: string value
-}
-
 /** Describes a docker container */
 struct DockerContainer {
   /** The container image to be run */

diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md
@@ -560,11 +560,40 @@ unified-container, the container can be omitted from your job config.
 
 ### Volume Object
 
-  param                  | type     | description
-  -----                  | :----:   | -----------
-  ```container_path```   | String   | Path on the host to mount.
-  ```host_path```        | String   | Mount point in the container.
-  ```mode```             | Enum     | Mode of the mount, can be 'RW' or 'RO'.
+  param                  | type                                       | description
+  -----                  | :----:                                     | -----------
+  ```container_path```   | String                                     | Path on the host to mount.
+  ```host_path```        | String                                     | Mount point in the container.
+  ```mode```             | Enum                                       | Mode of the mount, can be 'RW' or 'RO'.
+  ```source```           | Choice(DockerVolume, FileSecret, HostPath) | Type of volume to mount.
+
+### DockerVolume Object
+
+*Note: In order to use this feature, the mesos-agent must be configured to enable the `docker/volume` [isolator](http://mesos.apache.org/documentation/latest/isolators/docker-volume/).*
+
+  param         | type             | description
+  -----         | :----:           | -----------
+  ```driver```  | String           | A driver volume helper application such as flocker, convoy, rexray, and etc.
+  ```name```    | String           | A represententation of the volume that the driver can manage.
+  ```options``` | List(Parameter)  | An optional list of parameters to pass to `dvdcli` to influence the driver behavior.
+
+### FileSecret Object
+
+*Note: In order to use this feature, the mesos-agent must be configured to enable the `volume/secret` [isolator](http://mesos.apache.org/documentation/latest/secrets/).*
+*Note: In order to use this feature, the mesos-agent must be provided with a secret resolver [implementation](https://github.com/apache/mesos/blob/004fb5fa27c2992b11a2fa51a8ec5a3f3de404db/include/mesos/secret/resolver.hpp).*
+*Note: The interpretation of this objects `name` and `key` is entirely dependent on the behavior of the secret resolver implementation in use.*
+*Note: This object can only provide File-based Secrets.*
+
+  param         | type             | description
+  -----         | :----:           | -----------
+  ```name```    | String           | A name of a secret to resolve.
+  ```key```     | String           | An optional key typically used to reference a single value within a secret that contains arbitrary key-value pairs.
+
+### HostPath Object
+
+  param      | type   | description
+  -----      | :----: | -----------
+  ```path``` | String | Mount point in the container.
 
 ### AppcImage
 

diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
@@ -36,6 +36,8 @@
 import org.apache.aurora.gen.SlaPolicy;
 import org.apache.aurora.gen.TaskConfig;
 import org.apache.aurora.gen.TaskConstraint;
+import org.apache.aurora.gen.VolumeSource;
+import org.apache.aurora.gen.VolumeType;
 import org.apache.aurora.scheduler.TierManager;
 import org.apache.aurora.scheduler.base.JobKeys;
 import org.apache.aurora.scheduler.base.UserProvidedStrings;
@@ -476,6 +478,18 @@ public ITaskConfig validateAndPopulate(
       if (!settings.allowContainerVolumes && !container.getVolumes().isEmpty()) {
         throw new TaskDescriptionException(NO_CONTAINER_VOLUMES);
       }
+
+      if (settings.allowContainerVolumes  && !container.getVolumes().isEmpty()) {
+        builder.setContainer(Container.mesos(
+            container.newBuilder()
+                .setVolumes(container.getVolumes().stream()
+                    .map(v -> v.isSetVolumeType() ? v.newBuilder() : v.newBuilder()
+                        .setContainerPath(v.getContainerPath())
+                        .setMode(v.getMode())
+                        .setSource(VolumeSource.hostPath(v.getHostPath()))
+                        .setVolumeType(VolumeType.HOST_PATH))
+                    .collect(Collectors.toList()))));
+      }
     }
 
     validateSlaPolicy(builder, instanceCount);

diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
@@ -17,16 +17,17 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 import javax.inject.Inject;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
 import com.google.protobuf.ByteString;
 
 import org.apache.aurora.Protobufs;
 import org.apache.aurora.codec.ThriftBinaryCodec;
+import org.apache.aurora.gen.VolumeType;
 import org.apache.aurora.scheduler.base.JobKeys;
 import org.apache.aurora.scheduler.base.SchedulerException;
 import org.apache.aurora.scheduler.base.Tasks;
@@ -43,6 +44,7 @@
 import org.apache.aurora.scheduler.storage.entities.IMesosContainer;
 import org.apache.aurora.scheduler.storage.entities.IServerInfo;
 import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
+import org.apache.aurora.scheduler.storage.entities.IVolume;
 import org.apache.mesos.v1.Protos;
 import org.apache.mesos.v1.Protos.CommandInfo;
 import org.apache.mesos.v1.Protos.ContainerInfo;
@@ -170,7 +172,9 @@ public TaskInfo createFrom(IAssignedTask task, Offer offer, boolean revocable)
       if (LOG.isDebugEnabled()) {
         LOG.debug(
             "Setting task resources to {}",
-            Iterables.transform(resources, Protobufs::toString));
+            StreamSupport.stream(resources.spliterator(), false)
+                .map(Protobufs::toString)
+                .collect(Collectors.toList()));
       }
 
       TaskInfo.Builder taskBuilder = TaskInfo.newBuilder()
@@ -250,12 +254,9 @@ private Optional<ContainerInfo.Builder> configureTaskForImage(
         ContainerInfo.MesosInfo.Builder mesosContainerBuilder =
             ContainerInfo.MesosInfo.newBuilder();
 
-        Iterable<Protos.Volume> containerVolumes = Iterables.transform(mesosContainer.getVolumes(),
-            input -> Protos.Volume.newBuilder()
-            .setMode(Protos.Volume.Mode.valueOf(input.getMode().name()))
-            .setHostPath(input.getHostPath())
-            .setContainerPath(input.getContainerPath())
-            .build());
+        Iterable<Protos.Volume> containerVolumes = mesosContainer.getVolumes().stream()
+            .map(v -> getContainerVolume(v))
+            .collect(Collectors.toList());
 
         Protos.Volume volume = Protos.Volume.newBuilder()
             .setImage(imageBuilder)
@@ -274,13 +275,77 @@ private Optional<ContainerInfo.Builder> configureTaskForImage(
       return Optional.empty();
     }
 
+    private Protos.Volume getContainerVolume(IVolume volume) {
+      requireNonNull(volume);
+
+      if (volume.getHostPath() == null && volume.isSetVolumeType()) {
+        if (volume.getVolumeType() == VolumeType.HOST_PATH) {
+          return Protos.Volume.newBuilder()
+              .setMode(Protos.Volume.Mode.valueOf(volume.getMode().name()))
+              .setHostPath(volume.getSource().getHostPath())
+              .setContainerPath(volume.getContainerPath())
+              .build();
+        } else if (volume.getVolumeType() == VolumeType.DOCKER_VOLUME) {
+          Iterable<Protos.Parameter> options = volume.getSource().getDocker().getOptions().stream()
+              .map(item -> Protos.Parameter.newBuilder()
+                  .setKey(item.getName())
+                  .setValue(item.getValue()).build())
+              .collect(Collectors.toList());
+
+          return Protos.Volume.newBuilder()
+              .setMode(Protos.Volume.Mode.valueOf(volume.getMode().name()))
+              .setSource(Protos.Volume.Source.newBuilder()
+                  .setDockerVolume(Protos.Volume.Source.DockerVolume.newBuilder()
+                      .setDriver(volume.getSource().getDocker().getDriver())
+                      .setName(volume.getSource().getDocker().getName())
+                      .setDriverOptions(Protos.Parameters.newBuilder()
+                          .addAllParameter(options))
+                      .build())
+                  .setType(Protos.Volume.Source.Type.DOCKER_VOLUME)
+                  .build())
+              .setContainerPath(volume.getContainerPath())
+              .build();
+
+        } else if (volume.getVolumeType() == VolumeType.FILE_SECRET) {
+          Protos.Secret.Reference.Builder reference = Protos.Secret.Reference.newBuilder();
+          reference.setName(volume.getSource().getFileSecret().getName());
+          if (volume.getSource().getFileSecret().isSetKey()) {
+            reference.setKey(volume.getSource().getFileSecret().getKey());
+          }
+
+          return Protos.Volume.newBuilder()
+              .setMode(Protos.Volume.Mode.valueOf(volume.getMode().name()))
+              .setSource(Protos.Volume.Source.newBuilder()
+                  .setSecret(Protos.Secret.newBuilder()
+                      .setReference(reference.build())
+                      .setType(Protos.Secret.Type.REFERENCE)
+                      .build())
+                  .setType(Protos.Volume.Source.Type.SECRET)
+                  .build())
+              .setContainerPath(volume.getContainerPath())
+              .build();
+
+        }
+        throw new SchedulerException("Task had no supported volume set.");
+      }
+
+      return Protos.Volume.newBuilder()
+          .setMode(Protos.Volume.Mode.valueOf(volume.getMode().name()))
+          .setHostPath(volume.getHostPath())
+          .setContainerPath(volume.getContainerPath())
+          .build();
+    }
+
     private ContainerInfo getDockerContainerInfo(
         IDockerContainer config,
         Optional<String> executorName) {
 
-      Iterable<Protos.Parameter> parameters = Iterables.transform(config.getParameters(),
-          item -> Protos.Parameter.newBuilder().setKey(item.getName())
-            .setValue(item.getValue()).build());
+      Iterable<Protos.Parameter> parameters = config.getParameters().stream()
+          .map(item -> Protos.Parameter.newBuilder()
+              .setKey(item.getName())
+              .setValue(item.getValue())
+              .build())
+          .collect(Collectors.toList());
 
       ContainerInfo.DockerInfo.Builder dockerBuilder = ContainerInfo.DockerInfo.newBuilder()
           .setImage(config.getImage()).addAllParameters(parameters);
@@ -329,7 +394,9 @@ private ExecutorInfo.Builder configureTaskForExecutor(
       if (LOG.isDebugEnabled()) {
         LOG.debug(
             "Setting executor resources to {}",
-            Iterables.transform(executorResources, Protobufs::toString));
+            StreamSupport.stream(executorResources.spliterator(), false)
+                .map(Protobufs::toString)
+                .collect(Collectors.toList()));
       }
       builder.clearResources().addAllResources(executorResources);
       return builder;

diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py
@@ -22,6 +22,7 @@
 from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME
 
 
+
 # TODO(wickman) Bind {{mesos.instance}} to %shard_id%
 class MesosContext(Struct):
   # The instance id (i.e. replica id, shard id) in the context of a task
@@ -154,10 +155,28 @@ class DockerImage(Struct):
 
 Mode = Enum('RO', 'RW')
 
+
+class HostPath(Struct):
+  path = Required(String)
+
+
+class FileSecret(Struct):
+  name = Required(String)
+  key = String
+
+
+class DockerVolume(Struct):
+  driver = Required(String)
+  name = Required(String)
+  options = Default(List(Parameter), [])
+
+
 class Volume(Struct):
   container_path = Required(String)
-  host_path = Required(String)
+  host_path = String
   mode = Required(Mode)
+  source = Choice([FileSecret, DockerVolume, HostPath])
+
 
 class Mesos(Struct):
   image = Choice([AppcImage, DockerImage])