Skip to content

Commit

Permalink
cleanup: Support matrix update and code cleanup
Browse files Browse the repository at this point in the history 
- Updating the support matrix to point to the
suppotred ceph release.
- Code cleanup of cephfs resize and metadata
operations, since the implementations for them is
available on all the supported ceph versions.

Signed-off-by: karthik-us <[email protected]>
  • Loading branch information
@karthik-us
karthik-us committed Nov 16, 2023
1 parent 63f4887 commit aa63d74
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 117 deletions.
66 changes: 33 additions & 33 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,39 +81,39 @@ for its support details.

| Plugin | Features | Feature Status | CSI Driver Version | CSI Spec Version | Ceph Cluster Version | Kubernetes Version |
| ------ | --------------------------------------------------------- | -------------- | ------------------ | ---------------- | -------------------- | ------------------ |
| RBD | Dynamically provision, de-provision Block mode RWO volume | GA | >= v1.0.0 | >= v1.0.0 | Octopus (>=15.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision Block mode RWX volume | GA | >= v1.0.0 | >= v1.0.0 | Octopus (>=15.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision Block mode RWOP volume| Alpha | >= v3.5.0 | >= v1.5.0 | Octopus (>=15.0.0) | >= v1.22.0 |
| | Dynamically provision, de-provision File mode RWO volume | GA | >= v1.0.0 | >= v1.0.0 | Octopus (>=15.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWOP volume | Alpha | >= v3.5.0 | >= v1.5.0 | Octopus (>=15.0.0) | >= v1.22.0 |
| | Provision File Mode ROX volume from snapshot | Alpha | >= v3.0.0 | >= v1.0.0 | Octopus (>=v15.0.0) | >= v1.17.0 |
| | Provision File Mode ROX volume from another volume | Alpha | >= v3.0.0 | >= v1.0.0 | Octopus (>=v15.0.0) | >= v1.16.0 |
| | Provision Block Mode ROX volume from snapshot | Alpha | >= v3.0.0 | >= v1.0.0 | Octopus (>=v15.0.0) | >= v1.17.0 |
| | Provision Block Mode ROX volume from another volume | Alpha | >= v3.0.0 | >= v1.0.0 | Octopus (>=v15.0.0) | >= v1.16.0 |
| | Creating and deleting snapshot | GA | >= v1.0.0 | >= v1.0.0 | Octopus (>=15.0.0) | >= v1.17.0 |
| | Provision volume from snapshot | GA | >= v1.0.0 | >= v1.0.0 | Octopus (>=15.0.0) | >= v1.17.0 |
| | Provision volume from another volume | GA | >= v1.0.0 | >= v1.0.0 | Octopus (>=15.0.0) | >= v1.16.0 |
| | Expand volume | Beta | >= v2.0.0 | >= v1.1.0 | Octopus (>=15.0.0) | >= v1.15.0 |
| | Volume/PV Metrics of File Mode Volume | GA | >= v1.2.0 | >= v1.1.0 | Octopus (>=15.0.0) | >= v1.15.0 |
| | Volume/PV Metrics of Block Mode Volume | GA | >= v1.2.0 | >= v1.1.0 | Octopus (>=15.0.0) | >= v1.21.0 |
| | Topology Aware Provisioning Support | Alpha | >= v2.1.0 | >= v1.1.0 | Octopus (>=15.0.0) | >= v1.14.0 |
| CephFS | Dynamically provision, de-provision File mode RWO volume | GA | >= v1.1.0 | >= v1.0.0 | Octopus (>=15.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWX volume | GA | >= v1.1.0 | >= v1.0.0 | Octopus (>=v15.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode ROX volume | Alpha | >= v3.0.0 | >= v1.0.0 | Octopus (>=v15.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWOP volume | Alpha | >= v3.5.0 | >= v1.5.0 | Octopus (>=15.0.0) | >= v1.22.0 |
| | Creating and deleting snapshot | GA | >= v3.1.0 | >= v1.0.0 | Octopus (>=v15.2.4) | >= v1.17.0 |
| | Provision volume from snapshot | GA | >= v3.1.0 | >= v1.0.0 | Octopus (>=v15.2.4) | >= v1.17.0 |
| | Provision volume from another volume | GA | >= v3.1.0 | >= v1.0.0 | Octopus (>=v15.2.4) | >= v1.16.0 |
| | Expand volume | Beta | >= v2.0.0 | >= v1.1.0 | Octopus (>=v15.0.0) | >= v1.15.0 |
| | Volume/PV Metrics of File Mode Volume | GA | >= v1.2.0 | >= v1.1.0 | Octopus (>=v15.0.0) | >= v1.15.0 |
| NFS | Dynamically provision, de-provision File mode RWO volume | Alpha | >= v3.6.0 | >= v1.0.0 | Pacific (>=16.2.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWX volume | Alpha | >= v3.6.0 | >= v1.0.0 | Pacific (>=16.2.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode ROX volume | Alpha | >= v3.6.0 | >= v1.0.0 | Pacific (>=16.2.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWOP volume | Alpha | >= v3.6.0 | >= v1.5.0 | Pacific (>=16.2.0) | >= v1.22.0 |
| | Expand volume | Alpha | >= v3.7.0 | >= v1.1.0 | Pacific (>=16.2.0) | >= v1.15.0 |
| | Creating and deleting snapshot | Alpha | >= v3.7.0 | >= v1.1.0 | Pacific (>=16.2.0) | >= v1.17.0 |
| | Provision volume from snapshot | Alpha | >= v3.7.0 | >= v1.1.0 | Pacific (>=16.2.0) | >= v1.17.0 |
| | Provision volume from another volume | Alpha | >= v3.7.0 | >= v1.1.0 | Pacific (>=16.2.0) | >= v1.16.0 |
| RBD | Dynamically provision, de-provision Block mode RWO volume | GA | >= v1.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision Block mode RWX volume | GA | >= v1.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision Block mode RWOP volume| Alpha | >= v3.5.0 | >= v1.5.0 | Pacific (>=v16.0.0) | >= v1.22.0 |
| | Dynamically provision, de-provision File mode RWO volume | GA | >= v1.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWOP volume | Alpha | >= v3.5.0 | >= v1.5.0 | Pacific (>=v16.0.0) | >= v1.22.0 |
| | Provision File Mode ROX volume from snapshot | Alpha | >= v3.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.17.0 |
| | Provision File Mode ROX volume from another volume | Alpha | >= v3.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.16.0 |
| | Provision Block Mode ROX volume from snapshot | Alpha | >= v3.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.17.0 |
| | Provision Block Mode ROX volume from another volume | Alpha | >= v3.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.16.0 |
| | Creating and deleting snapshot | GA | >= v1.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.17.0 |
| | Provision volume from snapshot | GA | >= v1.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.17.0 |
| | Provision volume from another volume | GA | >= v1.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.16.0 |
| | Expand volume | Beta | >= v2.0.0 | >= v1.1.0 | Pacific (>=v16.0.0) | >= v1.15.0 |
| | Volume/PV Metrics of File Mode Volume | GA | >= v1.2.0 | >= v1.1.0 | Pacific (>=v16.0.0) | >= v1.15.0 |
| | Volume/PV Metrics of Block Mode Volume | GA | >= v1.2.0 | >= v1.1.0 | Pacific (>=v16.0.0) | >= v1.21.0 |
| | Topology Aware Provisioning Support | Alpha | >= v2.1.0 | >= v1.1.0 | Pacific (>=v16.0.0) | >= v1.14.0 |
| CephFS | Dynamically provision, de-provision File mode RWO volume | GA | >= v1.1.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWX volume | GA | >= v1.1.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode ROX volume | Alpha | >= v3.0.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWOP volume | Alpha | >= v3.5.0 | >= v1.5.0 | Pacific (>=v16.0.0) | >= v1.22.0 |
| | Creating and deleting snapshot | GA | >= v3.1.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.17.0 |
| | Provision volume from snapshot | GA | >= v3.1.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.17.0 |
| | Provision volume from another volume | GA | >= v3.1.0 | >= v1.0.0 | Pacific (>=v16.0.0) | >= v1.16.0 |
| | Expand volume | Beta | >= v2.0.0 | >= v1.1.0 | Pacific (>=v16.0.0) | >= v1.15.0 |
| | Volume/PV Metrics of File Mode Volume | GA | >= v1.2.0 | >= v1.1.0 | Pacific (>=v16.0.0) | >= v1.15.0 |
| NFS | Dynamically provision, de-provision File mode RWO volume | Alpha | >= v3.6.0 | >= v1.0.0 | Pacific (>=v16.2.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWX volume | Alpha | >= v3.6.0 | >= v1.0.0 | Pacific (>=v16.2.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode ROX volume | Alpha | >= v3.6.0 | >= v1.0.0 | Pacific (>=v16.2.0) | >= v1.14.0 |
| | Dynamically provision, de-provision File mode RWOP volume | Alpha | >= v3.6.0 | >= v1.5.0 | Pacific (>=v16.2.0) | >= v1.22.0 |
| | Expand volume | Alpha | >= v3.7.0 | >= v1.1.0 | Pacific (>=v16.2.0) | >= v1.15.0 |
| | Creating and deleting snapshot | Alpha | >= v3.7.0 | >= v1.1.0 | Pacific (>=v16.2.0) | >= v1.17.0 |
| | Provision volume from snapshot | Alpha | >= v3.7.0 | >= v1.1.0 | Pacific (>=v16.2.0) | >= v1.17.0 |
| | Provision volume from another volume | Alpha | >= v3.7.0 | >= v1.1.0 | Pacific (>=v16.2.0) | >= v1.16.0 |

`NOTE`: The `Alpha` status reflects possible non-backward
compatible changes in the future, and is thus not recommended
Expand Down
53 changes: 0 additions & 53 deletions internal/cephfs/core/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,55 +17,24 @@ limitations under the License.
package core

import (
"errors"
"fmt"
"strings"

fsAdmin "github.com/ceph/go-ceph/cephfs/admin"
)

const (
// clusterNameKey cluster Key, set on cephfs subvolume.
clusterNameKey = "csi.ceph.com/cluster/name"
)

// ErrSubVolMetadataNotSupported is returned when set/get/list/remove subvolume metadata options are not supported.
var ErrSubVolMetadataNotSupported = errors.New("subvolume metadata operations are not supported")

func (s *subVolumeClient) supportsSubVolMetadata() bool {
newLocalClusterState(s.clusterID)

return clusterAdditionalInfo[s.clusterID].subVolMetadataState != unsupported
}

func (s *subVolumeClient) isUnsupportedSubVolMetadata(err error) bool {
var invalid fsAdmin.NotImplementedError
if err != nil && errors.As(err, &invalid) {
// In case the error is other than invalid command return error to the caller.
clusterAdditionalInfo[s.clusterID].subVolMetadataState = unsupported

return false
}
clusterAdditionalInfo[s.clusterID].subVolMetadataState = supported

return true
}

// setMetadata sets custom metadata on the subvolume in a volume as a
// key-value pair.
func (s *subVolumeClient) setMetadata(key, value string) error {
var err error
if !s.supportsSubVolMetadata() {
return ErrSubVolMetadataNotSupported
}
fsa, err := s.conn.GetFSAdmin()
if err != nil {
return err
}
err = fsa.SetMetadata(s.FsName, s.SubvolumeGroup, s.VolID, key, value)
if !s.isUnsupportedSubVolMetadata(err) {
return ErrSubVolMetadataNotSupported
}

return err
}
Expand All @@ -74,17 +43,11 @@ func (s *subVolumeClient) setMetadata(key, value string) error {
// using the metadata key.
func (s *subVolumeClient) removeMetadata(key string) error {
var err error
if !s.supportsSubVolMetadata() {
return ErrSubVolMetadataNotSupported
}
fsa, err := s.conn.GetFSAdmin()
if err != nil {
return err
}
err = fsa.RemoveMetadata(s.FsName, s.SubvolumeGroup, s.VolID, key)
if !s.isUnsupportedSubVolMetadata(err) {
return ErrSubVolMetadataNotSupported
}

return err
}
Expand All @@ -97,21 +60,13 @@ func (s *subVolumeClient) SetAllMetadata(parameters map[string]string) error {

for k, v := range parameters {
err := s.setMetadata(k, v)
// If setMetadata is not supported return nil
if errors.Is(err, ErrSubVolMetadataNotSupported) {
return nil
}
if err != nil {
return fmt.Errorf("failed to set metadata key %q, value %q on subvolume %v: %w", k, v, s, err)
}
}

if s.clusterName != "" {
err := s.setMetadata(clusterNameKey, s.clusterName)
// If setMetadata is not supported return nil
if errors.Is(err, ErrSubVolMetadataNotSupported) {
return nil
}
if err != nil {
return fmt.Errorf("failed to set metadata key %q, value %q on subvolume %v: %w",
clusterNameKey, s.clusterName, s, err)
Expand All @@ -129,21 +84,13 @@ func (s *subVolumeClient) UnsetAllMetadata(keys []string) error {

for _, key := range keys {
err := s.removeMetadata(key)
// If setMetadata is not supported return nil
if errors.Is(err, ErrSubVolMetadataNotSupported) {
return nil
}
// TODO: replace string comparison with errno.
if err != nil && !strings.Contains(err.Error(), "No such file or directory") {
return fmt.Errorf("failed to unset metadata key %q on subvolume %v: %w", key, s, err)
}
}

err := s.removeMetadata(clusterNameKey)
// If setMetadata is not supported return nil
if errors.Is(err, ErrSubVolMetadataNotSupported) {
return nil
}
// TODO: replace string comparison with errno.
if err != nil && !strings.Contains(err.Error(), "No such file or directory") {
return fmt.Errorf("failed to unset metadata key %q on subvolume %v: %w", clusterNameKey, s, err)
Expand Down
42 changes: 11 additions & 31 deletions internal/cephfs/core/volume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,16 +199,13 @@ func (s *subVolumeClient) GetSubVolumeInfo(ctx context.Context) (*Subvolume, err
type operationState int64

const (
unknown operationState = iota
supported
supported operationState = iota
unsupported
)

type localClusterState struct {
// set the enum value i.e., unknown, supported,
// unsupported as per the state of the cluster.
resizeState operationState
subVolMetadataState operationState
subVolSnapshotMetadataState operationState
}

Expand Down Expand Up @@ -268,39 +265,22 @@ func (s *subVolumeClient) ExpandVolume(ctx context.Context, bytesQuota int64) er
return err
}

// ResizeVolume will try to use ceph fs subvolume resize command to resize the
// subvolume. If the command is not available as a fallback it will use
// CreateVolume to resize the subvolume.
// ResizeVolume will use ceph fs subvolume resize command to resize the subvolume.
func (s *subVolumeClient) ResizeVolume(ctx context.Context, bytesQuota int64) error {
newLocalClusterState(s.clusterID)
// resize subvolume when either it's supported, or when corresponding
// clusterID key was not present.
if clusterAdditionalInfo[s.clusterID].resizeState == unknown ||
clusterAdditionalInfo[s.clusterID].resizeState == supported {
fsa, err := s.conn.GetFSAdmin()
if err != nil {
log.ErrorLog(ctx, "could not get FSAdmin, can not resize volume %s:", s.FsName, err)

return err
}
_, err = fsa.ResizeSubVolume(s.FsName, s.SubvolumeGroup, s.VolID, fsAdmin.ByteCount(bytesQuota), true)
if err == nil {
clusterAdditionalInfo[s.clusterID].resizeState = supported
fsa, err := s.conn.GetFSAdmin()
if err != nil {
log.ErrorLog(ctx, "could not get FSAdmin, can not resize volume %s:", s.FsName, err)

return nil
}
var invalid fsAdmin.NotImplementedError
// In case the error is other than invalid command return error to the caller.
if !errors.As(err, &invalid) {
log.ErrorLog(ctx, "failed to resize subvolume %s in fs %s: %s", s.VolID, s.FsName, err)
return err
}

return err
}
_, err = fsa.ResizeSubVolume(s.FsName, s.SubvolumeGroup, s.VolID, fsAdmin.ByteCount(bytesQuota), true)
if err != nil {
log.ErrorLog(ctx, "failed to resize subvolume %s in fs %s: %s", s.VolID, s.FsName, err)
}
clusterAdditionalInfo[s.clusterID].resizeState = unsupported
s.Size = bytesQuota

return s.CreateVolume(ctx)
return err
}

// PurgSubVolume removes the subvolume.
Expand Down

0 comments on commit aa63d74

Please sign in to comment.