From 311ceb0bd189a96433f0612134071080337823f7 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 18 Jun 2024 10:58:26 +0200 Subject: [PATCH 1/4] journal: add option to store the groupID we need to have groupID stored and retrived when we are doing group level operations, we need to find out the groupID from the volumeID Signed-off-by: Madhu Rajanna --- internal/journal/voljournal.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/internal/journal/voljournal.go b/internal/journal/voljournal.go index b8aec1cd893..096375a68ac 100644 --- a/internal/journal/voljournal.go +++ b/internal/journal/voljournal.go @@ -131,6 +131,10 @@ type Config struct { // of this Ceph volume csiImageIDKey string + // CSI GroupName is per Ceph volume object omap, contains the group ID of + // this Ceph volume + csiGroupIDKey string + // CSI image-name key in per Ceph volume object map, containing RBD image-name // of this Ceph volume csiImageKey string @@ -174,6 +178,7 @@ func NewCSIVolumeJournal(suffix string) *Config { cephSnapSourceKey: "", namespace: "", csiImageIDKey: "csi.imageid", + csiGroupIDKey: "csi.groupid", encryptKMSKey: "csi.volume.encryptKMS", encryptionType: "csi.volume.encryptionType", ownerKey: "csi.volume.owner", @@ -686,6 +691,7 @@ type ImageAttributes struct { EncryptionType util.EncryptionType // Type of encryption used, if image encrypted Owner string // Contains the owner to be used in combination with KmsID (for some KMS) ImageID string // Contains the image id + GroupID string // Contains the group id of the image JournalPoolID int64 // Pool ID of the CSI journal pool, stored in big endian format (on-disk data) BackingSnapshotID string // ID of the snapshot on which the CephFS snapshot-backed volume is based } @@ -718,6 +724,7 @@ func (conn *Connection) GetImageAttributes( cj.csiImageIDKey, cj.ownerKey, cj.backingSnapshotIDKey, + cj.csiGroupIDKey, } values, err := getOMapValues( ctx, conn, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+objectUUID, @@ -736,6 +743,7 @@ func (conn *Connection) GetImageAttributes( imageAttributes.Owner = values[cj.ownerKey] imageAttributes.ImageID = values[cj.csiImageIDKey] imageAttributes.BackingSnapshotID = values[cj.backingSnapshotIDKey] + imageAttributes.GroupID = values[cj.csiGroupIDKey] // image key was added at a later point, so not all volumes will have this // key set when ceph-csi was upgraded @@ -795,6 +803,16 @@ func (conn *Connection) StoreAttribute(ctx context.Context, pool, reservedUUID, return nil } +// StoreGroupID stores an groupID in omap. +func (conn *Connection) StoreGroupID(ctx context.Context, pool, reservedUUID, groupID string) error { + err := conn.StoreAttribute(ctx, pool, reservedUUID, conn.config.csiGroupIDKey, groupID) + if err != nil { + return fmt.Errorf("failed to store groupID %w", err) + } + + return nil +} + // FetchAttribute fetches an attribute (key) in omap. func (conn *Connection) FetchAttribute(ctx context.Context, pool, reservedUUID, attribute string) (string, error) { key := conn.config.commonPrefix + attribute From 8774024e030b1308227974f8e81964b5d35697f7 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 18 Jun 2024 11:11:33 +0200 Subject: [PATCH 2/4] journal: remove snapshot specific name from group Adjusted method names to not have any specific things to volumesnapshot as we want to reuse the same journal for volumegroup as well. Signed-off-by: Madhu Rajanna --- internal/cephfs/groupcontrollerserver.go | 4 +-- internal/cephfs/store/volumegroup.go | 4 +-- internal/journal/volumegroupjournal.go | 38 +++++++++++++----------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/internal/cephfs/groupcontrollerserver.go b/internal/cephfs/groupcontrollerserver.go index 321ec5752e0..1d75709b8ec 100644 --- a/internal/cephfs/groupcontrollerserver.go +++ b/internal/cephfs/groupcontrollerserver.go @@ -461,7 +461,7 @@ func (cs *ControllerServer) createSnapshotAndAddMapping( } defer j.Destroy() // Add the snapshot to the volume group journal - err = j.AddVolumeSnapshotMapping(ctx, + err = j.AddVolumeMapping(ctx, vgo.MetadataPool, vgs.ReservedID, req.GetSourceVolumeId(), @@ -640,7 +640,7 @@ func (cs *ControllerServer) deleteSnapshotsAndUndoReservation(ctx context.Contex return err } // remove the entry from the omap - err = j.RemoveVolumeSnapshotMapping( + err = j.RemoveVolumeMapping( ctx, vgo.MetadataPool, vgsi.ReservedID, diff --git a/internal/cephfs/store/volumegroup.go b/internal/cephfs/store/volumegroup.go index 4286ad76fac..3d302ffc5de 100644 --- a/internal/cephfs/store/volumegroup.go +++ b/internal/cephfs/store/volumegroup.go @@ -169,7 +169,7 @@ func NewVolumeGroupOptionsFromID( vgs.RequestName = groupAttributes.RequestName vgs.FsVolumeGroupSnapshotName = groupAttributes.GroupName vgs.VolumeGroupSnapshotID = volumeGroupSnapshotID - vgs.VolumeSnapshotMap = groupAttributes.VolumeSnapshotMap + vgs.VolumeSnapshotMap = groupAttributes.VolumeMap return volOptions, &vgs, nil } @@ -208,7 +208,7 @@ func CheckVolumeGroupSnapExists( vgs.RequestName = volOptions.RequestName vgs.ReservedID = volGroupData.GroupUUID vgs.FsVolumeGroupSnapshotName = volGroupData.GroupName - vgs.VolumeSnapshotMap = volGroupData.VolumeGroupAttributes.VolumeSnapshotMap + vgs.VolumeSnapshotMap = volGroupData.VolumeGroupAttributes.VolumeMap // found a snapshot already available, process and return it! vgs.VolumeGroupSnapshotID, err = util.GenerateVolID(ctx, volOptions.Monitors, cr, volOptions.FscID, diff --git a/internal/journal/volumegroupjournal.go b/internal/journal/volumegroupjournal.go index 75f06a3554c..5f8f2b3b48f 100644 --- a/internal/journal/volumegroupjournal.go +++ b/internal/journal/volumegroupjournal.go @@ -41,7 +41,7 @@ type VolumeGroupJournal interface { UndoReservation( ctx context.Context, csiJournalPool, - snapshotGroupName, + groupName, reqName string) error // GetGroupAttributes fetches all keys and their values, from a UUID directory, // returning VolumeGroupAttributes structure. @@ -55,15 +55,17 @@ type VolumeGroupJournal interface { journalPoolID int64, reqName, namePrefix string) (string, string, error) - // AddVolumeSnapshotMapping adds a volumeID and snapshotID mapping to the UUID directory. - AddVolumeSnapshotMapping( + // AddVolumeMapping adds a volumeID and value mapping to the UUID + // directory. value can be anything which needs mapping, in case of + // volumegroupsnapshot its a snapshotID and its empty in case of volumegroup. + AddVolumeMapping( ctx context.Context, pool, reservedUUID, volumeID, - snapshotID string) error - // RemoveVolumeSnapshotMapping removes a volumeID and snapshotID mapping from the UUID directory. - RemoveVolumeSnapshotMapping( + value string) error + // RemoveVolumeMapping removes a volumeID mapping from the UUID directory. + RemoveVolumeMapping( ctx context.Context, pool, reservedUUID, @@ -222,7 +224,7 @@ func (vgjc *VolumeGroupJournalConnection) CheckReservation(ctx context.Context, volGroupData.GroupName = savedVolumeGroupAttributes.GroupName volGroupData.VolumeGroupAttributes = &VolumeGroupAttributes{} volGroupData.VolumeGroupAttributes.RequestName = savedVolumeGroupAttributes.RequestName - volGroupData.VolumeGroupAttributes.VolumeSnapshotMap = savedVolumeGroupAttributes.VolumeSnapshotMap + volGroupData.VolumeGroupAttributes.VolumeMap = savedVolumeGroupAttributes.VolumeMap return volGroupData, nil } @@ -361,9 +363,9 @@ func (vgjc *VolumeGroupJournalConnection) ReserveName(ctx context.Context, // VolumeGroupAttributes contains the request name and the volumeID's and // the corresponding snapshotID's. type VolumeGroupAttributes struct { - RequestName string // Contains the request name for the passed in UUID - GroupName string // Contains the group name - VolumeSnapshotMap map[string]string // Contains the volumeID and the corresponding snapshotID mapping + RequestName string // Contains the request name for the passed in UUID + GroupName string // Contains the group name + VolumeMap map[string]string // Contains the volumeID and the corresponding value mapping } func (vgjc *VolumeGroupJournalConnection) GetVolumeGroupAttributes( @@ -393,25 +395,25 @@ func (vgjc *VolumeGroupJournalConnection) GetVolumeGroupAttributes( // looking for volumeID/snapshotID mapping delete(values, cj.csiNameKey) delete(values, cj.csiImageKey) - groupAttributes.VolumeSnapshotMap = map[string]string{} + groupAttributes.VolumeMap = map[string]string{} for k, v := range values { - groupAttributes.VolumeSnapshotMap[k] = v + groupAttributes.VolumeMap[k] = v } return groupAttributes, nil } -func (vgjc *VolumeGroupJournalConnection) AddVolumeSnapshotMapping( +func (vgjc *VolumeGroupJournalConnection) AddVolumeMapping( ctx context.Context, pool, reservedUUID, volumeID, - snapshotID string, + value string, ) error { err := setOMapKeys(ctx, vgjc.connection, pool, vgjc.config.namespace, vgjc.config.cephUUIDDirectoryPrefix+reservedUUID, - map[string]string{volumeID: snapshotID}) + map[string]string{volumeID: value}) if err != nil { - log.ErrorLog(ctx, "failed adding volume snapshot mapping: %v", err) + log.ErrorLog(ctx, "failed adding volume mapping: %v", err) return err } @@ -419,7 +421,7 @@ func (vgjc *VolumeGroupJournalConnection) AddVolumeSnapshotMapping( return nil } -func (vgjc *VolumeGroupJournalConnection) RemoveVolumeSnapshotMapping( +func (vgjc *VolumeGroupJournalConnection) RemoveVolumeMapping( ctx context.Context, pool, reservedUUID, @@ -429,7 +431,7 @@ func (vgjc *VolumeGroupJournalConnection) RemoveVolumeSnapshotMapping( vgjc.config.cephUUIDDirectoryPrefix+reservedUUID, []string{volumeID}) if err != nil { - log.ErrorLog(ctx, "failed removing volume snapshot mapping: %v", err) + log.ErrorLog(ctx, "failed removing volume mapping from group: key %q: %v", volumeID, err) return err } From f60e70250bb856911d4e0bf193ee78bd28494805 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 18 Jun 2024 11:14:37 +0200 Subject: [PATCH 3/4] journal: support removing multiple volumeID Updating the code to support removing multiple volumeID's mapping from the group journal. Signed-off-by: Madhu Rajanna --- internal/cephfs/groupcontrollerserver.go | 2 +- internal/journal/volumegroupjournal.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/cephfs/groupcontrollerserver.go b/internal/cephfs/groupcontrollerserver.go index 1d75709b8ec..40421517c01 100644 --- a/internal/cephfs/groupcontrollerserver.go +++ b/internal/cephfs/groupcontrollerserver.go @@ -644,7 +644,7 @@ func (cs *ControllerServer) deleteSnapshotsAndUndoReservation(ctx context.Contex ctx, vgo.MetadataPool, vgsi.ReservedID, - volID) + []string{volID}) j.Destroy() if err != nil { log.ErrorLog(ctx, "failed to remove volume snapshot mapping: %v", err) diff --git a/internal/journal/volumegroupjournal.go b/internal/journal/volumegroupjournal.go index 5f8f2b3b48f..4663731aa93 100644 --- a/internal/journal/volumegroupjournal.go +++ b/internal/journal/volumegroupjournal.go @@ -64,12 +64,12 @@ type VolumeGroupJournal interface { reservedUUID, volumeID, value string) error - // RemoveVolumeMapping removes a volumeID mapping from the UUID directory. + // RemoveVolumeMapping removes volumeIDs mapping from the UUID directory. RemoveVolumeMapping( ctx context.Context, pool, - reservedUUID, - volumeID string) error + reservedUUID string, + volumeIDs []string) error } // VolumeGroupJournalConfig contains the configuration. @@ -424,14 +424,14 @@ func (vgjc *VolumeGroupJournalConnection) AddVolumeMapping( func (vgjc *VolumeGroupJournalConnection) RemoveVolumeMapping( ctx context.Context, pool, - reservedUUID, - volumeID string, + reservedUUID string, + volumeIDs []string, ) error { err := removeMapKeys(ctx, vgjc.connection, pool, vgjc.config.namespace, vgjc.config.cephUUIDDirectoryPrefix+reservedUUID, - []string{volumeID}) + volumeIDs) if err != nil { - log.ErrorLog(ctx, "failed removing volume mapping from group: key %q: %v", volumeID, err) + log.ErrorLog(ctx, "failed removing volume mapping from group: key: %q %v", volumeIDs, err) return err } From 4cd728dd65275dedf7096a64134063e47bf6af4a Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 18 Jun 2024 11:19:11 +0200 Subject: [PATCH 4/4] journal: add volumeMap to the group instead of adding single volumes to the group journal, support adding multiple volumeID's map to the group journal which is required for RBD as well. Signed-off-by: Madhu Rajanna --- internal/cephfs/groupcontrollerserver.go | 10 ++++---- internal/journal/volumegroupjournal.go | 30 ++++++++++++------------ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/internal/cephfs/groupcontrollerserver.go b/internal/cephfs/groupcontrollerserver.go index 40421517c01..b5dfeaabb84 100644 --- a/internal/cephfs/groupcontrollerserver.go +++ b/internal/cephfs/groupcontrollerserver.go @@ -461,11 +461,13 @@ func (cs *ControllerServer) createSnapshotAndAddMapping( } defer j.Destroy() // Add the snapshot to the volume group journal - err = j.AddVolumeMapping(ctx, + err = j.AddVolumesMapping(ctx, vgo.MetadataPool, vgs.ReservedID, - req.GetSourceVolumeId(), - resp.GetSnapshot().GetSnapshotId()) + map[string]string{ + req.GetSourceVolumeId(): resp.GetSnapshot().GetSnapshotId(), + }, + ) if err != nil { log.ErrorLog(ctx, "failed to add volume snapshot mapping: %v", err) // Delete the last created snapshot as its still not added to the @@ -640,7 +642,7 @@ func (cs *ControllerServer) deleteSnapshotsAndUndoReservation(ctx context.Contex return err } // remove the entry from the omap - err = j.RemoveVolumeMapping( + err = j.RemoveVolumesMapping( ctx, vgo.MetadataPool, vgsi.ReservedID, diff --git a/internal/journal/volumegroupjournal.go b/internal/journal/volumegroupjournal.go index 4663731aa93..990c38d85d7 100644 --- a/internal/journal/volumegroupjournal.go +++ b/internal/journal/volumegroupjournal.go @@ -55,17 +55,18 @@ type VolumeGroupJournal interface { journalPoolID int64, reqName, namePrefix string) (string, string, error) - // AddVolumeMapping adds a volumeID and value mapping to the UUID + // AddVolumesMapping adds a volumeMap map which contains volumeID's and its + // corresponding values mapping which need to be added to the UUID // directory. value can be anything which needs mapping, in case of - // volumegroupsnapshot its a snapshotID and its empty in case of volumegroup. - AddVolumeMapping( + // volumegroupsnapshot its a snapshotID and its empty in case of + // volumegroup. + AddVolumesMapping( ctx context.Context, pool, - reservedUUID, - volumeID, - value string) error - // RemoveVolumeMapping removes volumeIDs mapping from the UUID directory. - RemoveVolumeMapping( + reservedUUID string, + volumeMap map[string]string) error + // RemoveVolumesMapping removes volumeIDs mapping from the UUID directory. + RemoveVolumesMapping( ctx context.Context, pool, reservedUUID string, @@ -403,17 +404,16 @@ func (vgjc *VolumeGroupJournalConnection) GetVolumeGroupAttributes( return groupAttributes, nil } -func (vgjc *VolumeGroupJournalConnection) AddVolumeMapping( +func (vgjc *VolumeGroupJournalConnection) AddVolumesMapping( ctx context.Context, pool, - reservedUUID, - volumeID, - value string, + reservedUUID string, + volumeMap map[string]string, ) error { err := setOMapKeys(ctx, vgjc.connection, pool, vgjc.config.namespace, vgjc.config.cephUUIDDirectoryPrefix+reservedUUID, - map[string]string{volumeID: value}) + volumeMap) if err != nil { - log.ErrorLog(ctx, "failed adding volume mapping: %v", err) + log.ErrorLog(ctx, "failed to add volumeMap %v: %w ", volumeMap, err) return err } @@ -421,7 +421,7 @@ func (vgjc *VolumeGroupJournalConnection) AddVolumeMapping( return nil } -func (vgjc *VolumeGroupJournalConnection) RemoveVolumeMapping( +func (vgjc *VolumeGroupJournalConnection) RemoveVolumesMapping( ctx context.Context, pool, reservedUUID string,