rbd: VolumeGroupSnapshot support

Makefile

@@ -174,7 +174,7 @@ e2e.test: check-env
 
 .PHONY: rbd-group-snapshot
 rbd-group-snapshot:
- go build -o _output/rbd-group-snapshot ./tools/rbd-group-snapshot
+ go build $(GO_TAGS) -o _output/rbd-group-snapshot ./tools/rbd-group-snapshot
 
 #
 # Update the generated deploy/ files when the template changed. This requires

build.env

@@ -15,8 +15,8 @@ CSI_IMAGE_VERSION=canary
 CSI_UPGRADE_VERSION=v3.12.1
 
 # Ceph version to use
-BASE_IMAGE=quay.io/ceph/ceph:v19
-CEPH_VERSION=squid
+BASE_IMAGE=quay.ceph.io/ceph-ci/ceph:main
+CEPH_VERSION=main
 
 # standard Golang options
 GOLANG_VERSION=1.22.5

deploy/cephcsi/image/Dockerfile

@@ -68,6 +68,7 @@ COPY . ${SRC_DIR}
 
 # Build executable
 RUN make cephcsi
+#RUN make rbd-group-snapshot
 
 #-- Final container
 FROM updated_base
@@ -80,6 +81,7 @@ LABEL maintainers="Ceph-CSI Authors" \
  description="Ceph-CSI Plugin"
 
 COPY --from=builder ${SRC_DIR}/_output/cephcsi /usr/local/bin/cephcsi
+COPY --from=builder ${SRC_DIR}/_output/rbd-group-snapshot /usr/local/bin/rbd-group-snapshot
 
 # verify that all dynamically linked libraries are available
 RUN [ $(ldd /usr/local/bin/cephcsi | grep -c '=> not found') = '0' ]

examples/rbd/groupsnapshotclass.yaml

@@ -3,17 +3,10 @@ apiVersion: groupsnapshot.storage.k8s.io/v1alpha1
 kind: VolumeGroupSnapshotClass
 metadata:
  name: csi-rbdplugin-groupsnapclass
-driver: rbd.csi.ceph.com
+driver: openshift-storage.rbd.csi.ceph.com
 parameters:
- # String representing a Ceph cluster to provision storage from.
- # Should be unique across all Ceph clusters in use for provisioning,
- # cannot be greater than 36 bytes in length, and should remain immutable for
- # the lifetime of the StorageClass in use
- clusterID: <cluster-id>
-
- # eg: pool: rbdpool
- pool: <rbd-pool-name>
-
- csi.storage.k8s.io/group-snapshotter-secret-name: csi-rbd-secret
- csi.storage.k8s.io/group-snapshotter-secret-namespace: default
+ clusterID: openshift-storage
+ pool: ocs-storagecluster-cephblockpool
+ csi.storage.k8s.io/group-snapshotter-secret-name: rook-csi-rbd-provisioner
+ csi.storage.k8s.io/group-snapshotter-secret-namespace: openshift-storage
 deletionPolicy: Delete

examples/rbd/pvc-restore.yaml

@@ -4,9 +4,9 @@ kind: PersistentVolumeClaim
 metadata:
  name: rbd-pvc-restore
 spec:
- storageClassName: csi-rbd-sc
+ storageClassName: ocs-storagecluster-ceph-rbd
  dataSource:
- name: rbd-pvc-snapshot
+ name: snapshot-93c8391e95487ac9d6028dc2ca24d1ab61c2079e9bcec234c40fd65cca036dd5-2024-09-25-9.35.6
  kind: VolumeSnapshot
  apiGroup: snapshot.storage.k8s.io
  accessModes:

examples/rbd/pvc.yaml

@@ -11,4 +11,4 @@ spec:
  resources:
  requests:
  storage: 1Gi
- storageClassName: csi-rbd-sc
+ storageClassName: ocs-storagecluster-ceph-rbd

go.mod

@@ -7,7 +7,7 @@ require (
  github.com/aws/aws-sdk-go v1.55.5
  github.com/aws/aws-sdk-go-v2/service/sts v1.32.1
  github.com/ceph/ceph-csi/api v0.0.0-00010101000000-000000000000
- github.com/ceph/go-ceph v0.29.1-0.20240925141413-065319c78733
+ github.com/ceph/go-ceph v0.29.1-0.20241009173949-08e5b6217b1b
  github.com/container-storage-interface/spec v1.10.0
  github.com/csi-addons/spec v0.2.1-0.20240730084235-3958a5b17d24
  github.com/gemalto/kmip-go v0.0.10

go.sum

@@ -1440,8 +1440,8 @@ github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
-github.com/ceph/go-ceph v0.29.1-0.20240925141413-065319c78733 h1:vX8mfbKwE24CbO5t1Xc02u53pjWAsyvjgAMJk7o7nsQ=
-github.com/ceph/go-ceph v0.29.1-0.20240925141413-065319c78733/go.mod h1:OJFju/Xmtb7ihHo/aXOayw6RhVOUGNke5EwTipwaf6A=
+github.com/ceph/go-ceph v0.29.1-0.20241009173949-08e5b6217b1b h1:xWhxqbBTRGFnUqeaE95wbgLOB9U5FXqjGz4w0PDmVPE=
+github.com/ceph/go-ceph v0.29.1-0.20241009173949-08e5b6217b1b/go.mod h1:OJFju/Xmtb7ihHo/aXOayw6RhVOUGNke5EwTipwaf6A=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

internal/rbd/driver/driver.go

@@ -25,6 +25,7 @@ import (
  csiaddons "github.com/ceph/ceph-csi/internal/csi-addons/server"
  csicommon "github.com/ceph/ceph-csi/internal/csi-common"
  "github.com/ceph/ceph-csi/internal/rbd"
+ "github.com/ceph/ceph-csi/internal/rbd/features"
  "github.com/ceph/ceph-csi/internal/util"
  "github.com/ceph/ceph-csi/internal/util/k8s"
  "github.com/ceph/ceph-csi/internal/util/log"
@@ -123,6 +124,19 @@ func (r *Driver) Run(conf *util.Config) {
  csi.VolumeCapability_AccessMode_SINGLE_NODE_SINGLE_WRITER,
  csi.VolumeCapability_AccessMode_SINGLE_NODE_MULTI_WRITER,
  })
+
+ // GroupSnapGetInfo is used within the VolumeGroupSnapshot implementation
+ vgsSupported, vgsErr := features.SupportsGroupSnapGetInfo()
+ if vgsSupported {
+ r.cd.AddGroupControllerServiceCapabilities([]csi.GroupControllerServiceCapability_RPC_Type{
+ csi.GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT,
+ })
+ } else {
+ log.DefaultLog("not enabling VolumeGroupSnapshot service capability")
+ }
+ if vgsErr != nil {
+ log.ErrorLogMsg("failed detecting VolumeGroupSnapshot support: %v", vgsErr)
+ }
  }
 
  if k8s.RunsOnKubernetes() && conf.IsNodeServer {
@@ -178,6 +192,7 @@ func (r *Driver) Run(conf *util.Config) {
  IS: r.ids,
  CS: r.cs,
  NS: r.ns,
+ GS: r.cs,
  }
  s.Start(conf.Endpoint, srv, csicommon.MiddlewareServerOptionConfig{
  LogSlowOpInterval: conf.LogSlowOpInterval,

internal/rbd/group/group_snapshot.go

@@ -0,0 +1,239 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package group
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/container-storage-interface/spec/lib/go/csi"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ "github.com/ceph/ceph-csi/internal/rbd/types"
+ "github.com/ceph/ceph-csi/internal/util"
+ "github.com/ceph/ceph-csi/internal/util/log"
+)
+
+// volumeGroupSnapshot handles all requests for 'rbd group snap' operations.
+type volumeGroupSnapshot struct {
+ commonVolumeGroup
+
+ snapshots []types.Snapshot
+ // snapshots is a list of rbd-images that are part of the group. The ID
+ // of each snapshot is stored in the journal.
+
+ // snapshotsToFree contains Snapshots that were resolved during
+ // GetVolumeGroupSnapshot.
+ snapshotsToFree []types.Snapshot
+}
+
+// verify that volumeGroupSnapshot implements the VolumeGroupSnapshot interface.
+var _ types.VolumeGroupSnapshot = &volumeGroupSnapshot{}
+
+// GetVolumeGroupSnapshot initializes a new VolumeGroupSnapshot object that can
+// be used to inspect and delete a group of snapshots that was created by a
+// VolumeGroup.
+func GetVolumeGroupSnapshot(
+ ctx context.Context,
+ id string,
+ csiDriver string,
+ creds *util.Credentials,
+ snapshotResolver types.SnapshotResolver,
+) (types.VolumeGroupSnapshot, error) {
+ cleanVGS := true
+
+ vgs := &volumeGroupSnapshot{}
+ err := vgs.initCommonVolumeGroup(ctx, id, csiDriver, creds)
+ if err != nil {
+ return nil, fmt.Errorf("failed to initialize volume group snapshot with id %q: %w", id, err)
+ }
+ defer func() {
+ if cleanVGS {
+ vgs.Destroy(ctx)
+ }
+ }()
+
+ attrs, err := vgs.getVolumeGroupAttributes(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get volume attributes for id %q: %w", vgs, err)
+ }
+
+ var snapshots []types.Snapshot
+ // it is needed to free the previously allocated snapshots in case of an error
+ defer func() {
+ // snapshotsToFree is empty in case of an error, let .Destroy() handle it otherwise
+ if len(vgs.snapshotsToFree) > 0 {
+ return
+ }
+
+ for _, s := range snapshots {
+ s.Destroy(ctx)
+ }
+ }()
+ for snapID := range attrs.VolumeMap {
+ snap, err := snapshotResolver.GetSnapshotByID(ctx, snapID)
+ if err != nil {
+ // free the previously allocated snapshots
+ for _, s := range snapshots {
+ s.Destroy(ctx)
+ }
+
+ return nil, fmt.Errorf("failed to resolve snapshot image %q for volume group snapshot %q: %w", snapID, vgs, err)
+ }
+
+ log.DebugLog(ctx, "resolved snapshot id %q to snapshot %q", snapID, snap)
+
+ snapshots = append(snapshots, snap)
+ }
+
+ vgs.snapshots = snapshots
+ // all allocated snapshots need to be free'd at Destroy() time
+ vgs.snapshotsToFree = snapshots
+
+ cleanVGS = false
+ log.DebugLog(ctx, "GetVolumeGroupSnapshot(%s) returns %+v", id, *vgs)
+
+ return vgs, nil
+}
+
+// NewVolumeGroupSnapshot creates a new VolumeGroupSnapshot object with the
+// given slice of Snapshots and adds the objectmapping to the journal.
+func NewVolumeGroupSnapshot(
+ ctx context.Context,
+ id string,
+ csiDriver string,
+ creds *util.Credentials,
+ snapshots []types.Snapshot,
+) (types.VolumeGroupSnapshot, error) {
+ cleanupVGS := true
+
+ vgs := &volumeGroupSnapshot{}
+ err := vgs.initCommonVolumeGroup(ctx, id, csiDriver, creds)
+ if err != nil {
+ return nil, fmt.Errorf("failed to initialize volume group snapshot with id %q: %w", id, err)
+ }
+ defer func() {
+ if cleanupVGS {
+ vgs.Destroy(ctx)
+ }
+ }()
+
+ vgs.snapshots = snapshots
+ vgs.snapshotsToFree = snapshots
+
+ // create the journal entry
+ _ /* attrs */, err = vgs.getVolumeGroupAttributes(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get volume attributes for id %q: %w", vgs, err)
+ }
+
+ volumeMap := make(map[string]string)
+
+ // add the CSI handles of each snapshot to the journal
+ for _, snapshot := range snapshots {
+ handle, snapErr := snapshot.GetID(ctx)
+ if snapErr != nil {
+ return nil, fmt.Errorf("failed to get ID for snapshot %q of volume group snapshot %q: %w", snapshot, vgs, snapErr)
+ }
+
+ name, snapErr := snapshot.GetName(ctx)
+ if snapErr != nil {
+ return nil, fmt.Errorf("failed to get name for snapshot %q of volume group snapshot %q: %w", snapshot, vgs, snapErr)
+ }
+
+ volumeMap[handle] = name
+ }
+
+ j, err := vgs.getJournal()
+ if err != nil {
+ return nil, err
+ }
+
+ err = j.AddVolumesMapping(ctx, vgs.pool, vgs.objectUUID, volumeMap)
+ if err != nil {
+ return nil, fmt.Errorf("failed to add volume mapping for volume group snapshot %q: %w", vgs, err)
+ }
+
+ // all done successfully, no need to cleanup the returned vgs
+ cleanupVGS = false
+
+ return vgs, nil
+}
+
+// ToCSI creates a CSI-Addons type for the VolumeGroupSnapshot.
+func (vgs *volumeGroupSnapshot) ToCSI(ctx context.Context) (*csi.VolumeGroupSnapshot, error) {
+ snapshots, err := vgs.ListSnapshots(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list snapshots for volume group %q: %w", vgs, err)
+ }
+
+ csiSnapshots := make([]*csi.Snapshot, len(snapshots))
+ for i, snap := range snapshots {
+ csiSnapshots[i], err = snap.ToCSI(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to convert snapshot %q to CSI type: %w", snap, err)
+ }
+ }
+
+ id, err := vgs.GetID(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get id for volume group snapshot %q: %w", vgs, err)
+ }
+
+ ct, err := vgs.GetCreationTime(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get creation time for volume group snapshot %q: %w", vgs, err)
+ }
+
+ return &csi.VolumeGroupSnapshot{
+ GroupSnapshotId: id,
+ Snapshots: csiSnapshots,
+ CreationTime: timestamppb.New(*ct),
+ ReadyToUse: true,
+ }, nil
+}
+
+// Destroy frees the resources used by the volumeGroupSnapshot.
+func (vgs *volumeGroupSnapshot) Destroy(ctx context.Context) {
+ // free the volumes that were allocated in GetVolumeGroup()
+ if len(vgs.snapshotsToFree) > 0 {
+ for _, volume := range vgs.snapshotsToFree {
+ volume.Destroy(ctx)
+ }
+ vgs.snapshotsToFree = make([]types.Snapshot, 0)
+ }
+
+ vgs.commonVolumeGroup.Destroy(ctx)
+}
+
+// Delete removes all snapshots and eventually the volume group snapshot.
+func (vgs *volumeGroupSnapshot) Delete(ctx context.Context) error {
+ for _, snapshot := range vgs.snapshots {
+ log.DebugLog(ctx, "deleting snapshot image %q for volume group snapshot %q", snapshot, vgs)
+
+ err := snapshot.Delete(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to delete snapshot %q as part of volume groups snapshot %q: %w", snapshot, vgs, err)
+ }
+ }
+
+ return vgs.commonVolumeGroup.Delete(ctx)
+}
+
+func (vgs *volumeGroupSnapshot) ListSnapshots(ctx context.Context) ([]types.Snapshot, error) {
+ return vgs.snapshots, nil
+}