rbd: Implement FenceClusterNetwork and UnfenceClusterNetwork

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/ceph/go-ceph v0.13.0
 	github.com/container-storage-interface/spec v1.5.0
 	github.com/csi-addons/replication-lib-utils v0.2.0
-	github.com/csi-addons/spec v0.1.2-0.20211220083702-c779b23cf97c
+	github.com/csi-addons/spec v0.1.2-0.20211220115741-32fa508dadbe
 	github.com/golang/protobuf v1.5.2
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0

go.sum

@@ -243,8 +243,8 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ
 github.com/csi-addons/replication-lib-utils v0.2.0 h1:tGs42wfjkObbBo/98a3uxTFWEJ1dq5PIMqPWtdLd040=
 github.com/csi-addons/replication-lib-utils v0.2.0/go.mod h1:ROQlEsc2EerVtc/K/C+6Hx8pqaQ9MVy9xFFpyKfI9lc=
 github.com/csi-addons/spec v0.1.0/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
-github.com/csi-addons/spec v0.1.2-0.20211220083702-c779b23cf97c h1:VVPkxQWRwXzGktb00o2m/wxN4xy/la8qSsTK/GRqg/g=
-github.com/csi-addons/spec v0.1.2-0.20211220083702-c779b23cf97c/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
+github.com/csi-addons/spec v0.1.2-0.20211220115741-32fa508dadbe h1:Q2sxgtdRV4Je1R2eLCUPrR/KQZxkSbesGrpCjl0/mU4=
+github.com/csi-addons/spec v0.1.2-0.20211220115741-32fa508dadbe/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
 github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU=

internal/csi-addons/networkfence/fencing.go

@@ -0,0 +1,194 @@
+/*
+Copyright 2022 The Ceph-CSI Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package networkfence
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+
+	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/log"
+
+	"github.com/csi-addons/spec/lib/go/fence"
+)
+
+const blocklistTime = "157784760"
+
+// NetworkFence contains the CIDR blocks to be blocked.
+type NetworkFence struct {
+	Cidr     []string
+	Monitors string
+	cr       *util.Credentials
+}
+
+// NewNetworkFence returns a networkFence struct object from the Network fence/unfence request.
+func NewNetworkFence(
+	ctx context.Context,
+	cr *util.Credentials,
+	cidrs []*fence.CIDR,
+	fenceOptions map[string]string) (*NetworkFence, error) {
+	var err error
+	nwFence := &NetworkFence{}
+
+	nwFence.Cidr, err = GetCIDR(cidrs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get list of CIDRs: %w", err)
+	}
+
+	clusterID, err := util.GetClusterID(fenceOptions)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch clusterID: %w", err)
+	}
+
+	nwFence.Monitors, _, err = util.GetMonsAndClusterID(ctx, clusterID, false)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get monitors for clusterID %q: %w", clusterID, err)
+	}
+
+	nwFence.cr = cr
+
+	return nwFence, nil
+}
+
+// addCephBlocklist adds an IP to ceph osd blocklist.
+func (nf *NetworkFence) addCephBlocklist(ctx context.Context, ip string) error {
+	arg := []string{
+		"--id", nf.cr.ID,
+		"--keyfile=" + nf.cr.KeyFile,
+		"-m", nf.Monitors,
+	}
+	// TODO: add blocklist till infinity.
+	// Currently, ceph does not provide the functionality to blocklist IPs
+	// for infinite time. As a workaround, add a blocklist for 5 YEARS to
+	// represent infinity from ceph-csi side.
+	// At any point in this time, the IPs can be unblocked by an UnfenceClusterReq.
+	// This needs to be updated once ceph provides functionality for the same.
+	cmd := []string{"osd", "blocklist", "add", ip, blocklistTime}
+	cmd = append(cmd, arg...)
+	_, _, err := util.ExecCommand(ctx, "ceph", cmd...)
+	if err != nil {
+		return fmt.Errorf("failed to blocklist IP %q: %w", ip, err)
+	}
+	log.DebugLog(ctx, "blocklisted IP %q successfully", ip)
+
+	return nil
+}
+
+// AddNetworkFence blocks access for all the IPs in the IP range mentioned via the CIDR block
+// using a network fence.
+func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error {
+	// for each CIDR block, convert it into a range of IPs so as to perform blocklisting operation.
+	for _, cidr := range nf.Cidr {
+		// fetch the list of IPs from a CIDR block
+		hosts, err := getIPRange(cidr)
+		if err != nil {
+			return fmt.Errorf("failed to convert CIDR block %s to corresponding IP range: %w", cidr, err)
+		}
+
+		// add ceph blocklist for each IP in the range mentioned by the CIDR
+		for _, host := range hosts {
+			err = nf.addCephBlocklist(ctx, host)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// getIPRange returns a list of IPs from the IP range
+// corresponding to a CIDR block.
+func getIPRange(cidr string) ([]string, error) {
+	var hosts []string
+	netIP, ipnet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return nil, err
+	}
+	for ip := netIP.Mask(ipnet.Mask); ipnet.Contains(ip); incIP(ip) {
+		hosts = append(hosts, ip.String())
+	}
+
+	return hosts, nil
+}
+
+// incIP is an helper function for getIPRange() for incrementing
+// IP values to return all IPs in a range.
+func incIP(ip net.IP) {
+	for j := len(ip) - 1; j >= 0; j-- {
+		ip[j]++
+		if ip[j] > 0 {
+			break
+		}
+	}
+}
+
+// Cidrs is a list of CIDR structs.
+type Cidrs []*fence.CIDR
+
+// GetCIDR converts a CIDR struct list to a list.
+func GetCIDR(cidrs Cidrs) ([]string, error) {
+	var cidrList []string
+	for _, cidr := range cidrs {
+		cidrList = append(cidrList, cidr.Cidr)
+	}
+	if len(cidrList) < 1 {
+		return nil, errors.New("the CIDR cannot be empty")
+	}
+
+	return cidrList, nil
+}
+
+// removeCephBlocklist removes an IP from ceph osd blocklist.
+func (nf *NetworkFence) removeCephBlocklist(ctx context.Context, ip string) error {
+	arg := []string{
+		"--id", nf.cr.ID,
+		"--keyfile=" + nf.cr.KeyFile,
+		"-m", nf.Monitors,
+	}
+	cmd := []string{"osd", "blocklist", "rm", ip}
+	cmd = append(cmd, arg...)
+
+	_, stdErr, err := util.ExecCommand(ctx, "ceph", cmd...)
+	if err != nil {
+		return fmt.Errorf("failed to unblock IP %q: %v %w", ip, stdErr, err)
+	}
+	log.DebugLog(ctx, "unblocked IP %q successfully", ip)
+
+	return nil
+}
+
+// RemoveNetworkFence unblocks access for all the IPs in the IP range mentioned via the CIDR block
+// using a network fence.
+func (nf *NetworkFence) RemoveNetworkFence(ctx context.Context) error {
+	// for each CIDR block, convert it into a range of IPs so as to undo blocklisting operation.
+	for _, cidr := range nf.Cidr {
+		// fetch the list of IPs from a CIDR block
+		hosts, err := getIPRange(cidr)
+		if err != nil {
+			return fmt.Errorf("failed to convert CIDR block %s to corresponding IP range", cidr)
+		}
+		// remove ceph blocklist for each IP in the range mentioned by the CIDR
+		for _, host := range hosts {
+			err := nf.removeCephBlocklist(ctx, host)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

internal/csi-addons/networkfence/fencing_test.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2022 The Ceph-CSI Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package networkfence
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetIPRange(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		cidr        string
+		expectedIPs []string
+	}{
+		{
+			cidr:        "192.168.1.0/31",
+			expectedIPs: []string{"192.168.1.0", "192.168.1.1"},
+		},
+		{
+			cidr:        "10.0.0.0/30",
+			expectedIPs: []string{"10.0.0.0", "10.0.0.1", "10.0.0.2", "10.0.0.3"},
+		},
+		{
+			cidr:        "fd4a:ecbc:cafd:4e49::/127",
+			expectedIPs: []string{"fd4a:ecbc:cafd:4e49::", "fd4a:ecbc:cafd:4e49::1"},
+		},
+	}
+	for _, tt := range tests {
+		ts := tt
+		t.Run(ts.cidr, func(t *testing.T) {
+			t.Parallel()
+			got, err := getIPRange(ts.cidr)
+			assert.NoError(t, err)
+
+			// validate if number of IPs in the range is same as expected, if not, fail.
+			assert.ElementsMatch(t, ts.expectedIPs, got)
+		})
+	}
+}

internal/csi-addons/rbd/identity.go

@@ -82,6 +82,12 @@ func (is *IdentityServer) GetCapabilities(
 						Type: identity.Capability_ReclaimSpace_OFFLINE,
 					},
 				},
+			}, &identity.Capability{
+				Type: &identity.Capability_NetworkFence_{
+					NetworkFence: &identity.Capability_NetworkFence{
+						Type: identity.Capability_NetworkFence_NETWORK_FENCE,
+					},
+				},
 			})
 	}
 

internal/csi-addons/rbd/network_fence.go

@@ -0,0 +1,114 @@
+/*
+Copyright 2022 The Ceph-CSI Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"errors"
+
+	nf "github.com/ceph/ceph-csi/internal/csi-addons/networkfence"
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/csi-addons/spec/lib/go/fence"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// FenceControllerServer struct of rbd CSI driver with supported methods
+// of CSI-addons networkfence controller service spec.
+type FenceControllerServer struct {
+	*fence.UnimplementedFenceControllerServer
+}
+
+// NewFenceControllerServer creates a new IdentityServer which handles
+// the Identity Service requests from the CSI-Addons specification.
+func NewFenceControllerServer() *FenceControllerServer {
+	return &FenceControllerServer{}
+}
+
+func (fcs *FenceControllerServer) RegisterService(server grpc.ServiceRegistrar) {
+	fence.RegisterFenceControllerServer(server, fcs)
+}
+
+// validateFenceClusterNetworkReq checks the sanity of FenceClusterNetworkRequest.
+func validateNetworkFenceReq(fenceClients []*fence.CIDR, options map[string]string) error {
+	if len(fenceClients) == 0 {
+		return errors.New("CIDR block cannot be empty")
+	}
+
+	if value, ok := options["clusterID"]; !ok || value == "" {
+		return errors.New("missing or empty clusterID")
+	}
+
+	return nil
+}
+
+// FenceClusterNetwork blocks access to a CIDR block by creating a network fence.
+// It adds the range of IPs to the osd blocklist, which helps ceph in denying access
+// to the malicious clients to prevent data corruption.
+func (fcs *FenceControllerServer) FenceClusterNetwork(
+	ctx context.Context,
+	req *fence.FenceClusterNetworkRequest) (*fence.FenceClusterNetworkResponse, error) {
+	err := validateNetworkFenceReq(req.GetCidrs(), req.Parameters)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	cr, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	nwFence, err := nf.NewNetworkFence(ctx, cr, req.Cidrs, req.GetParameters())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	err = nwFence.AddNetworkFence(ctx)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to fence CIDR block %q: %s", nwFence.Cidr, err.Error())
+	}
+
+	return &fence.FenceClusterNetworkResponse{}, nil
+}
+
+// UnfenceClusterNetwork unblocks the access to a CIDR block by removing the network fence.
+func (fcs *FenceControllerServer) UnfenceClusterNetwork(
+	ctx context.Context,
+	req *fence.UnfenceClusterNetworkRequest) (*fence.UnfenceClusterNetworkResponse, error) {
+	err := validateNetworkFenceReq(req.GetCidrs(), req.Parameters)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	cr, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	nwFence, err := nf.NewNetworkFence(ctx, cr, req.Cidrs, req.GetParameters())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	err = nwFence.RemoveNetworkFence(ctx)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to unfence CIDR block %q: %s", nwFence.Cidr, err.Error())
+	}
+
+	return &fence.UnfenceClusterNetworkResponse{}, nil
+}