Skip to content

Commit

Permalink
Encrypt keys before saving in OMAP file.
Browse files Browse the repository at this point in the history 
Fixes ceph#960

Signed-off-by: Gil Bregman <[email protected]>
  • Loading branch information
@gbregman
gbregman committed Nov 20, 2024
1 parent 53e945a commit cd77ecf
Show file tree
Hide file tree
Showing 12 changed files with 402 additions and 265 deletions.
1 change: 1 addition & 0 deletions ceph-nvmeof.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ state_update_notify = True
state_update_timeout_in_msec = 2000
state_update_interval_sec = 5
enable_spdk_discovery_controller = False
enable_key_encryption = True
#omap_file_lock_duration = 20
#omap_file_lock_retries = 30
#omap_file_lock_retry_sleep_interval = 1.0
Expand Down
30 changes: 27 additions & 3 deletions control/grpc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
from .config import GatewayConfig
from .utils import GatewayEnumUtils
from .utils import GatewayUtils
from .utils import GatewayUtilsCrypto
from .utils import GatewayLogger
from .state import GatewayState, GatewayStateHandler, OmapLock
from .cephutils import CephUtils
Expand Down Expand Up @@ -331,6 +332,8 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rp
self.max_namespaces_per_subsystem = self.config.getint_with_default("gateway", "max_namespaces_per_subsystem", GatewayService.MAX_NAMESPACES_PER_SUBSYSTEM_DEFAULT)
self.max_hosts_per_subsystem = self.config.getint_with_default("gateway", "max_hosts_per_subsystem", GatewayService.MAX_HOSTS_PER_SUBSYS_DEFAULT)
self.gateway_pool = self.config.get_with_default("ceph", "pool", "")
self.enable_key_encryption = self.config.getboolean_with_default("gateway", "enable_key_encryption", True)
self.logger.error(f"XXXXXX enable_key_encryption: {self.enable_key_encryption}") ##################
self.ana_map = defaultdict(dict)
self.cluster_nonce = {}
self.bdev_cluster = {}
Expand Down Expand Up @@ -978,6 +981,10 @@ def create_subsystem_safe(self, request, context):
if context:
# Update gateway state
try:
request.encrypted_dhchap_key = b""
if self.enable_key_encryption and request.dhchap_key:
request.encrypted_dhchap_key = GatewayUtilsCrypto.encrypt_text(request.dhchap_key)
request.dhchap_key = ""
json_req = json_format.MessageToJson(
request, preserving_proto_field_name=True, including_default_value_fields=True)
self.gateway_state.add_subsystem(request.subsystem_nqn, json_req)
Expand Down Expand Up @@ -2547,6 +2554,13 @@ def add_host_safe(self, request, context):
if context:
# Update gateway state
try:
if self.enable_key_encryption:
if request.dhchap_key:
request.encrypted_dhchap_key = GatewayUtilsCrypto.encrypt_text(request.dhchap_key)
request.dhchap_key = ""
if request.psk:
request.encrypted_psk = GatewayUtilsCrypto.encrypt_text(request.psk)
request.psk = ""
json_req = json_format.MessageToJson(
request, preserving_proto_field_name=True, including_default_value_fields=True)
self.gateway_state.add_host(request.subsystem_nqn, request.host_nqn, json_req)
Expand Down Expand Up @@ -2793,12 +2807,17 @@ def change_host_key_safe(self, request, context):
if context:
# Update gateway state
try:
encrypted_dhchap_key = b""
if self.enable_key_encryption and request.dhchap_key:
encrypted_dhchap_key = GatewayUtilsCrypto.encrypt_text(request.dhchap_key)
request.dhchap_key = ""
add_req = pb2.add_host_req(subsystem_nqn=request.subsystem_nqn,
host_nqn=request.host_nqn,
psk=host_psk,
dhchap_key=request.dhchap_key)
dhchap_key=request.dhchap_key,
encrypted_dhchap_key=encrypted_dhchap_key)
json_req = json_format.MessageToJson(
add_req, preserving_proto_field_name=True, including_default_value_fields=True)
add_req, preserving_proto_field_name=True, including_default_value_fields=True)
self.gateway_state.add_host(request.subsystem_nqn, request.host_nqn, json_req)
except Exception as ex:
errmsg = f"Error persisting host change key for host {request.host_nqn} in {request.subsystem_nqn}"
Expand Down Expand Up @@ -3501,12 +3520,17 @@ def change_subsystem_key_safe(self, request, context):

assert subsys_entry, f"Can't find entry for subsystem {request.subsystem_nqn}"
try:
encrypted_dhchap_key = b""
if self.enable_key_encryption and request.dhchap_key:
encrypted_dhchap_key = GatewayUtilsCrypto.encrypt_text(request.dhchap_key)
request.dhchap_key = ""
create_req = pb2.create_subsystem_req(subsystem_nqn=request.subsystem_nqn,
serial_number=subsys_entry["serial_number"],
max_namespaces=subsys_entry["max_namespaces"],
enable_ha=subsys_entry["enable_ha"],
no_group_append=subsys_entry["no_group_append"],
dhchap_key=request.dhchap_key)
dhchap_key=request.dhchap_key,
encrypted_dhchap_key=encrypted_dhchap_key)
json_req = json_format.MessageToJson(
create_req, preserving_proto_field_name=True, including_default_value_fields=True)
self.gateway_state.add_subsystem(request.subsystem_nqn, json_req)
Expand Down
3 changes: 3 additions & 0 deletions control/proto/gateway.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ message create_subsystem_req {
bool enable_ha = 4;
optional bool no_group_append = 5;
optional string dhchap_key = 6;
optional bytes encrypted_dhchap_key = 7;
}

message delete_subsystem_req {
Expand All @@ -215,6 +216,8 @@ message add_host_req {
string host_nqn = 2;
optional string psk = 3;
optional string dhchap_key = 4;
optional bytes encrypted_psk = 5;
optional bytes encrypted_dhchap_key = 6;
}

message change_host_key_req {
Expand Down
10 changes: 10 additions & 0 deletions control/server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from .config import GatewayConfig
from .utils import GatewayLogger
from .utils import GatewayUtils
from .utils import GatewayUtilsCrypto
from .cephutils import CephUtils
from .prometheus import start_exporter

Expand Down Expand Up @@ -764,6 +765,9 @@ def gateway_rpc_caller(self, requests, is_add_req):
if key.startswith(GatewayState.SUBSYSTEM_PREFIX):
if is_add_req:
req = json_format.Parse(val, pb2.create_subsystem_req(), ignore_unknown_fields=True)
if req.encrypted_dhchap_key:
req.dhchap_key = GatewayUtilsCrypto.decrypt_text(req.encrypted_dhchap_key)
req.encrypted_dhchap_key = b""
self.gateway_rpc.create_subsystem(req)
else:
req = json_format.Parse(val,
Expand All @@ -789,6 +793,12 @@ def gateway_rpc_caller(self, requests, is_add_req):
elif key.startswith(GatewayState.HOST_PREFIX):
if is_add_req:
req = json_format.Parse(val, pb2.add_host_req(), ignore_unknown_fields=True)
if req.encrypted_dhchap_key:
req.dhchap_key = GatewayUtilsCrypto.decrypt_text(req.encrypted_dhchap_key)
req.encrypted_dhchap_key = b""
if req.encrypted_psk:
req.psk = GatewayUtilsCrypto.decrypt_text(req.encrypted_psk)
req.encrypted_psk = b""
self.gateway_rpc.add_host(req)
else:
req = json_format.Parse(val, pb2.remove_host_req(), ignore_unknown_fields=True)
Expand Down
54 changes: 35 additions & 19 deletions control/state.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from abc import ABC, abstractmethod
from .utils import GatewayLogger
from .utils import GatewayUtils
from .utils import GatewayUtilsCrypto
from google.protobuf import json_format
from .proto import gateway_pb2 as pb2

Expand Down Expand Up @@ -770,26 +771,30 @@ def host_only_key_changed(self, old_val, new_val):
old_req = json_format.Parse(old_val, pb2.add_host_req(), ignore_unknown_fields=True )
except json_format.ParseError:
self.logger.exception(f"Got exception parsing {old_val}")
return (False, None)
return (False, None, b"")
try:
new_req = json_format.Parse(new_val, pb2.add_host_req(), ignore_unknown_fields=True)
except json_format.ParseError:
self.logger.exeption(f"Got exception parsing {new_val}")
return (False, None)
return (False, None, b"")
if not old_req or not new_req:
self.logger.debug(f"Failed to parse requests, old: {old_val} -> {old_req}, new: {new_val} -> {new_req}")
return (False, None)
return (False, None, b"")
assert old_req != new_req, f"Something was wrong we shouldn't get identical old and new values ({old_req})"
# Because of Json formatting of empty fields we might get a difference here, so just use the same values for empty
if not old_req.dhchap_key:
old_req.dhchap_key = ""
if not old_req.encrypted_dhchap_key:
old_req.encrypted_dhchap_key = b""
if not new_req.dhchap_key:
new_req.dhchap_key = ""
if not new_req.encrypted_dhchap_key:
new_req.encrypted_dhchap_key = b""
old_req.dhchap_key = new_req.dhchap_key
if old_req != new_req:
# Something besides the keys is different
return (False, None)
return (True, new_req.dhchap_key)
return (False, None, b"")
return (True, new_req.dhchap_key, new_req.encrypted_dhchap_key)

def subsystem_only_key_changed(self, old_val, new_val):
# If only the dhchap key field has changed we can use change_key request instead of re-adding the subsystem
Expand All @@ -799,26 +804,31 @@ def subsystem_only_key_changed(self, old_val, new_val):
old_req = json_format.Parse(old_val, pb2.create_subsystem_req(), ignore_unknown_fields=True )
except json_format.ParseError:
self.logger.exception(f"Got exception parsing {old_val}")
return (False, None)
return (False, None, b"")
try:
new_req = json_format.Parse(new_val, pb2.create_subsystem_req(), ignore_unknown_fields=True)
except json_format.ParseError:
self.logger.exeption(f"Got exception parsing {new_val}")
return (False, None)
return (False, None, b"")
if not old_req or not new_req:
self.logger.debug(f"Failed to parse requests, old: {old_val} -> {old_req}, new: {new_val} -> {new_req}")
return (False, None)
return (False, None, b"")
assert old_req != new_req, f"Something was wrong we shouldn't get identical old and new values ({old_req})"
# Because of Json formatting of empty fields we might get a difference here, so just use the same values for empty
if not old_req.dhchap_key:
old_req.dhchap_key = ""
if not old_req.encrypted_dhchap_key:
old_req.encrypted_dhchap_key = b""
if not new_req.dhchap_key:
new_req.dhchap_key = ""
if not new_req.encrypted_dhchap_key:
new_req.encrypted_dhchap_key = b""
old_req.dhchap_key = new_req.dhchap_key
old_req.encrypted_dhchap_key = new_req.encrypted_dhchap_key
if old_req != new_req:
# Something besides the keys is different
return (False, None)
return (True, new_req.dhchap_key)
return (False, None, b"")
return (True, new_req.dhchap_key, new_req.encrypted_dhchap_key)

def break_namespace_key(self, ns_key: str):
if not ns_key.startswith(GatewayState.NAMESPACE_PREFIX):
Expand Down Expand Up @@ -952,21 +962,23 @@ def update(self) -> bool:
elif key.startswith(host_prefix):
try:
(should_process,
new_dhchap_key) = self.host_only_key_changed(local_state_dict[key], omap_state_dict[key])
new_dhchap_key,
new_encypted_dhchap_key) = self.host_only_key_changed(local_state_dict[key], omap_state_dict[key])
if should_process:
assert new_dhchap_key, "Shouldn't get here with an empty dhchap key"
self.logger.debug(f"Found {key} where only the key has changed. The new DHCHAP key is {new_dhchap_key}")
only_host_key_changed.append((key, new_dhchap_key))
self.logger.debug(f"Found {key} where only the key has changed. The new DHCHAP key is {new_dhchap_key}, encrypted key is {new_encypted_dhchap_key}")
only_host_key_changed.append((key, new_dhchap_key, new_encypted_dhchap_key))
except Exception as ex:
self.logger.warning("Got exception checking host for key change")
elif key.startswith(subsystem_prefix):
try:
(should_process,
new_dhchap_key) = self.subsystem_only_key_changed(local_state_dict[key], omap_state_dict[key])
new_dhchap_key,
new_encypted_dhchap_key) = self.subsystem_only_key_changed(local_state_dict[key], omap_state_dict[key])
if should_process:
assert new_dhchap_key, "Shouldn't get here with an empty dhchap key"
self.logger.debug(f"Found {key} where only the key has changed. The new DHCHAP key is {new_dhchap_key}")
only_subsystem_key_changed.append((key, new_dhchap_key))
assert new_dhchap_key or new_encypted_dhchap_key, "Shouldn't get here with an empty DHCHAP key"
self.logger.debug(f"Found {key} where only the key has changed. The new DHCHAP key is {new_dhchap_key}, the encrypted key is {new_encypted_dhchap_key}")
only_subsystem_key_changed.append((key, new_dhchap_key, new_encypted_dhchap_key))
except Exception as ex:
self.logger.warning("Got exception checking subsystem for key change")

Expand All @@ -989,7 +1001,7 @@ def update(self) -> bool:
except Exception as ex:
self.logger.error(f"Exception formatting change namespace load balancing group request:\n{ex}")

for host_key, new_dhchap_key in only_host_key_changed:
for host_key, new_dhchap_key, new_encypted_dhchap_key in only_host_key_changed:
subsys_nqn = None
host_nqn = None
try:
Expand All @@ -1003,6 +1015,8 @@ def update(self) -> bool:
if subsys_nqn and host_nqn:
try:
host_key_key = GatewayState.build_host_key_key(subsys_nqn, host_nqn)
if new_encypted_dhchap_key:
new_dhchap_key = GatewayUtilsCrypto.decrypt_text(new_encypted_dhchap_key)
req = pb2.change_host_key_req(subsystem_nqn=subsys_nqn, host_nqn=host_nqn,
dhchap_key=new_dhchap_key)
json_req = json_format.MessageToJson(req, preserving_proto_field_name=True,
Expand All @@ -1011,7 +1025,7 @@ def update(self) -> bool:
except Exception as ex:
self.logger.error(f"Exception formatting change host key request:\n{ex}")

for subsys_key, new_dhchap_key in only_subsystem_key_changed:
for subsys_key, new_dhchap_key, new_encypted_dhchap_key in only_subsystem_key_changed:
subsys_nqn = None
try:
changed.pop(subsys_key)
Expand All @@ -1021,6 +1035,8 @@ def update(self) -> bool:
if subsys_nqn:
try:
subsys_key_key = GatewayState.build_subsystem_key_key(subsys_nqn)
if new_encypted_dhchap_key:
new_dhchap_key = GatewayUtilsCrypto.decrypt_text(new_encypted_dhchap_key)
req = pb2.change_subsystem_key_req(subsystem_nqn=subsys_nqn, dhchap_key=new_dhchap_key)
json_req = json_format.MessageToJson(req, preserving_proto_field_name=True,
including_default_value_fields=True)
Expand Down
21 changes: 20 additions & 1 deletion control/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
import shutil
import netifaces
from typing import Tuple, List

import nacl.secret
import nacl.utils

class GatewayEnumUtils:
def get_value_from_key(e_type, keyval, ignore_case = False):
Expand Down Expand Up @@ -182,6 +183,24 @@ def is_valid_nqn(nqn):

return (0, os.strerror(0))

class GatewayUtilsCrypto:
ENCRYPTION_KEY = b"Q\xba\x7f._i\x9f%\xed\x96\xe0\xaf\xab k\x9e\xd6\xb6c\xb8\xee\xf6\xa7\xb1s\xf7\xf4\x89f'\x85b"
secret_box = None

@staticmethod
def encrypt_text(msg : str) -> bytes:
if GatewayUtilsCrypto.secret_box == None:
GatewayUtilsCrypto.secret_box = nacl.secret.SecretBox(GatewayUtilsCrypto.ENCRYPTION_KEY)
encrypted = GatewayUtilsCrypto.secret_box.encrypt(msg.encode("utf-8"))
return encrypted

@staticmethod
def decrypt_text(msg : bytes) -> str:
if GatewayUtilsCrypto.secret_box == None:
GatewayUtilsCrypto.secret_box = nacl.secret.SecretBox(GatewayUtilsCrypto.ENCRYPTION_KEY)
plain = GatewayUtilsCrypto.secret_box.decrypt(msg)
return plain.decode("utf-8")

class GatewayLogger:
CEPH_LOG_DIRECTORY = "/var/log/ceph/"
MAX_LOG_FILE_SIZE_DEFAULT = 10
Expand Down
Loading

0 comments on commit cd77ecf

Please sign in to comment.