+2098

create-external-cluster-resources.py

reviewed

··· 1 1 + """ 2 2 + Copyright 2020 The Rook Authors. All rights reserved. 3 3 + 4 4 + Licensed under the Apache License, Version 2.0 (the "License"); 5 5 + you may not use this file except in compliance with the License. 6 6 + You may obtain a copy of the License at 7 7 + 8 8 + http://www.apache.org/licenses/LICENSE-2.0 9 9 + 10 10 + Unless required by applicable law or agreed to in writing, software 11 11 + distributed under the License is distributed on an "AS IS" BASIS, 12 12 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 13 + See the License for the specific language governing permissions and 14 14 + limitations under the License. 15 15 + """ 16 16 + 17 17 + import errno 18 18 + import sys 19 19 + import json 20 20 + import argparse 21 21 + import re 22 22 + import subprocess 23 23 + import hmac 24 24 + from hashlib import sha1 as sha 25 25 + from os import linesep as LINESEP 26 26 + from os import path 27 27 + from email.utils import formatdate 28 28 + import requests 29 29 + from requests.auth import AuthBase 30 30 + 31 31 + py3k = False 32 32 + if sys.version_info.major >= 3: 33 33 + py3k = True 34 34 + import urllib.parse 35 35 + from ipaddress import ip_address, IPv4Address 36 36 + 37 37 + ModuleNotFoundError = ImportError 38 38 + 39 39 + try: 40 40 + import rados 41 41 + except ModuleNotFoundError as noModErr: 42 42 + print(f"Error: {noModErr}\nExiting the script...") 43 43 + sys.exit(1) 44 44 + 45 45 + try: 46 46 + import rbd 47 47 + except ModuleNotFoundError as noModErr: 48 48 + print(f"Error: {noModErr}\nExiting the script...") 49 49 + sys.exit(1) 50 50 + 51 51 + try: 52 52 + # for 2.7.x 53 53 + from StringIO import StringIO 54 54 + except ModuleNotFoundError: 55 55 + # for 3.x 56 56 + from io import StringIO 57 57 + 58 58 + try: 59 59 + # for 2.7.x 60 60 + from urlparse import urlparse 61 61 + from urllib import urlencode as urlencode 62 62 + except ModuleNotFoundError: 63 63 + # for 3.x 64 64 + from urllib.parse import urlparse 65 65 + from urllib.parse import urlencode as urlencode 66 66 + 67 67 + try: 68 68 + from base64 import encodestring 69 69 + except: 70 70 + from base64 import encodebytes as encodestring 71 71 + 72 72 + 73 73 + class ExecutionFailureException(Exception): 74 74 + pass 75 75 + 76 76 + 77 77 + ################################################ 78 78 + ################## DummyRados ################## 79 79 + ################################################ 80 80 + # this is mainly for testing and could be used where 'rados' is not available 81 81 + 82 82 + 83 83 + class DummyRados(object): 84 84 + def __init__(self): 85 85 + self.return_val = 0 86 86 + self.err_message = "" 87 87 + self.state = "connected" 88 88 + self.cmd_output_map = {} 89 89 + self.cmd_names = {} 90 90 + self._init_cmd_output_map() 91 91 + self.dummy_host_ip_map = {} 92 92 + 93 93 + def _init_cmd_output_map(self): 94 94 + json_file_name = "test-data/ceph-status-out" 95 95 + script_dir = path.abspath(path.dirname(__file__)) 96 96 + ceph_status_str = "" 97 97 + with open( 98 98 + path.join(script_dir, json_file_name), mode="r", encoding="UTF-8" 99 99 + ) as json_file: 100 100 + ceph_status_str = json_file.read() 101 101 + self.cmd_names["fs ls"] = """{"format": "json", "prefix": "fs ls"}""" 102 102 + self.cmd_names["quorum_status"] = ( 103 103 + """{"format": "json", "prefix": "quorum_status"}""" 104 104 + ) 105 105 + self.cmd_names["mgr services"] = ( 106 106 + """{"format": "json", "prefix": "mgr services"}""" 107 107 + ) 108 108 + # all the commands and their output 109 109 + self.cmd_output_map[self.cmd_names["fs ls"]] = ( 110 110 + """[{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":2,"data_pool_ids":[3],"data_pools":["myfs-replicated"]}]""" 111 111 + ) 112 112 + self.cmd_output_map[self.cmd_names["quorum_status"]] = ( 113 113 + """{"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_leader_name":"a","quorum_age":14385,"features":{"quorum_con":"4540138292836696063","quorum_mon":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus"]},"monmap":{"epoch":1,"fsid":"af4e1673-0b72-402d-990a-22d2919d0f1c","modified":"2020-05-07T03:36:39.918035Z","created":"2020-05-07T03:36:39.918035Z","min_mon_release":15,"min_mon_release_name":"octopus","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"10.110.205.174:3300","nonce":0},{"type":"v1","addr":"10.110.205.174:6789","nonce":0}]},"addr":"10.110.205.174:6789/0","public_addr":"10.110.205.174:6789/0","priority":0,"weight":0}]}}""" 114 114 + ) 115 115 + self.cmd_output_map[self.cmd_names["mgr services"]] = ( 116 116 + """{"dashboard":"https://ceph-dashboard:8443/","prometheus":"http://ceph-dashboard-db:9283/"}""" 117 117 + ) 118 118 + self.cmd_output_map[ 119 119 + """{"caps": ["mon", "allow r, allow command quorum_status", "osd", "profile rbd-read-only, allow rwx pool=default.rgw.meta, allow r pool=.rgw.root, allow rw pool=default.rgw.control, allow x pool=default.rgw.buckets.index"], "entity": "client.healthchecker", "format": "json", "prefix": "auth get-or-create"}""" 120 120 + ] = """[{"entity":"client.healthchecker","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow r, allow command quorum_status","osd":"profile rbd-read-only, allow rwx pool=default.rgw.meta, allow r pool=.rgw.root, allow rw pool=default.rgw.control, allow x pool=default.rgw.buckets.index"}}]""" 121 121 + self.cmd_output_map[ 122 122 + """{"caps": ["mon", "profile rbd, allow command 'osd blocklist'", "osd", "profile rbd"], "entity": "client.csi-rbd-node", "format": "json", "prefix": "auth get-or-create"}""" 123 123 + ] = """[{"entity":"client.csi-rbd-node","key":"AQBOgrNeHbK1AxAAubYBeV8S1U/GPzq5SVeq6g==","caps":{"mon":"profile rbd, allow command 'osd blocklist'","osd":"profile rbd"}}]""" 124 124 + self.cmd_output_map[ 125 125 + """{"caps": ["mon", "profile rbd, allow command 'osd blocklist'", "mgr", "allow rw", "osd", "profile rbd"], "entity": "client.csi-rbd-provisioner", "format": "json", "prefix": "auth get-or-create"}""" 126 126 + ] = """[{"entity":"client.csi-rbd-provisioner","key":"AQBNgrNe1geyKxAA8ekViRdE+hss5OweYBkwNg==","caps":{"mgr":"allow rw","mon":"profile rbd, allow command 'osd blocklist'","osd":"profile rbd"}}]""" 127 127 + self.cmd_output_map[ 128 128 + """{"caps": ["mon", "allow r, allow command 'osd blocklist'", "mgr", "allow rw", "osd", "allow rw tag cephfs *=*", "mds", "allow rw"], "entity": "client.csi-cephfs-node", "format": "json", "prefix": "auth get-or-create"}""" 129 129 + ] = """[{"entity":"client.csi-cephfs-node","key":"AQBOgrNeENunKxAAPCmgE7R6G8DcXnaJ1F32qg==","caps":{"mds":"allow rw","mgr":"allow rw","mon":"allow r, allow command 'osd blocklist'","osd":"allow rw tag cephfs *=*"}}]""" 130 130 + self.cmd_output_map[ 131 131 + """{"caps": ["mon", "allow r, allow command 'osd blocklist'", "mgr", "allow rw", "osd", "allow rw tag cephfs metadata=*"], "entity": "client.csi-cephfs-provisioner", "format": "json", "prefix": "auth get-or-create"}""" 132 132 + ] = """[{"entity":"client.csi-cephfs-provisioner","key":"AQBOgrNeAFgcGBAAvGqKOAD0D3xxmVY0R912dg==","caps":{"mgr":"allow rw","mon":"allow r, allow command 'osd blocklist'","osd":"allow rw tag cephfs metadata=*"}}]""" 133 133 + self.cmd_output_map[ 134 134 + """{"caps": ["mon", "allow r, allow command 'osd blocklist'", "mgr", "allow rw", "osd", "allow rw tag cephfs metadata=*"], "entity": "client.csi-cephfs-provisioner-openshift-storage", "format": "json", "prefix": "auth get-or-create"}""" 135 135 + ] = """[{"entity":"client.csi-cephfs-provisioner-openshift-storage","key":"BQBOgrNeAFgcGBAAvGqKOAD0D3xxmVY0R912dg==","caps":{"mgr":"allow rw","mon":"allow r, allow command 'osd blocklist'","osd":"allow rw tag cephfs metadata=*"}}]""" 136 136 + self.cmd_output_map[ 137 137 + """{"caps": ["mon", "allow r, allow command 'osd blocklist'", "mgr", "allow rw", "osd", "allow rw tag cephfs metadata=myfs"], "entity": "client.csi-cephfs-provisioner-openshift-storage-myfs", "format": "json", "prefix": "auth get-or-create"}""" 138 138 + ] = """[{"entity":"client.csi-cephfs-provisioner-openshift-storage-myfs","key":"CQBOgrNeAFgcGBAAvGqKOAD0D3xxmVY0R912dg==","caps":{"mgr":"allow rw","mon":"allow r, allow command 'osd blocklist'","osd":"allow rw tag cephfs metadata=myfs"}}]""" 139 139 + self.cmd_output_map[ 140 140 + """{"caps": ["mon", "allow r, allow command quorum_status, allow command version", "mgr", "allow command config", "osd", "profile rbd-read-only, allow rwx pool=default.rgw.meta, allow r pool=.rgw.root, allow rw pool=default.rgw.control, allow rx pool=default.rgw.log, allow x pool=default.rgw.buckets.index"], "entity": "client.healthchecker", "format": "json", "prefix": "auth get-or-create"}""" 141 141 + ] = """[{"entity":"client.healthchecker","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon": "allow r, allow command quorum_status, allow command version", "mgr": "allow command config", "osd": "profile rbd-read-only, allow rwx pool=default.rgw.meta, allow r pool=.rgw.root, allow rw pool=default.rgw.control, allow rx pool=default.rgw.log, allow x pool=default.rgw.buckets.index"}}]""" 142 142 + self.cmd_output_map[ 143 143 + """{"caps": ["mon", "allow r, allow command quorum_status, allow command version", "mgr", "allow command config", "osd", "profile rbd-read-only, allow rwx pool=default.rgw.meta, allow r pool=.rgw.root, allow rw pool=default.rgw.control, allow rx pool=default.rgw.log, allow x pool=default.rgw.buckets.index"], "entity": "client.healthchecker", "format": "json", "prefix": "auth caps"}""" 144 144 + ] = """[{"entity":"client.healthchecker","key":"AQDFkbNeft5bFRAATndLNUSRKruozxiZi3lrdA==","caps":{"mon": "allow r, allow command quorum_status, allow command version", "mgr": "allow command config", "osd": "profile rbd-read-only, allow rwx pool=default.rgw.meta, allow r pool=.rgw.root, allow rw pool=default.rgw.control, allow rx pool=default.rgw.log, allow x pool=default.rgw.buckets.index"}}]""" 145 145 + self.cmd_output_map["""{"format": "json", "prefix": "mgr services"}"""] = ( 146 146 + """{"dashboard": "http://rook-ceph-mgr-a-57cf9f84bc-f4jnl:7000/", "prometheus": "http://rook-ceph-mgr-a-57cf9f84bc-f4jnl:9283/"}""" 147 147 + ) 148 148 + self.cmd_output_map[ 149 149 + """{"entity": "client.healthchecker", "format": "json", "prefix": "auth get"}""" 150 150 + ] = """{"dashboard": "http://rook-ceph-mgr-a-57cf9f84bc-f4jnl:7000/", "prometheus": "http://rook-ceph-mgr-a-57cf9f84bc-f4jnl:9283/"}""" 151 151 + self.cmd_output_map[ 152 152 + """{"entity": "client.healthchecker", "format": "json", "prefix": "auth get"}""" 153 153 + ] = """[{"entity":"client.healthchecker","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon": "allow r, allow command quorum_status, allow command version", "mgr": "allow command config", "osd": "profile rbd-read-only, allow rwx pool=default.rgw.meta, allow r pool=.rgw.root, allow rw pool=default.rgw.control, allow rx pool=default.rgw.log, allow x pool=default.rgw.buckets.index"}}]""" 154 154 + self.cmd_output_map[ 155 155 + """{"entity": "client.csi-cephfs-node", "format": "json", "prefix": "auth get"}""" 156 156 + ] = """[]""" 157 157 + self.cmd_output_map[ 158 158 + """{"entity": "client.csi-rbd-node", "format": "json", "prefix": "auth get"}""" 159 159 + ] = """[]""" 160 160 + self.cmd_output_map[ 161 161 + """{"entity": "client.csi-rbd-provisioner", "format": "json", "prefix": "auth get"}""" 162 162 + ] = """[]""" 163 163 + self.cmd_output_map[ 164 164 + """{"entity": "client.csi-cephfs-provisioner", "format": "json", "prefix": "auth get"}""" 165 165 + ] = """[]""" 166 166 + self.cmd_output_map[ 167 167 + """{"entity": "client.csi-cephfs-provisioner-openshift-storage", "format": "json", "prefix": "auth get"}""" 168 168 + ] = """[]""" 169 169 + self.cmd_output_map[ 170 170 + """{"entity": "client.csi-cephfs-provisioner-openshift-storage-myfs", "format": "json", "prefix": "auth get"}""" 171 171 + ] = """[]""" 172 172 + self.cmd_output_map[ 173 173 + """{"entity": "client.csi-cephfs-provisioner", "format": "json", "prefix": "auth get"}""" 174 174 + ] = """[{"entity":"client.csi-cephfs-provisioner","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow r", "mgr":"allow rw", "osd":"allow rw tag cephfs metadata=*"}}]""" 175 175 + self.cmd_output_map[ 176 176 + """{"caps": ["mon", "allow r, allow command 'osd blocklist'", "mgr", "allow rw", "osd", "allow rw tag cephfs metadata=*"], "entity": "client.csi-cephfs-provisioner", "format": "json", "prefix": "auth caps"}""" 177 177 + ] = """[{"entity":"client.csi-cephfs-provisioner","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow r, allow command 'osd blocklist'", "mgr":"allow rw", "osd":"allow rw tag cephfs metadata=*"}}]""" 178 178 + self.cmd_output_map['{"format": "json", "prefix": "status"}'] = ceph_status_str 179 179 + 180 180 + def shutdown(self): 181 181 + pass 182 182 + 183 183 + def get_fsid(self): 184 184 + return "af4e1673-0b72-402d-990a-22d2919d0f1c" 185 185 + 186 186 + def conf_read_file(self): 187 187 + pass 188 188 + 189 189 + def connect(self): 190 190 + pass 191 191 + 192 192 + def pool_exists(self, pool_name): 193 193 + return True 194 194 + 195 195 + def mon_command(self, cmd, out): 196 196 + json_cmd = json.loads(cmd) 197 197 + json_cmd_str = json.dumps(json_cmd, sort_keys=True) 198 198 + cmd_output = self.cmd_output_map[json_cmd_str] 199 199 + return self.return_val, cmd_output, str(self.err_message.encode("utf-8")) 200 200 + 201 201 + def _convert_hostname_to_ip(self, host_name): 202 202 + ip_reg_x = re.compile(r"\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}") 203 203 + # if provided host is directly an IP address, return the same 204 204 + if ip_reg_x.match(host_name): 205 205 + return host_name 206 206 + import random 207 207 + 208 208 + host_ip = self.dummy_host_ip_map.get(host_name, "") 209 209 + if not host_ip: 210 210 + host_ip = f"172.9.{random.randint(0, 254)}.{random.randint(0, 254)}" 211 211 + self.dummy_host_ip_map[host_name] = host_ip 212 212 + del random 213 213 + return host_ip 214 214 + 215 215 + @classmethod 216 216 + def Rados(conffile=None): 217 217 + return DummyRados() 218 218 + 219 219 + 220 220 + class S3Auth(AuthBase): 221 221 + """Attaches AWS Authentication to the given Request object.""" 222 222 + 223 223 + service_base_url = "s3.amazonaws.com" 224 224 + 225 225 + def __init__(self, access_key, secret_key, service_url=None): 226 226 + if service_url: 227 227 + self.service_base_url = service_url 228 228 + self.access_key = str(access_key) 229 229 + self.secret_key = str(secret_key) 230 230 + 231 231 + def __call__(self, r): 232 232 + # Create date header if it is not created yet. 233 233 + if "date" not in r.headers and "x-amz-date" not in r.headers: 234 234 + r.headers["date"] = formatdate(timeval=None, localtime=False, usegmt=True) 235 235 + signature = self.get_signature(r) 236 236 + if py3k: 237 237 + signature = signature.decode("utf-8") 238 238 + r.headers["Authorization"] = f"AWS {self.access_key}:{signature}" 239 239 + return r 240 240 + 241 241 + def get_signature(self, r): 242 242 + canonical_string = self.get_canonical_string(r.url, r.headers, r.method) 243 243 + if py3k: 244 244 + key = self.secret_key.encode("utf-8") 245 245 + msg = canonical_string.encode("utf-8") 246 246 + else: 247 247 + key = self.secret_key 248 248 + msg = canonical_string 249 249 + h = hmac.new(key, msg, digestmod=sha) 250 250 + return encodestring(h.digest()).strip() 251 251 + 252 252 + def get_canonical_string(self, url, headers, method): 253 253 + parsedurl = urlparse(url) 254 254 + objectkey = parsedurl.path[1:] 255 255 + 256 256 + bucket = parsedurl.netloc[: -len(self.service_base_url)] 257 257 + if len(bucket) > 1: 258 258 + # remove last dot 259 259 + bucket = bucket[:-1] 260 260 + 261 261 + interesting_headers = {"content-md5": "", "content-type": "", "date": ""} 262 262 + for key in headers: 263 263 + lk = key.lower() 264 264 + try: 265 265 + lk = lk.decode("utf-8") 266 266 + except: 267 267 + pass 268 268 + if headers[key] and ( 269 269 + lk in interesting_headers.keys() or lk.startswith("x-amz-") 270 270 + ): 271 271 + interesting_headers[lk] = headers[key].strip() 272 272 + 273 273 + # If x-amz-date is used it supersedes the date header. 274 274 + if not py3k: 275 275 + if "x-amz-date" in interesting_headers: 276 276 + interesting_headers["date"] = "" 277 277 + else: 278 278 + if "x-amz-date" in interesting_headers: 279 279 + interesting_headers["date"] = "" 280 280 + 281 281 + buf = f"{method}\n" 282 282 + for key in sorted(interesting_headers.keys()): 283 283 + val = interesting_headers[key] 284 284 + if key.startswith("x-amz-"): 285 285 + buf += f"{key}:{val}\n" 286 286 + else: 287 287 + buf += f"{val}\n" 288 288 + 289 289 + # append the bucket if it exists 290 290 + if bucket != "": 291 291 + buf += f"/{bucket}" 292 292 + 293 293 + # add the objectkey. even if it doesn't exist, add the slash 294 294 + buf += f"/{objectkey}" 295 295 + 296 296 + return buf 297 297 + 298 298 + 299 299 + class RadosJSON: 300 300 + EXTERNAL_USER_NAME = "client.healthchecker" 301 301 + EXTERNAL_RGW_ADMIN_OPS_USER_NAME = "rgw-admin-ops-user" 302 302 + EMPTY_OUTPUT_LIST = "Empty output list" 303 303 + DEFAULT_RGW_POOL_PREFIX = "default" 304 304 + DEFAULT_MONITORING_ENDPOINT_PORT = "9283" 305 305 + 306 306 + @classmethod 307 307 + def gen_arg_parser(cls, args_to_parse=None): 308 308 + argP = argparse.ArgumentParser() 309 309 + 310 310 + common_group = argP.add_argument_group("common") 311 311 + common_group.add_argument("--verbose", "-v", action="store_true", default=False) 312 312 + common_group.add_argument( 313 313 + "--ceph-conf", "-c", help="Provide a ceph conf file.", type=str 314 314 + ) 315 315 + common_group.add_argument( 316 316 + "--keyring", "-k", help="Path to ceph keyring file.", type=str 317 317 + ) 318 318 + common_group.add_argument( 319 319 + "--run-as-user", 320 320 + "-u", 321 321 + default="", 322 322 + type=str, 323 323 + help="Provides a user name to check the cluster's health status, must be prefixed by 'client.'", 324 324 + ) 325 325 + common_group.add_argument( 326 326 + "--cluster-name", 327 327 + default="", 328 328 + help="Kubernetes cluster name(legacy flag), Note: Either use this or --k8s-cluster-name", 329 329 + ) 330 330 + common_group.add_argument( 331 331 + "--k8s-cluster-name", default="", help="Kubernetes cluster name" 332 332 + ) 333 333 + common_group.add_argument( 334 334 + "--namespace", 335 335 + default="", 336 336 + help="Namespace where CephCluster is running", 337 337 + ) 338 338 + common_group.add_argument( 339 339 + "--rgw-pool-prefix", default="", help="RGW Pool prefix" 340 340 + ) 341 341 + common_group.add_argument( 342 342 + "--restricted-auth-permission", 343 343 + default=False, 344 344 + help="Restrict cephCSIKeyrings auth permissions to specific pools, cluster." 345 345 + + "Mandatory flags that need to be set are --rbd-data-pool-name, and --k8s-cluster-name." 346 346 + + "--cephfs-filesystem-name flag can also be passed in case of cephfs user restriction, so it can restrict user to particular cephfs filesystem" 347 347 + + "sample run: `python3 /etc/ceph/create-external-cluster-resources.py --cephfs-filesystem-name myfs --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --restricted-auth-permission true`" 348 348 + + "Note: Restricting the csi-users per pool, and per cluster will require creating new csi-users and new secrets for that csi-users." 349 349 + + "So apply these secrets only to new `Consumer cluster` deployment while using the same `Source cluster`.", 350 350 + ) 351 351 + common_group.add_argument( 352 352 + "--v2-port-enable", 353 353 + action="store_true", 354 354 + default=False, 355 355 + help="Enable v2 mon port(3300) for mons", 356 356 + ) 357 357 + 358 358 + output_group = argP.add_argument_group("output") 359 359 + output_group.add_argument( 360 360 + "--format", 361 361 + "-t", 362 362 + choices=["json", "bash"], 363 363 + default="json", 364 364 + help="Provides the output format (json | bash)", 365 365 + ) 366 366 + output_group.add_argument( 367 367 + "--output", 368 368 + "-o", 369 369 + default="", 370 370 + help="Output will be stored into the provided file", 371 371 + ) 372 372 + output_group.add_argument( 373 373 + "--cephfs-filesystem-name", 374 374 + default="", 375 375 + help="Provides the name of the Ceph filesystem", 376 376 + ) 377 377 + output_group.add_argument( 378 378 + "--cephfs-metadata-pool-name", 379 379 + default="", 380 380 + help="Provides the name of the cephfs metadata pool", 381 381 + ) 382 382 + output_group.add_argument( 383 383 + "--cephfs-data-pool-name", 384 384 + default="", 385 385 + help="Provides the name of the cephfs data pool", 386 386 + ) 387 387 + output_group.add_argument( 388 388 + "--rbd-data-pool-name", 389 389 + default="", 390 390 + required=False, 391 391 + help="Provides the name of the RBD datapool", 392 392 + ) 393 393 + output_group.add_argument( 394 394 + "--alias-rbd-data-pool-name", 395 395 + default="", 396 396 + required=False, 397 397 + help="Provides an alias for the RBD data pool name, necessary if a special character is present in the pool name such as a period or underscore", 398 398 + ) 399 399 + output_group.add_argument( 400 400 + "--rgw-endpoint", 401 401 + default="", 402 402 + required=False, 403 403 + help="RADOS Gateway endpoint (in `<IPv4>:<PORT>` or `<[IPv6]>:<PORT>` or `<FQDN>:<PORT>` format)", 404 404 + ) 405 405 + output_group.add_argument( 406 406 + "--rgw-tls-cert-path", 407 407 + default="", 408 408 + required=False, 409 409 + help="RADOS Gateway endpoint TLS certificate", 410 410 + ) 411 411 + output_group.add_argument( 412 412 + "--rgw-skip-tls", 413 413 + required=False, 414 414 + default=False, 415 415 + help="Ignore TLS certification validation when a self-signed certificate is provided (NOT RECOMMENDED", 416 416 + ) 417 417 + output_group.add_argument( 418 418 + "--monitoring-endpoint", 419 419 + default="", 420 420 + required=False, 421 421 + help="Ceph Manager prometheus exporter endpoints (comma separated list of (format `<IPv4>` or `<[IPv6]>` or `<FQDN>`) entries of active and standby mgrs)", 422 422 + ) 423 423 + output_group.add_argument( 424 424 + "--monitoring-endpoint-port", 425 425 + default="", 426 426 + required=False, 427 427 + help="Ceph Manager prometheus exporter port", 428 428 + ) 429 429 + output_group.add_argument( 430 430 + "--skip-monitoring-endpoint", 431 431 + default=False, 432 432 + action="store_true", 433 433 + help="Do not check for a monitoring endpoint for the Ceph cluster", 434 434 + ) 435 435 + output_group.add_argument( 436 436 + "--rbd-metadata-ec-pool-name", 437 437 + default="", 438 438 + required=False, 439 439 + help="Provides the name of erasure coded RBD metadata pool", 440 440 + ) 441 441 + output_group.add_argument( 442 442 + "--dry-run", 443 443 + default=False, 444 444 + action="store_true", 445 445 + help="Dry run prints the executed commands without running them", 446 446 + ) 447 447 + output_group.add_argument( 448 448 + "--rados-namespace", 449 449 + default="", 450 450 + required=False, 451 451 + help="Divides a pool into separate logical namespaces, used for creating RBD PVC in a CephBlockPoolRadosNamespace (should be lower case)", 452 452 + ) 453 453 + output_group.add_argument( 454 454 + "--subvolume-group", 455 455 + default="", 456 456 + required=False, 457 457 + help="provides the name of the subvolume group", 458 458 + ) 459 459 + output_group.add_argument( 460 460 + "--rgw-realm-name", 461 461 + default="", 462 462 + required=False, 463 463 + help="provides the name of the rgw-realm", 464 464 + ) 465 465 + output_group.add_argument( 466 466 + "--rgw-zone-name", 467 467 + default="", 468 468 + required=False, 469 469 + help="provides the name of the rgw-zone", 470 470 + ) 471 471 + output_group.add_argument( 472 472 + "--rgw-zonegroup-name", 473 473 + default="", 474 474 + required=False, 475 475 + help="provides the name of the rgw-zonegroup", 476 476 + ) 477 477 + output_group.add_argument( 478 478 + "--topology-pools", 479 479 + default="", 480 480 + required=False, 481 481 + help="comma-separated list of topology-constrained rbd pools", 482 482 + ) 483 483 + output_group.add_argument( 484 484 + "--topology-failure-domain-label", 485 485 + default="", 486 486 + required=False, 487 487 + help="k8s cluster failure domain label (example: zone, rack, or host) for the topology-pools that match the ceph domain", 488 488 + ) 489 489 + output_group.add_argument( 490 490 + "--topology-failure-domain-values", 491 491 + default="", 492 492 + required=False, 493 493 + help="comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the `topology-pools` list", 494 494 + ) 495 495 + 496 496 + upgrade_group = argP.add_argument_group("upgrade") 497 497 + upgrade_group.add_argument( 498 498 + "--upgrade", 499 499 + action="store_true", 500 500 + default=False, 501 501 + help="Upgrades the cephCSIKeyrings(For example: client.csi-cephfs-provisioner) and client.healthchecker ceph users with new permissions needed for the new cluster version and older permission will still be applied." 502 502 + + "Sample run: `python3 /etc/ceph/create-external-cluster-resources.py --upgrade`, this will upgrade all the default csi users(non-restricted)" 503 503 + + "For restricted users(For example: client.csi-cephfs-provisioner-openshift-storage-myfs), users created using --restricted-auth-permission flag need to pass mandatory flags" 504 504 + + "mandatory flags: '--rbd-data-pool-name, --k8s-cluster-name and --run-as-user' flags while upgrading" 505 505 + + "in case of cephfs users if you have passed --cephfs-filesystem-name flag while creating user then while upgrading it will be mandatory too" 506 506 + + "Sample run: `python3 /etc/ceph/create-external-cluster-resources.py --upgrade --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --run-as-user client.csi-rbd-node-rookstorage-replicapool`" 507 507 + + "PS: An existing non-restricted user cannot be converted to a restricted user by upgrading." 508 508 + + "Upgrade flag should only be used to append new permissions to users, it shouldn't be used for changing user already applied permission, for example you shouldn't change in which pool user has access", 509 509 + ) 510 510 + 511 511 + if args_to_parse: 512 512 + assert ( 513 513 + type(args_to_parse) == list 514 514 + ), "Argument to 'gen_arg_parser' should be a list" 515 515 + else: 516 516 + args_to_parse = sys.argv[1:] 517 517 + return argP.parse_args(args_to_parse) 518 518 + 519 519 + def validate_rbd_metadata_ec_pool_name(self): 520 520 + if self._arg_parser.rbd_metadata_ec_pool_name: 521 521 + rbd_metadata_ec_pool_name = self._arg_parser.rbd_metadata_ec_pool_name 522 522 + rbd_pool_name = self._arg_parser.rbd_data_pool_name 523 523 + 524 524 + if rbd_pool_name == "": 525 525 + raise ExecutionFailureException( 526 526 + "Flag '--rbd-data-pool-name' should not be empty" 527 527 + ) 528 528 + 529 529 + if rbd_metadata_ec_pool_name == "": 530 530 + raise ExecutionFailureException( 531 531 + "Flag '--rbd-metadata-ec-pool-name' should not be empty" 532 532 + ) 533 533 + 534 534 + cmd_json = {"prefix": "osd dump", "format": "json"} 535 535 + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) 536 536 + if ret_val != 0 or len(json_out) == 0: 537 537 + raise ExecutionFailureException( 538 538 + f"{cmd_json['prefix']} command failed.\n" 539 539 + f"Error: {err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST}" 540 540 + ) 541 541 + metadata_pool_exist, pool_exist = False, False 542 542 + 543 543 + for key in json_out["pools"]: 544 544 + # if erasure_code_profile is empty and pool name exists then it replica pool 545 545 + if ( 546 546 + key["erasure_code_profile"] == "" 547 547 + and key["pool_name"] == rbd_metadata_ec_pool_name 548 548 + ): 549 549 + metadata_pool_exist = True 550 550 + # if erasure_code_profile is not empty and pool name exists then it is ec pool 551 551 + if key["erasure_code_profile"] and key["pool_name"] == rbd_pool_name: 552 552 + pool_exist = True 553 553 + 554 554 + if not metadata_pool_exist: 555 555 + raise ExecutionFailureException( 556 556 + "Provided rbd_ec_metadata_pool name," 557 557 + f" {rbd_metadata_ec_pool_name}, does not exist" 558 558 + ) 559 559 + if not pool_exist: 560 560 + raise ExecutionFailureException( 561 561 + f"Provided rbd_data_pool name, {rbd_pool_name}, does not exist" 562 562 + ) 563 563 + return rbd_metadata_ec_pool_name 564 564 + 565 565 + def dry_run(self, msg): 566 566 + if self._arg_parser.dry_run: 567 567 + print("Execute: " + "'" + msg + "'") 568 568 + 569 569 + def validate_rgw_endpoint_tls_cert(self): 570 570 + if self._arg_parser.rgw_tls_cert_path: 571 571 + with open(self._arg_parser.rgw_tls_cert_path, encoding="utf8") as f: 572 572 + contents = f.read() 573 573 + return contents.rstrip() 574 574 + 575 575 + def _check_conflicting_options(self): 576 576 + if not self._arg_parser.upgrade and not self._arg_parser.rbd_data_pool_name: 577 577 + raise ExecutionFailureException( 578 578 + "Either '--upgrade' or '--rbd-data-pool-name <pool_name>' should be specified" 579 579 + ) 580 580 + 581 581 + def _invalid_endpoint(self, endpoint_str): 582 582 + # extract the port by getting the last split on `:` delimiter 583 583 + try: 584 584 + endpoint_str_ip, port = endpoint_str.rsplit(":", 1) 585 585 + except ValueError: 586 586 + raise ExecutionFailureException(f"Not a proper endpoint: {endpoint_str}") 587 587 + 588 588 + try: 589 589 + if endpoint_str_ip[0] == "[": 590 590 + endpoint_str_ip = endpoint_str_ip[1 : len(endpoint_str_ip) - 1] 591 591 + ip_type = ( 592 592 + "IPv4" if type(ip_address(endpoint_str_ip)) is IPv4Address else "IPv6" 593 593 + ) 594 594 + except ValueError: 595 595 + ip_type = "FQDN" 596 596 + if not port.isdigit(): 597 597 + raise ExecutionFailureException(f"Port not valid: {port}") 598 598 + intPort = int(port) 599 599 + if intPort < 1 or intPort > 2**16 - 1: 600 600 + raise ExecutionFailureException(f"Out of range port number: {port}") 601 601 + 602 602 + return ip_type 603 603 + 604 604 + def endpoint_dial(self, endpoint_str, ip_type, timeout=3, cert=None): 605 605 + # if the 'cluster' instance is a dummy one, 606 606 + # don't try to reach out to the endpoint 607 607 + if isinstance(self.cluster, DummyRados): 608 608 + return "", "", "" 609 609 + if ip_type == "IPv6": 610 610 + try: 611 611 + endpoint_str_ip, endpoint_str_port = endpoint_str.rsplit(":", 1) 612 612 + except ValueError: 613 613 + raise ExecutionFailureException( 614 614 + f"Not a proper endpoint: {endpoint_str}" 615 615 + ) 616 616 + if endpoint_str_ip[0] != "[": 617 617 + endpoint_str_ip = "[" + endpoint_str_ip + "]" 618 618 + endpoint_str = ":".join([endpoint_str_ip, endpoint_str_port]) 619 619 + 620 620 + protocols = ["http", "https"] 621 621 + response_error = None 622 622 + for prefix in protocols: 623 623 + try: 624 624 + ep = f"{prefix}://{endpoint_str}" 625 625 + verify = None 626 626 + # If verify is set to a path to a directory, 627 627 + # the directory must have been processed using the c_rehash utility supplied with OpenSSL. 628 628 + if prefix == "https" and self._arg_parser.rgw_skip_tls: 629 629 + verify = False 630 630 + r = requests.head(ep, timeout=timeout, verify=False) 631 631 + elif prefix == "https" and cert: 632 632 + verify = cert 633 633 + r = requests.head(ep, timeout=timeout, verify=cert) 634 634 + else: 635 635 + r = requests.head(ep, timeout=timeout) 636 636 + if r.status_code == 200: 637 637 + return prefix, verify, "" 638 638 + except Exception as err: 639 639 + response_error = err 640 640 + continue 641 641 + sys.stderr.write( 642 642 + f"unable to connect to endpoint: {endpoint_str}, failed error: {response_error}" 643 643 + ) 644 644 + return ( 645 645 + "", 646 646 + "", 647 647 + ("-1"), 648 648 + ) 649 649 + 650 650 + def __init__(self, arg_list=None): 651 651 + self.out_map = {} 652 652 + self._excluded_keys = set() 653 653 + self._arg_parser = self.gen_arg_parser(args_to_parse=arg_list) 654 654 + self._check_conflicting_options() 655 655 + self.run_as_user = self._arg_parser.run_as_user 656 656 + self.output_file = self._arg_parser.output 657 657 + self.ceph_conf = self._arg_parser.ceph_conf 658 658 + self.ceph_keyring = self._arg_parser.keyring 659 659 + # if user not provided, give a default user 660 660 + if not self.run_as_user and not self._arg_parser.upgrade: 661 661 + self.run_as_user = self.EXTERNAL_USER_NAME 662 662 + if not self._arg_parser.rgw_pool_prefix and not self._arg_parser.upgrade: 663 663 + self._arg_parser.rgw_pool_prefix = self.DEFAULT_RGW_POOL_PREFIX 664 664 + if self.ceph_conf: 665 665 + kwargs = {} 666 666 + if self.ceph_keyring: 667 667 + kwargs["conf"] = {"keyring": self.ceph_keyring} 668 668 + self.cluster = rados.Rados(conffile=self.ceph_conf, **kwargs) 669 669 + else: 670 670 + self.cluster = rados.Rados() 671 671 + self.cluster.conf_read_file() 672 672 + self.cluster.connect() 673 673 + 674 674 + def shutdown(self): 675 675 + if self.cluster.state == "connected": 676 676 + self.cluster.shutdown() 677 677 + 678 678 + def get_fsid(self): 679 679 + if self._arg_parser.dry_run: 680 680 + return self.dry_run("ceph fsid") 681 681 + return str(self.cluster.get_fsid()) 682 682 + 683 683 + def _common_cmd_json_gen(self, cmd_json): 684 684 + cmd = json.dumps(cmd_json, sort_keys=True) 685 685 + ret_val, cmd_out, err_msg = self.cluster.mon_command(cmd, b"") 686 686 + if self._arg_parser.verbose: 687 687 + print(f"Command Input: {cmd}") 688 688 + print( 689 689 + f"Return Val: {ret_val}\nCommand Output: {cmd_out}\n" 690 690 + f"Error Message: {err_msg}\n----------\n" 691 691 + ) 692 692 + json_out = {} 693 693 + # if there is no error (i.e; ret_val is ZERO) and 'cmd_out' is not empty 694 694 + # then convert 'cmd_out' to a json output 695 695 + if ret_val == 0 and cmd_out: 696 696 + json_out = json.loads(cmd_out) 697 697 + return ret_val, json_out, err_msg 698 698 + 699 699 + def get_ceph_external_mon_data(self): 700 700 + cmd_json = {"prefix": "quorum_status", "format": "json"} 701 701 + if self._arg_parser.dry_run: 702 702 + return self.dry_run("ceph " + cmd_json["prefix"]) 703 703 + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) 704 704 + # if there is an unsuccessful attempt, 705 705 + if ret_val != 0 or len(json_out) == 0: 706 706 + raise ExecutionFailureException( 707 707 + "'quorum_status' command failed.\n" 708 708 + f"Error: {err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST}" 709 709 + ) 710 710 + q_leader_name = json_out["quorum_leader_name"] 711 711 + q_leader_details = {} 712 712 + q_leader_matching_list = [ 713 713 + l for l in json_out["monmap"]["mons"] if l["name"] == q_leader_name 714 714 + ] 715 715 + if len(q_leader_matching_list) == 0: 716 716 + raise ExecutionFailureException("No matching 'mon' details found") 717 717 + q_leader_details = q_leader_matching_list[0] 718 718 + # get the address vector of the quorum-leader 719 719 + q_leader_addrvec = q_leader_details.get("public_addrs", {}).get("addrvec", []) 720 720 + ip_addr = str(q_leader_details["public_addr"].split("/")[0]) 721 721 + 722 722 + if self._arg_parser.v2_port_enable: 723 723 + if q_leader_addrvec[0]["type"] == "v2": 724 724 + ip_addr = q_leader_addrvec[0]["addr"] 725 725 + elif len(q_leader_addrvec) > 1 and q_leader_addrvec[1]["type"] == "v2": 726 726 + ip_addr = q_leader_addrvec[1]["addr"] 727 727 + else: 728 728 + sys.stderr.write( 729 729 + "'v2' address type not present, and 'v2-port-enable' flag is provided" 730 730 + ) 731 731 + 732 732 + return f"{str(q_leader_name)}={ip_addr}" 733 733 + 734 734 + def _convert_hostname_to_ip(self, host_name, port, ip_type): 735 735 + # if 'cluster' instance is a dummy type, 736 736 + # call the dummy instance's "convert" method 737 737 + if not host_name: 738 738 + raise ExecutionFailureException("Empty hostname provided") 739 739 + if isinstance(self.cluster, DummyRados): 740 740 + return self.cluster._convert_hostname_to_ip(host_name) 741 741 + 742 742 + if ip_type == "FQDN": 743 743 + # check which ip FQDN should be converted to, IPv4 or IPv6 744 744 + # check the host ip, the endpoint ip type would be similar to host ip 745 745 + cmd_json = {"prefix": "orch host ls", "format": "json"} 746 746 + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) 747 747 + # if there is an unsuccessful attempt, 748 748 + if ret_val != 0 or len(json_out) == 0: 749 749 + raise ExecutionFailureException( 750 750 + "'orch host ls' command failed.\n" 751 751 + f"Error: {err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST}" 752 752 + ) 753 753 + host_addr = json_out[0]["addr"] 754 754 + # add :80 sample port in ip_type, as _invalid_endpoint also verify port 755 755 + host_ip_type = self._invalid_endpoint(host_addr + ":80") 756 756 + import socket 757 757 + 758 758 + # example output [(<AddressFamily.AF_INET: 2>, <SocketKind.SOCK_STREAM: 1>, 6, '', ('93.184.216.34', 80)), ...] 759 759 + # we need to get 93.184.216.34 so it would be ip[0][4][0] 760 760 + if host_ip_type == "IPv6": 761 761 + ip = socket.getaddrinfo( 762 762 + host_name, port, family=socket.AF_INET6, proto=socket.IPPROTO_TCP 763 763 + ) 764 764 + elif host_ip_type == "IPv4": 765 765 + ip = socket.getaddrinfo( 766 766 + host_name, port, family=socket.AF_INET, proto=socket.IPPROTO_TCP 767 767 + ) 768 768 + del socket 769 769 + return ip[0][4][0] 770 770 + return host_name 771 771 + 772 772 + def get_active_and_standby_mgrs(self): 773 773 + if self._arg_parser.dry_run: 774 774 + return "", self.dry_run("ceph status") 775 775 + monitoring_endpoint_port = self._arg_parser.monitoring_endpoint_port 776 776 + monitoring_endpoint_ip_list = self._arg_parser.monitoring_endpoint 777 777 + standby_mgrs = [] 778 778 + if not monitoring_endpoint_ip_list: 779 779 + cmd_json = {"prefix": "status", "format": "json"} 780 780 + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) 781 781 + # if there is an unsuccessful attempt, 782 782 + if ret_val != 0 or len(json_out) == 0: 783 783 + raise ExecutionFailureException( 784 784 + "'mgr services' command failed.\n" 785 785 + f"Error: {err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST}" 786 786 + ) 787 787 + monitoring_endpoint = ( 788 788 + json_out.get("mgrmap", {}).get("services", {}).get("prometheus", "") 789 789 + ) 790 790 + if not monitoring_endpoint: 791 791 + raise ExecutionFailureException( 792 792 + "can't find monitoring_endpoint, prometheus module might not be enabled, " 793 793 + "enable the module by running 'ceph mgr module enable prometheus'" 794 794 + ) 795 795 + # now check the stand-by mgr-s 796 796 + standby_arr = json_out.get("mgrmap", {}).get("standbys", []) 797 797 + for each_standby in standby_arr: 798 798 + if "name" in each_standby.keys(): 799 799 + standby_mgrs.append(each_standby["name"]) 800 800 + try: 801 801 + parsed_endpoint = urlparse(monitoring_endpoint) 802 802 + except ValueError: 803 803 + raise ExecutionFailureException( 804 804 + f"invalid endpoint: {monitoring_endpoint}" 805 805 + ) 806 806 + monitoring_endpoint_ip_list = parsed_endpoint.hostname 807 807 + if not monitoring_endpoint_port: 808 808 + monitoring_endpoint_port = str(parsed_endpoint.port) 809 809 + 810 810 + # if monitoring endpoint port is not set, put a default mon port 811 811 + if not monitoring_endpoint_port: 812 812 + monitoring_endpoint_port = self.DEFAULT_MONITORING_ENDPOINT_PORT 813 813 + 814 814 + # user could give comma and space separated inputs (like --monitoring-endpoint="<ip1>, <ip2>") 815 815 + monitoring_endpoint_ip_list = monitoring_endpoint_ip_list.replace(",", " ") 816 816 + monitoring_endpoint_ip_list_split = monitoring_endpoint_ip_list.split() 817 817 + # if monitoring-endpoint could not be found, raise an error 818 818 + if len(monitoring_endpoint_ip_list_split) == 0: 819 819 + raise ExecutionFailureException("No 'monitoring-endpoint' found") 820 820 + # first ip is treated as the main monitoring-endpoint 821 821 + monitoring_endpoint_ip = monitoring_endpoint_ip_list_split[0] 822 822 + # rest of the ip-s are added to the 'standby_mgrs' list 823 823 + standby_mgrs.extend(monitoring_endpoint_ip_list_split[1:]) 824 824 + failed_ip = monitoring_endpoint_ip 825 825 + 826 826 + monitoring_endpoint = ":".join( 827 827 + [monitoring_endpoint_ip, monitoring_endpoint_port] 828 828 + ) 829 829 + ip_type = self._invalid_endpoint(monitoring_endpoint) 830 830 + try: 831 831 + monitoring_endpoint_ip = self._convert_hostname_to_ip( 832 832 + monitoring_endpoint_ip, monitoring_endpoint_port, ip_type 833 833 + ) 834 834 + # collect all the 'stand-by' mgr ips 835 835 + mgr_ips = [] 836 836 + for each_standby_mgr in standby_mgrs: 837 837 + failed_ip = each_standby_mgr 838 838 + mgr_ips.append( 839 839 + self._convert_hostname_to_ip( 840 840 + each_standby_mgr, monitoring_endpoint_port, ip_type 841 841 + ) 842 842 + ) 843 843 + except: 844 844 + raise ExecutionFailureException( 845 845 + f"Conversion of host: {failed_ip} to IP failed. " 846 846 + "Please enter the IP addresses of all the ceph-mgrs with the '--monitoring-endpoint' flag" 847 847 + ) 848 848 + 849 849 + _, _, err = self.endpoint_dial(monitoring_endpoint, ip_type) 850 850 + if err == "-1": 851 851 + raise ExecutionFailureException(err) 852 852 + # add the validated active mgr IP into the first index 853 853 + mgr_ips.insert(0, monitoring_endpoint_ip) 854 854 + all_mgr_ips_str = ",".join(mgr_ips) 855 855 + return all_mgr_ips_str, monitoring_endpoint_port 856 856 + 857 857 + def check_user_exist(self, user): 858 858 + cmd_json = {"prefix": "auth get", "entity": f"{user}", "format": "json"} 859 859 + ret_val, json_out, _ = self._common_cmd_json_gen(cmd_json) 860 860 + if ret_val != 0 or len(json_out) == 0: 861 861 + return "" 862 862 + return str(json_out[0]["key"]) 863 863 + 864 864 + def get_cephfs_provisioner_caps_and_entity(self): 865 865 + entity = "client.csi-cephfs-provisioner" 866 866 + caps = { 867 867 + "mon": "allow r, allow command 'osd blocklist'", 868 868 + "mgr": "allow rw", 869 869 + "osd": "allow rw tag cephfs metadata=*", 870 870 + } 871 871 + if self._arg_parser.restricted_auth_permission: 872 872 + k8s_cluster_name = self._arg_parser.k8s_cluster_name 873 873 + if k8s_cluster_name == "": 874 874 + raise ExecutionFailureException( 875 875 + "k8s_cluster_name not found, please set the '--k8s-cluster-name' flag" 876 876 + ) 877 877 + cephfs_filesystem = self._arg_parser.cephfs_filesystem_name 878 878 + if cephfs_filesystem == "": 879 879 + entity = f"{entity}-{k8s_cluster_name}" 880 880 + else: 881 881 + entity = f"{entity}-{k8s_cluster_name}-{cephfs_filesystem}" 882 882 + caps["osd"] = f"allow rw tag cephfs metadata={cephfs_filesystem}" 883 883 + 884 884 + return caps, entity 885 885 + 886 886 + def get_cephfs_node_caps_and_entity(self): 887 887 + entity = "client.csi-cephfs-node" 888 888 + caps = { 889 889 + "mon": "allow r, allow command 'osd blocklist'", 890 890 + "mgr": "allow rw", 891 891 + "osd": "allow rw tag cephfs *=*", 892 892 + "mds": "allow rw", 893 893 + } 894 894 + if self._arg_parser.restricted_auth_permission: 895 895 + k8s_cluster_name = self._arg_parser.k8s_cluster_name 896 896 + if k8s_cluster_name == "": 897 897 + raise ExecutionFailureException( 898 898 + "k8s_cluster_name not found, please set the '--k8s-cluster-name' flag" 899 899 + ) 900 900 + cephfs_filesystem = self._arg_parser.cephfs_filesystem_name 901 901 + if cephfs_filesystem == "": 902 902 + entity = f"{entity}-{k8s_cluster_name}" 903 903 + else: 904 904 + entity = f"{entity}-{k8s_cluster_name}-{cephfs_filesystem}" 905 905 + caps["osd"] = f"allow rw tag cephfs *={cephfs_filesystem}" 906 906 + 907 907 + return caps, entity 908 908 + 909 909 + def get_entity( 910 910 + self, 911 911 + entity, 912 912 + rbd_pool_name, 913 913 + alias_rbd_pool_name, 914 914 + k8s_cluster_name, 915 915 + rados_namespace, 916 916 + ): 917 917 + if ( 918 918 + rbd_pool_name.count(".") != 0 919 919 + or rbd_pool_name.count("_") != 0 920 920 + or alias_rbd_pool_name != "" 921 921 + # checking alias_rbd_pool_name is not empty as there maybe a special character used other than . or _ 922 922 + ): 923 923 + if alias_rbd_pool_name == "": 924 924 + raise ExecutionFailureException( 925 925 + "please set the '--alias-rbd-data-pool-name' flag as the rbd data pool name contains '.' or '_'" 926 926 + ) 927 927 + if ( 928 928 + alias_rbd_pool_name.count(".") != 0 929 929 + or alias_rbd_pool_name.count("_") != 0 930 930 + ): 931 931 + raise ExecutionFailureException( 932 932 + "'--alias-rbd-data-pool-name' flag value should not contain '.' or '_'" 933 933 + ) 934 934 + entity = f"{entity}-{k8s_cluster_name}-{alias_rbd_pool_name}" 935 935 + else: 936 936 + entity = f"{entity}-{k8s_cluster_name}-{rbd_pool_name}" 937 937 + 938 938 + if rados_namespace: 939 939 + entity = f"{entity}-{rados_namespace}" 940 940 + return entity 941 941 + 942 942 + def get_rbd_provisioner_caps_and_entity(self): 943 943 + entity = "client.csi-rbd-provisioner" 944 944 + caps = { 945 945 + "mon": "profile rbd, allow command 'osd blocklist'", 946 946 + "mgr": "allow rw", 947 947 + "osd": "profile rbd", 948 948 + } 949 949 + if self._arg_parser.restricted_auth_permission: 950 950 + rbd_pool_name = self._arg_parser.rbd_data_pool_name 951 951 + alias_rbd_pool_name = self._arg_parser.alias_rbd_data_pool_name 952 952 + k8s_cluster_name = self._arg_parser.k8s_cluster_name 953 953 + rados_namespace = self._arg_parser.rados_namespace 954 954 + if rbd_pool_name == "": 955 955 + raise ExecutionFailureException( 956 956 + "mandatory flag not found, please set the '--rbd-data-pool-name' flag" 957 957 + ) 958 958 + if k8s_cluster_name == "": 959 959 + raise ExecutionFailureException( 960 960 + "mandatory flag not found, please set the '--k8s-cluster-name' flag" 961 961 + ) 962 962 + entity = self.get_entity( 963 963 + entity, 964 964 + rbd_pool_name, 965 965 + alias_rbd_pool_name, 966 966 + k8s_cluster_name, 967 967 + rados_namespace, 968 968 + ) 969 969 + if rados_namespace != "": 970 970 + caps["osd"] = ( 971 971 + f"profile rbd pool={rbd_pool_name} namespace={rados_namespace}" 972 972 + ) 973 973 + else: 974 974 + caps["osd"] = f"profile rbd pool={rbd_pool_name}" 975 975 + 976 976 + return caps, entity 977 977 + 978 978 + def get_rbd_node_caps_and_entity(self): 979 979 + entity = "client.csi-rbd-node" 980 980 + caps = { 981 981 + "mon": "profile rbd, allow command 'osd blocklist'", 982 982 + "osd": "profile rbd", 983 983 + } 984 984 + if self._arg_parser.restricted_auth_permission: 985 985 + rbd_pool_name = self._arg_parser.rbd_data_pool_name 986 986 + alias_rbd_pool_name = self._arg_parser.alias_rbd_data_pool_name 987 987 + k8s_cluster_name = self._arg_parser.k8s_cluster_name 988 988 + rados_namespace = self._arg_parser.rados_namespace 989 989 + if rbd_pool_name == "": 990 990 + raise ExecutionFailureException( 991 991 + "mandatory flag not found, please set the '--rbd-data-pool-name' flag" 992 992 + ) 993 993 + if k8s_cluster_name == "": 994 994 + raise ExecutionFailureException( 995 995 + "mandatory flag not found, please set the '--k8s-cluster-name' flag" 996 996 + ) 997 997 + entity = self.get_entity( 998 998 + entity, 999 999 + rbd_pool_name, 1000 1000 + alias_rbd_pool_name, 1001 1001 + k8s_cluster_name, 1002 1002 + rados_namespace, 1003 1003 + ) 1004 1004 + if rados_namespace != "": 1005 1005 + caps["osd"] = ( 1006 1006 + f"profile rbd pool={rbd_pool_name} namespace={rados_namespace}" 1007 1007 + ) 1008 1008 + else: 1009 1009 + caps["osd"] = f"profile rbd pool={rbd_pool_name}" 1010 1010 + 1011 1011 + return caps, entity 1012 1012 + 1013 1013 + def get_defaultUser_caps_and_entity(self): 1014 1014 + entity = self.run_as_user 1015 1015 + caps = { 1016 1016 + "mon": "allow r, allow command quorum_status, allow command version", 1017 1017 + "mgr": "allow command config", 1018 1018 + "osd": f"profile rbd-read-only, allow rwx pool={self._arg_parser.rgw_pool_prefix}.rgw.meta, allow r pool=.rgw.root, allow rw pool={self._arg_parser.rgw_pool_prefix}.rgw.control, allow rx pool={self._arg_parser.rgw_pool_prefix}.rgw.log, allow x pool={self._arg_parser.rgw_pool_prefix}.rgw.buckets.index", 1019 1019 + } 1020 1020 + 1021 1021 + return caps, entity 1022 1022 + 1023 1023 + def get_caps_and_entity(self, user_name): 1024 1024 + if "client.csi-cephfs-provisioner" in user_name: 1025 1025 + if "client.csi-cephfs-provisioner" != user_name: 1026 1026 + self._arg_parser.restricted_auth_permission = True 1027 1027 + return self.get_cephfs_provisioner_caps_and_entity() 1028 1028 + if "client.csi-cephfs-node" in user_name: 1029 1029 + if "client.csi-cephfs-node" != user_name: 1030 1030 + self._arg_parser.restricted_auth_permission = True 1031 1031 + return self.get_cephfs_node_caps_and_entity() 1032 1032 + if "client.csi-rbd-provisioner" in user_name: 1033 1033 + if "client.csi-rbd-provisioner" != user_name: 1034 1034 + self._arg_parser.restricted_auth_permission = True 1035 1035 + return self.get_rbd_provisioner_caps_and_entity() 1036 1036 + if "client.csi-rbd-node" in user_name: 1037 1037 + if "client.csi-rbd-node" != user_name: 1038 1038 + self._arg_parser.restricted_auth_permission = True 1039 1039 + return self.get_rbd_node_caps_and_entity() 1040 1040 + if "client.healthchecker" in user_name: 1041 1041 + if "client.healthchecker" != user_name: 1042 1042 + self._arg_parser.restricted_auth_permission = True 1043 1043 + return self.get_defaultUser_caps_and_entity() 1044 1044 + 1045 1045 + raise ExecutionFailureException( 1046 1046 + f"no user found with user_name: {user_name}, " 1047 1047 + "get_caps_and_entity command failed.\n" 1048 1048 + ) 1049 1049 + 1050 1050 + def create_cephCSIKeyring_user(self, user): 1051 1051 + """ 1052 1052 + command: ceph auth get-or-create client.csi-cephfs-provisioner mon 'allow r' mgr 'allow rw' osd 'allow rw tag cephfs metadata=*' 1053 1053 + """ 1054 1054 + caps, entity = self.get_caps_and_entity(user) 1055 1055 + cmd_json = { 1056 1056 + "prefix": "auth get-or-create", 1057 1057 + "entity": entity, 1058 1058 + "caps": [cap for cap_list in list(caps.items()) for cap in cap_list], 1059 1059 + "format": "json", 1060 1060 + } 1061 1061 + 1062 1062 + if self._arg_parser.dry_run: 1063 1063 + return ( 1064 1064 + self.dry_run( 1065 1065 + "ceph " 1066 1066 + + cmd_json["prefix"] 1067 1067 + + " " 1068 1068 + + cmd_json["entity"] 1069 1069 + + " " 1070 1070 + + " ".join(cmd_json["caps"]) 1071 1071 + ), 1072 1072 + "", 1073 1073 + ) 1074 1074 + # check if user already exist 1075 1075 + user_key = self.check_user_exist(entity) 1076 1076 + if user_key != "": 1077 1077 + return user_key, f"{entity.split('.', 1)[1]}" 1078 1078 + # entity.split('.',1)[1] to rename entity(client.csi-rbd-node) as csi-rbd-node 1079 1079 + 1080 1080 + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) 1081 1081 + # if there is an unsuccessful attempt, 1082 1082 + if ret_val != 0 or len(json_out) == 0: 1083 1083 + raise ExecutionFailureException( 1084 1084 + f"'auth get-or-create {user}' command failed.\n" 1085 1085 + f"Error: {err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST}" 1086 1086 + ) 1087 1087 + return str(json_out[0]["key"]), f"{entity.split('.', 1)[1]}" 1088 1088 + # entity.split('.',1)[1] to rename entity(client.csi-rbd-node) as csi-rbd-node 1089 1089 + 1090 1090 + def get_cephfs_data_pool_details(self): 1091 1091 + cmd_json = {"prefix": "fs ls", "format": "json"} 1092 1092 + if self._arg_parser.dry_run: 1093 1093 + return self.dry_run("ceph " + cmd_json["prefix"]) 1094 1094 + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) 1095 1095 + # if there is an unsuccessful attempt, report an error 1096 1096 + if ret_val != 0: 1097 1097 + # if fs and data_pool arguments are not set, silently return 1098 1098 + if ( 1099 1099 + self._arg_parser.cephfs_filesystem_name == "" 1100 1100 + and self._arg_parser.cephfs_data_pool_name == "" 1101 1101 + ): 1102 1102 + return 1103 1103 + # if user has provided any of the 1104 1104 + # '--cephfs-filesystem-name' or '--cephfs-data-pool-name' arguments, 1105 1105 + # raise an exception as we are unable to verify the args 1106 1106 + raise ExecutionFailureException( 1107 1107 + f"'fs ls' ceph call failed with error: {err_msg}" 1108 1108 + ) 1109 1109 + 1110 1110 + matching_json_out = {} 1111 1111 + # if '--cephfs-filesystem-name' argument is provided, 1112 1112 + # check whether the provided filesystem-name exists or not 1113 1113 + if self._arg_parser.cephfs_filesystem_name: 1114 1114 + # get the matching list 1115 1115 + matching_json_out_list = [ 1116 1116 + matched 1117 1117 + for matched in json_out 1118 1118 + if str(matched["name"]) == self._arg_parser.cephfs_filesystem_name 1119 1119 + ] 1120 1120 + # unable to find a matching fs-name, raise an error 1121 1121 + if len(matching_json_out_list) == 0: 1122 1122 + raise ExecutionFailureException( 1123 1123 + f"Filesystem provided, '{self._arg_parser.cephfs_filesystem_name}', " 1124 1124 + f"is not found in the fs-list: {[str(x['name']) for x in json_out]}" 1125 1125 + ) 1126 1126 + matching_json_out = matching_json_out_list[0] 1127 1127 + # if cephfs filesystem name is not provided, 1128 1128 + # try to get a default fs name by doing the following 1129 1129 + else: 1130 1130 + # a. check if there is only one filesystem is present 1131 1131 + if len(json_out) == 1: 1132 1132 + matching_json_out = json_out[0] 1133 1133 + # b. or else, check if data_pool name is provided 1134 1134 + elif self._arg_parser.cephfs_data_pool_name: 1135 1135 + # and if present, check whether there exists a fs which has the data_pool 1136 1136 + for eachJ in json_out: 1137 1137 + if self._arg_parser.cephfs_data_pool_name in eachJ["data_pools"]: 1138 1138 + matching_json_out = eachJ 1139 1139 + break 1140 1140 + # if there is no matching fs exists, that means provided data_pool name is invalid 1141 1141 + if not matching_json_out: 1142 1142 + raise ExecutionFailureException( 1143 1143 + f"Provided data_pool name, {self._arg_parser.cephfs_data_pool_name}," 1144 1144 + " does not exists" 1145 1145 + ) 1146 1146 + # c. if nothing is set and couldn't find a default, 1147 1147 + else: 1148 1148 + # just return silently 1149 1149 + return 1150 1150 + 1151 1151 + if matching_json_out: 1152 1152 + self._arg_parser.cephfs_filesystem_name = str(matching_json_out["name"]) 1153 1153 + self._arg_parser.cephfs_metadata_pool_name = str( 1154 1154 + matching_json_out["metadata_pool"] 1155 1155 + ) 1156 1156 + 1157 1157 + if isinstance(matching_json_out["data_pools"], list): 1158 1158 + # if the user has already provided data-pool-name, 1159 1159 + # through --cephfs-data-pool-name 1160 1160 + if self._arg_parser.cephfs_data_pool_name: 1161 1161 + # if the provided name is not matching with the one in the list 1162 1162 + if ( 1163 1163 + self._arg_parser.cephfs_data_pool_name 1164 1164 + not in matching_json_out["data_pools"] 1165 1165 + ): 1166 1166 + raise ExecutionFailureException( 1167 1167 + f"Provided data-pool-name: '{self._arg_parser.cephfs_data_pool_name}', " 1168 1168 + "doesn't match from the data-pools list: " 1169 1169 + f"{[str(x) for x in matching_json_out['data_pools']]}" 1170 1170 + ) 1171 1171 + # if data_pool name is not provided, 1172 1172 + # then try to find a default data pool name 1173 1173 + else: 1174 1174 + # if no data_pools exist, silently return 1175 1175 + if len(matching_json_out["data_pools"]) == 0: 1176 1176 + return 1177 1177 + self._arg_parser.cephfs_data_pool_name = str( 1178 1178 + matching_json_out["data_pools"][0] 1179 1179 + ) 1180 1180 + # if there are more than one 'data_pools' exist, 1181 1181 + # then warn the user that we are using the selected name 1182 1182 + if len(matching_json_out["data_pools"]) > 1: 1183 1183 + print( 1184 1184 + "WARNING: Multiple data pools detected: " 1185 1185 + f"{[str(x) for x in matching_json_out['data_pools']]}\n" 1186 1186 + f"Using the data-pool: '{self._arg_parser.cephfs_data_pool_name}'\n" 1187 1187 + ) 1188 1188 + 1189 1189 + def create_checkerKey(self, user): 1190 1190 + caps, entity = self.get_caps_and_entity(user) 1191 1191 + cmd_json = { 1192 1192 + "prefix": "auth get-or-create", 1193 1193 + "entity": entity, 1194 1194 + "caps": [cap for cap_list in list(caps.items()) for cap in cap_list], 1195 1195 + "format": "json", 1196 1196 + } 1197 1197 + 1198 1198 + if self._arg_parser.dry_run: 1199 1199 + return self.dry_run( 1200 1200 + "ceph " 1201 1201 + + cmd_json["prefix"] 1202 1202 + + " " 1203 1203 + + cmd_json["entity"] 1204 1204 + + " " 1205 1205 + + " ".join(cmd_json["caps"]) 1206 1206 + ) 1207 1207 + # check if user already exist 1208 1208 + user_key = self.check_user_exist(entity) 1209 1209 + if user_key != "": 1210 1210 + return user_key 1211 1211 + 1212 1212 + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) 1213 1213 + # if there is an unsuccessful attempt, 1214 1214 + if ret_val != 0 or len(json_out) == 0: 1215 1215 + raise ExecutionFailureException( 1216 1216 + f"'auth get-or-create {self.run_as_user}' command failed\n" 1217 1217 + f"Error: {err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST}" 1218 1218 + ) 1219 1219 + return str(json_out[0]["key"]) 1220 1220 + 1221 1221 + def get_ceph_dashboard_link(self): 1222 1222 + cmd_json = {"prefix": "mgr services", "format": "json"} 1223 1223 + if self._arg_parser.dry_run: 1224 1224 + return self.dry_run("ceph " + cmd_json["prefix"]) 1225 1225 + ret_val, json_out, _ = self._common_cmd_json_gen(cmd_json) 1226 1226 + # if there is an unsuccessful attempt, 1227 1227 + if ret_val != 0 or len(json_out) == 0: 1228 1228 + return None 1229 1229 + if "dashboard" not in json_out: 1230 1230 + return None 1231 1231 + return json_out["dashboard"] 1232 1232 + 1233 1233 + def create_rgw_admin_ops_user(self): 1234 1234 + cmd = [ 1235 1235 + "radosgw-admin", 1236 1236 + "user", 1237 1237 + "create", 1238 1238 + "--uid", 1239 1239 + self.EXTERNAL_RGW_ADMIN_OPS_USER_NAME, 1240 1240 + "--display-name", 1241 1241 + "Rook RGW Admin Ops user", 1242 1242 + "--caps", 1243 1243 + "buckets=*;users=*;usage=read;metadata=read;zone=read", 1244 1244 + "--rgw-realm", 1245 1245 + self._arg_parser.rgw_realm_name, 1246 1246 + "--rgw-zonegroup", 1247 1247 + self._arg_parser.rgw_zonegroup_name, 1248 1248 + "--rgw-zone", 1249 1249 + self._arg_parser.rgw_zone_name, 1250 1250 + ] 1251 1251 + if self._arg_parser.dry_run: 1252 1252 + return self.dry_run("ceph " + " ".join(cmd)) 1253 1253 + try: 1254 1254 + output = subprocess.check_output(cmd, stderr=subprocess.PIPE) 1255 1255 + except subprocess.CalledProcessError as execErr: 1256 1256 + # if the user already exists, we just query it 1257 1257 + if execErr.returncode == errno.EEXIST: 1258 1258 + cmd = [ 1259 1259 + "radosgw-admin", 1260 1260 + "user", 1261 1261 + "info", 1262 1262 + "--uid", 1263 1263 + self.EXTERNAL_RGW_ADMIN_OPS_USER_NAME, 1264 1264 + "--rgw-realm", 1265 1265 + self._arg_parser.rgw_realm_name, 1266 1266 + "--rgw-zonegroup", 1267 1267 + self._arg_parser.rgw_zonegroup_name, 1268 1268 + "--rgw-zone", 1269 1269 + self._arg_parser.rgw_zone_name, 1270 1270 + ] 1271 1271 + try: 1272 1272 + output = subprocess.check_output(cmd, stderr=subprocess.PIPE) 1273 1273 + except subprocess.CalledProcessError as execErr: 1274 1274 + err_msg = ( 1275 1275 + f"failed to execute command {cmd}. Output: {execErr.output}. " 1276 1276 + f"Code: {execErr.returncode}. Error: {execErr.stderr}" 1277 1277 + ) 1278 1278 + sys.stderr.write(err_msg) 1279 1279 + return None, None, False, "-1" 1280 1280 + else: 1281 1281 + err_msg = ( 1282 1282 + f"failed to execute command {cmd}. Output: {execErr.output}. " 1283 1283 + f"Code: {execErr.returncode}. Error: {execErr.stderr}" 1284 1284 + ) 1285 1285 + sys.stderr.write(err_msg) 1286 1286 + return None, None, False, "-1" 1287 1287 + 1288 1288 + # if it is python2, don't check for ceph version for adding `info=read` cap(rgw_validation) 1289 1289 + if sys.version_info.major < 3: 1290 1290 + jsonoutput = json.loads(output) 1291 1291 + return ( 1292 1292 + jsonoutput["keys"][0]["access_key"], 1293 1293 + jsonoutput["keys"][0]["secret_key"], 1294 1294 + False, 1295 1295 + "", 1296 1296 + ) 1297 1297 + 1298 1298 + # separately add info=read caps for rgw-endpoint ip validation 1299 1299 + info_cap_supported = True 1300 1300 + cmd = [ 1301 1301 + "radosgw-admin", 1302 1302 + "caps", 1303 1303 + "add", 1304 1304 + "--uid", 1305 1305 + self.EXTERNAL_RGW_ADMIN_OPS_USER_NAME, 1306 1306 + "--caps", 1307 1307 + "info=read", 1308 1308 + "--rgw-realm", 1309 1309 + self._arg_parser.rgw_realm_name, 1310 1310 + "--rgw-zonegroup", 1311 1311 + self._arg_parser.rgw_zonegroup_name, 1312 1312 + "--rgw-zone", 1313 1313 + self._arg_parser.rgw_zone_name, 1314 1314 + ] 1315 1315 + try: 1316 1316 + output = subprocess.check_output(cmd, stderr=subprocess.PIPE) 1317 1317 + except subprocess.CalledProcessError as execErr: 1318 1318 + # if the ceph version not supported for adding `info=read` cap(rgw_validation) 1319 1319 + if ( 1320 1320 + "could not add caps: unable to add caps: info=read\n" 1321 1321 + in execErr.stderr.decode("utf-8") 1322 1322 + and execErr.returncode == 244 1323 1323 + ): 1324 1324 + info_cap_supported = False 1325 1325 + else: 1326 1326 + err_msg = ( 1327 1327 + f"failed to execute command {cmd}. Output: {execErr.output}. " 1328 1328 + f"Code: {execErr.returncode}. Error: {execErr.stderr}" 1329 1329 + ) 1330 1330 + sys.stderr.write(err_msg) 1331 1331 + return None, None, False, "-1" 1332 1332 + 1333 1333 + jsonoutput = json.loads(output) 1334 1334 + return ( 1335 1335 + jsonoutput["keys"][0]["access_key"], 1336 1336 + jsonoutput["keys"][0]["secret_key"], 1337 1337 + info_cap_supported, 1338 1338 + "", 1339 1339 + ) 1340 1340 + 1341 1341 + def validate_rbd_pool(self, pool_name): 1342 1342 + if not self.cluster.pool_exists(pool_name): 1343 1343 + raise ExecutionFailureException( 1344 1344 + f"The provided pool, '{pool_name}', does not exist" 1345 1345 + ) 1346 1346 + 1347 1347 + def init_rbd_pool(self, rbd_pool_name): 1348 1348 + if isinstance(self.cluster, DummyRados): 1349 1349 + return 1350 1350 + ioctx = self.cluster.open_ioctx(rbd_pool_name) 1351 1351 + rbd_inst = rbd.RBD() 1352 1352 + rbd_inst.pool_init(ioctx, True) 1353 1353 + 1354 1354 + def validate_rados_namespace(self): 1355 1355 + rbd_pool_name = self._arg_parser.rbd_data_pool_name 1356 1356 + rados_namespace = self._arg_parser.rados_namespace 1357 1357 + if rados_namespace == "": 1358 1358 + return 1359 1359 + if rados_namespace.islower() == False: 1360 1360 + raise ExecutionFailureException( 1361 1361 + f"The provided rados Namespace, '{rados_namespace}', " 1362 1362 + f"contains upper case" 1363 1363 + ) 1364 1364 + rbd_inst = rbd.RBD() 1365 1365 + ioctx = self.cluster.open_ioctx(rbd_pool_name) 1366 1366 + if rbd_inst.namespace_exists(ioctx, rados_namespace) is False: 1367 1367 + raise ExecutionFailureException( 1368 1368 + f"The provided rados Namespace, '{rados_namespace}', " 1369 1369 + f"is not found in the pool '{rbd_pool_name}'" 1370 1370 + ) 1371 1371 + 1372 1372 + def get_or_create_subvolume_group(self, subvolume_group, cephfs_filesystem_name): 1373 1373 + cmd = [ 1374 1374 + "ceph", 1375 1375 + "fs", 1376 1376 + "subvolumegroup", 1377 1377 + "getpath", 1378 1378 + cephfs_filesystem_name, 1379 1379 + subvolume_group, 1380 1380 + ] 1381 1381 + try: 1382 1382 + _ = subprocess.check_output(cmd, stderr=subprocess.PIPE) 1383 1383 + except subprocess.CalledProcessError: 1384 1384 + cmd = [ 1385 1385 + "ceph", 1386 1386 + "fs", 1387 1387 + "subvolumegroup", 1388 1388 + "create", 1389 1389 + cephfs_filesystem_name, 1390 1390 + subvolume_group, 1391 1391 + ] 1392 1392 + try: 1393 1393 + _ = subprocess.check_output(cmd, stderr=subprocess.PIPE) 1394 1394 + except subprocess.CalledProcessError: 1395 1395 + raise ExecutionFailureException( 1396 1396 + f"subvolume group {subvolume_group} is not able to get created" 1397 1397 + ) 1398 1398 + 1399 1399 + def pin_subvolume( 1400 1400 + self, subvolume_group, cephfs_filesystem_name, pin_type, pin_setting 1401 1401 + ): 1402 1402 + cmd = [ 1403 1403 + "ceph", 1404 1404 + "fs", 1405 1405 + "subvolumegroup", 1406 1406 + "pin", 1407 1407 + cephfs_filesystem_name, 1408 1408 + subvolume_group, 1409 1409 + pin_type, 1410 1410 + pin_setting, 1411 1411 + ] 1412 1412 + try: 1413 1413 + _ = subprocess.check_output(cmd, stderr=subprocess.PIPE) 1414 1414 + except subprocess.CalledProcessError: 1415 1415 + raise ExecutionFailureException( 1416 1416 + f"subvolume group {subvolume_group} is not able to get pinned" 1417 1417 + ) 1418 1418 + 1419 1419 + def get_rgw_fsid(self, base_url, verify): 1420 1420 + access_key = self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"] 1421 1421 + secret_key = self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"] 1422 1422 + rgw_endpoint = self._arg_parser.rgw_endpoint 1423 1423 + base_url = base_url + "://" + rgw_endpoint + "/admin/info?" 1424 1424 + params = {"format": "json"} 1425 1425 + request_url = base_url + urlencode(params) 1426 1426 + 1427 1427 + try: 1428 1428 + r = requests.get( 1429 1429 + request_url, 1430 1430 + auth=S3Auth(access_key, secret_key, rgw_endpoint), 1431 1431 + verify=verify, 1432 1432 + ) 1433 1433 + except requests.exceptions.Timeout: 1434 1434 + sys.stderr.write( 1435 1435 + f"invalid endpoint:, not able to call admin-ops api{rgw_endpoint}" 1436 1436 + ) 1437 1437 + return "", "-1" 1438 1438 + r1 = r.json() 1439 1439 + if r1 is None or r1.get("info") is None: 1440 1440 + sys.stderr.write( 1441 1441 + f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid." 1442 1442 + ) 1443 1443 + return ( 1444 1444 + "", 1445 1445 + "-1", 1446 1446 + ) 1447 1447 + 1448 1448 + return r1["info"]["storage_backends"][0]["cluster_id"], "" 1449 1449 + 1450 1450 + def validate_rgw_endpoint(self, info_cap_supported): 1451 1451 + # if the 'cluster' instance is a dummy one, 1452 1452 + # don't try to reach out to the endpoint 1453 1453 + if isinstance(self.cluster, DummyRados): 1454 1454 + return 1455 1455 + 1456 1456 + rgw_endpoint = self._arg_parser.rgw_endpoint 1457 1457 + 1458 1458 + # validate rgw endpoint only if ip address is passed 1459 1459 + ip_type = self._invalid_endpoint(rgw_endpoint) 1460 1460 + 1461 1461 + # check if the rgw endpoint is reachable 1462 1462 + cert = None 1463 1463 + if not self._arg_parser.rgw_skip_tls and self.validate_rgw_endpoint_tls_cert(): 1464 1464 + cert = self._arg_parser.rgw_tls_cert_path 1465 1465 + base_url, verify, err = self.endpoint_dial(rgw_endpoint, ip_type, cert=cert) 1466 1466 + if err != "": 1467 1467 + return "-1" 1468 1468 + 1469 1469 + # check if the rgw endpoint belongs to the same cluster 1470 1470 + # only check if `info` cap is supported 1471 1471 + if info_cap_supported: 1472 1472 + fsid = self.get_fsid() 1473 1473 + rgw_fsid, err = self.get_rgw_fsid(base_url, verify) 1474 1474 + if err == "-1": 1475 1475 + return "-1" 1476 1476 + if fsid != rgw_fsid: 1477 1477 + sys.stderr.write( 1478 1478 + f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid. We are validating by calling the adminops api through rgw-endpoint and validating the cluster_id '{rgw_fsid}' is equal to the ceph cluster fsid '{fsid}'" 1479 1479 + ) 1480 1480 + return "-1" 1481 1481 + 1482 1482 + # check if the rgw endpoint pool exist 1483 1483 + # only validate if rgw_pool_prefix is passed else it will take default value and we don't create these default pools 1484 1484 + if self._arg_parser.rgw_pool_prefix != "default": 1485 1485 + rgw_pools_to_validate = [ 1486 1486 + f"{self._arg_parser.rgw_pool_prefix}.rgw.meta", 1487 1487 + ".rgw.root", 1488 1488 + f"{self._arg_parser.rgw_pool_prefix}.rgw.control", 1489 1489 + f"{self._arg_parser.rgw_pool_prefix}.rgw.log", 1490 1490 + ] 1491 1491 + for _rgw_pool_to_validate in rgw_pools_to_validate: 1492 1492 + if not self.cluster.pool_exists(_rgw_pool_to_validate): 1493 1493 + sys.stderr.write( 1494 1494 + f"The provided pool, '{_rgw_pool_to_validate}', does not exist" 1495 1495 + ) 1496 1496 + return "-1" 1497 1497 + 1498 1498 + return "" 1499 1499 + 1500 1500 + def validate_rgw_multisite(self, rgw_multisite_config_name, rgw_multisite_config): 1501 1501 + if rgw_multisite_config != "": 1502 1502 + cmd = [ 1503 1503 + "radosgw-admin", 1504 1504 + rgw_multisite_config, 1505 1505 + "get", 1506 1506 + "--rgw-" + rgw_multisite_config, 1507 1507 + rgw_multisite_config_name, 1508 1508 + ] 1509 1509 + try: 1510 1510 + _ = subprocess.check_output(cmd, stderr=subprocess.PIPE) 1511 1511 + except subprocess.CalledProcessError as execErr: 1512 1512 + err_msg = ( 1513 1513 + f"failed to execute command {cmd}. Output: {execErr.output}. " 1514 1514 + f"Code: {execErr.returncode}. Error: {execErr.stderr}" 1515 1515 + ) 1516 1516 + sys.stderr.write(err_msg) 1517 1517 + return "-1" 1518 1518 + return "" 1519 1519 + 1520 1520 + def convert_comma_separated_to_array(self, value): 1521 1521 + return value.split(",") 1522 1522 + 1523 1523 + def raise_exception_if_any_topology_flag_is_missing(self): 1524 1524 + if ( 1525 1525 + ( 1526 1526 + self._arg_parser.topology_pools != "" 1527 1527 + and ( 1528 1528 + self._arg_parser.topology_failure_domain_label == "" 1529 1529 + or self._arg_parser.topology_failure_domain_values == "" 1530 1530 + ) 1531 1531 + ) 1532 1532 + or ( 1533 1533 + self._arg_parser.topology_failure_domain_label != "" 1534 1534 + and ( 1535 1535 + self._arg_parser.topology_pools == "" 1536 1536 + or self._arg_parser.topology_failure_domain_values == "" 1537 1537 + ) 1538 1538 + ) 1539 1539 + or ( 1540 1540 + self._arg_parser.topology_failure_domain_values != "" 1541 1541 + and ( 1542 1542 + self._arg_parser.topology_pools == "" 1543 1543 + or self._arg_parser.topology_failure_domain_label == "" 1544 1544 + ) 1545 1545 + ) 1546 1546 + ): 1547 1547 + raise ExecutionFailureException( 1548 1548 + "provide all the topology flags --topology-pools, --topology-failure-domain-label, --topology-failure-domain-values" 1549 1549 + ) 1550 1550 + 1551 1551 + def validate_topology_values(self, topology_pools, topology_fd): 1552 1552 + if len(topology_pools) != len(topology_fd): 1553 1553 + raise ExecutionFailureException( 1554 1554 + f"The provided topology pools, '{topology_pools}', and " 1555 1555 + f"topology failure domain, '{topology_fd}'," 1556 1556 + f"are of different length, '{len(topology_pools)}' and '{len(topology_fd)}' respctively" 1557 1557 + ) 1558 1558 + return 1559 1559 + 1560 1560 + def validate_topology_rbd_pools(self, topology_rbd_pools): 1561 1561 + for pool in topology_rbd_pools: 1562 1562 + self.validate_rbd_pool(pool) 1563 1563 + 1564 1564 + def init_topology_rbd_pools(self, topology_rbd_pools): 1565 1565 + for pool in topology_rbd_pools: 1566 1566 + self.init_rbd_pool(pool) 1567 1567 + 1568 1568 + def _gen_output_map(self): 1569 1569 + if self.out_map: 1570 1570 + return 1571 1571 + # support legacy flag with upgrades 1572 1572 + if self._arg_parser.cluster_name: 1573 1573 + self._arg_parser.k8s_cluster_name = self._arg_parser.cluster_name 1574 1574 + self._arg_parser.k8s_cluster_name = ( 1575 1575 + self._arg_parser.k8s_cluster_name.lower() 1576 1576 + ) # always convert cluster name to lowercase characters 1577 1577 + self.validate_rbd_pool(self._arg_parser.rbd_data_pool_name) 1578 1578 + self.init_rbd_pool(self._arg_parser.rbd_data_pool_name) 1579 1579 + self.validate_rados_namespace() 1580 1580 + self._excluded_keys.add("K8S_CLUSTER_NAME") 1581 1581 + self.get_cephfs_data_pool_details() 1582 1582 + self.out_map["NAMESPACE"] = self._arg_parser.namespace 1583 1583 + self.out_map["K8S_CLUSTER_NAME"] = self._arg_parser.k8s_cluster_name 1584 1584 + self.out_map["ROOK_EXTERNAL_FSID"] = self.get_fsid() 1585 1585 + self.out_map["ROOK_EXTERNAL_USERNAME"] = self.run_as_user 1586 1586 + self.out_map["ROOK_EXTERNAL_CEPH_MON_DATA"] = self.get_ceph_external_mon_data() 1587 1587 + self.out_map["ROOK_EXTERNAL_USER_SECRET"] = self.create_checkerKey( 1588 1588 + "client.healthchecker" 1589 1589 + ) 1590 1590 + self.out_map["ROOK_EXTERNAL_DASHBOARD_LINK"] = self.get_ceph_dashboard_link() 1591 1591 + ( 1592 1592 + self.out_map["CSI_RBD_NODE_SECRET"], 1593 1593 + self.out_map["CSI_RBD_NODE_SECRET_NAME"], 1594 1594 + ) = self.create_cephCSIKeyring_user("client.csi-rbd-node") 1595 1595 + ( 1596 1596 + self.out_map["CSI_RBD_PROVISIONER_SECRET"], 1597 1597 + self.out_map["CSI_RBD_PROVISIONER_SECRET_NAME"], 1598 1598 + ) = self.create_cephCSIKeyring_user("client.csi-rbd-provisioner") 1599 1599 + self.out_map["CEPHFS_POOL_NAME"] = self._arg_parser.cephfs_data_pool_name 1600 1600 + self.out_map["CEPHFS_METADATA_POOL_NAME"] = ( 1601 1601 + self._arg_parser.cephfs_metadata_pool_name 1602 1602 + ) 1603 1603 + self.out_map["CEPHFS_FS_NAME"] = self._arg_parser.cephfs_filesystem_name 1604 1604 + self.out_map["RESTRICTED_AUTH_PERMISSION"] = ( 1605 1605 + self._arg_parser.restricted_auth_permission 1606 1606 + ) 1607 1607 + self.out_map["RADOS_NAMESPACE"] = self._arg_parser.rados_namespace 1608 1608 + self.out_map["SUBVOLUME_GROUP"] = self._arg_parser.subvolume_group 1609 1609 + self.out_map["CSI_CEPHFS_NODE_SECRET"] = "" 1610 1610 + self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"] = "" 1611 1611 + # create CephFS node and provisioner keyring only when MDS exists 1612 1612 + if self.out_map["CEPHFS_FS_NAME"] and self.out_map["CEPHFS_POOL_NAME"]: 1613 1613 + ( 1614 1614 + self.out_map["CSI_CEPHFS_NODE_SECRET"], 1615 1615 + self.out_map["CSI_CEPHFS_NODE_SECRET_NAME"], 1616 1616 + ) = self.create_cephCSIKeyring_user("client.csi-cephfs-node") 1617 1617 + ( 1618 1618 + self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"], 1619 1619 + self.out_map["CSI_CEPHFS_PROVISIONER_SECRET_NAME"], 1620 1620 + ) = self.create_cephCSIKeyring_user("client.csi-cephfs-provisioner") 1621 1621 + # create the default "csi" subvolumegroup 1622 1622 + self.get_or_create_subvolume_group( 1623 1623 + "csi", self._arg_parser.cephfs_filesystem_name 1624 1624 + ) 1625 1625 + # pin the default "csi" subvolumegroup 1626 1626 + self.pin_subvolume( 1627 1627 + "csi", self._arg_parser.cephfs_filesystem_name, "distributed", "1" 1628 1628 + ) 1629 1629 + if self.out_map["SUBVOLUME_GROUP"]: 1630 1630 + self.get_or_create_subvolume_group( 1631 1631 + self._arg_parser.subvolume_group, 1632 1632 + self._arg_parser.cephfs_filesystem_name, 1633 1633 + ) 1634 1634 + self.pin_subvolume( 1635 1635 + self._arg_parser.subvolume_group, 1636 1636 + self._arg_parser.cephfs_filesystem_name, 1637 1637 + "distributed", 1638 1638 + "1", 1639 1639 + ) 1640 1640 + self.out_map["RGW_TLS_CERT"] = "" 1641 1641 + self.out_map["MONITORING_ENDPOINT"] = "" 1642 1642 + self.out_map["MONITORING_ENDPOINT_PORT"] = "" 1643 1643 + if not self._arg_parser.skip_monitoring_endpoint: 1644 1644 + ( 1645 1645 + self.out_map["MONITORING_ENDPOINT"], 1646 1646 + self.out_map["MONITORING_ENDPOINT_PORT"], 1647 1647 + ) = self.get_active_and_standby_mgrs() 1648 1648 + self.out_map["RBD_POOL_NAME"] = self._arg_parser.rbd_data_pool_name 1649 1649 + self.out_map["RBD_METADATA_EC_POOL_NAME"] = ( 1650 1650 + self.validate_rbd_metadata_ec_pool_name() 1651 1651 + ) 1652 1652 + self.out_map["TOPOLOGY_POOLS"] = self._arg_parser.topology_pools 1653 1653 + self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"] = ( 1654 1654 + self._arg_parser.topology_failure_domain_label 1655 1655 + ) 1656 1656 + self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] = ( 1657 1657 + self._arg_parser.topology_failure_domain_values 1658 1658 + ) 1659 1659 + if ( 1660 1660 + self._arg_parser.topology_pools != "" 1661 1661 + and self._arg_parser.topology_failure_domain_label != "" 1662 1662 + and self._arg_parser.topology_failure_domain_values != "" 1663 1663 + ): 1664 1664 + self.validate_topology_values( 1665 1665 + self.convert_comma_separated_to_array(self.out_map["TOPOLOGY_POOLS"]), 1666 1666 + self.convert_comma_separated_to_array( 1667 1667 + self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] 1668 1668 + ), 1669 1669 + ) 1670 1670 + self.validate_topology_rbd_pools( 1671 1671 + self.convert_comma_separated_to_array(self.out_map["TOPOLOGY_POOLS"]) 1672 1672 + ) 1673 1673 + self.init_topology_rbd_pools( 1674 1674 + self.convert_comma_separated_to_array(self.out_map["TOPOLOGY_POOLS"]) 1675 1675 + ) 1676 1676 + else: 1677 1677 + self.raise_exception_if_any_topology_flag_is_missing() 1678 1678 + 1679 1679 + self.out_map["RGW_POOL_PREFIX"] = self._arg_parser.rgw_pool_prefix 1680 1680 + self.out_map["RGW_ENDPOINT"] = "" 1681 1681 + if self._arg_parser.rgw_endpoint: 1682 1682 + if self._arg_parser.dry_run: 1683 1683 + self.create_rgw_admin_ops_user() 1684 1684 + else: 1685 1685 + if ( 1686 1686 + self._arg_parser.rgw_realm_name != "" 1687 1687 + and self._arg_parser.rgw_zonegroup_name != "" 1688 1688 + and self._arg_parser.rgw_zone_name != "" 1689 1689 + ): 1690 1690 + err = self.validate_rgw_multisite( 1691 1691 + self._arg_parser.rgw_realm_name, "realm" 1692 1692 + ) 1693 1693 + err = self.validate_rgw_multisite( 1694 1694 + self._arg_parser.rgw_zonegroup_name, "zonegroup" 1695 1695 + ) 1696 1696 + err = self.validate_rgw_multisite( 1697 1697 + self._arg_parser.rgw_zone_name, "zone" 1698 1698 + ) 1699 1699 + 1700 1700 + if ( 1701 1701 + self._arg_parser.rgw_realm_name == "" 1702 1702 + and self._arg_parser.rgw_zonegroup_name == "" 1703 1703 + and self._arg_parser.rgw_zone_name == "" 1704 1704 + ) or ( 1705 1705 + self._arg_parser.rgw_realm_name != "" 1706 1706 + and self._arg_parser.rgw_zonegroup_name != "" 1707 1707 + and self._arg_parser.rgw_zone_name != "" 1708 1708 + ): 1709 1709 + ( 1710 1710 + self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"], 1711 1711 + self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"], 1712 1712 + info_cap_supported, 1713 1713 + err, 1714 1714 + ) = self.create_rgw_admin_ops_user() 1715 1715 + err = self.validate_rgw_endpoint(info_cap_supported) 1716 1716 + if self._arg_parser.rgw_tls_cert_path: 1717 1717 + self.out_map["RGW_TLS_CERT"] = ( 1718 1718 + self.validate_rgw_endpoint_tls_cert() 1719 1719 + ) 1720 1720 + # if there is no error, set the RGW_ENDPOINT 1721 1721 + if err != "-1": 1722 1722 + self.out_map["RGW_ENDPOINT"] = self._arg_parser.rgw_endpoint 1723 1723 + else: 1724 1724 + err = "Please provide all the RGW multisite parameters or none of them" 1725 1725 + sys.stderr.write(err) 1726 1726 + 1727 1727 + def gen_shell_out(self): 1728 1728 + self._gen_output_map() 1729 1729 + shOutIO = StringIO() 1730 1730 + for k, v in self.out_map.items(): 1731 1731 + if v and k not in self._excluded_keys: 1732 1732 + shOutIO.write(f"export {k}={v}{LINESEP}") 1733 1733 + shOut = shOutIO.getvalue() 1734 1734 + shOutIO.close() 1735 1735 + return shOut 1736 1736 + 1737 1737 + def gen_json_out(self): 1738 1738 + self._gen_output_map() 1739 1739 + if self._arg_parser.dry_run: 1740 1740 + return "" 1741 1741 + json_out = [ 1742 1742 + { 1743 1743 + "name": "rook-ceph-mon-endpoints", 1744 1744 + "kind": "ConfigMap", 1745 1745 + "data": { 1746 1746 + "data": self.out_map["ROOK_EXTERNAL_CEPH_MON_DATA"], 1747 1747 + "maxMonId": "0", 1748 1748 + "mapping": "{}", 1749 1749 + }, 1750 1750 + }, 1751 1751 + { 1752 1752 + "name": "rook-ceph-mon", 1753 1753 + "kind": "Secret", 1754 1754 + "data": { 1755 1755 + "admin-secret": "admin-secret", 1756 1756 + "fsid": self.out_map["ROOK_EXTERNAL_FSID"], 1757 1757 + "mon-secret": "mon-secret", 1758 1758 + }, 1759 1759 + }, 1760 1760 + { 1761 1761 + "name": "rook-ceph-operator-creds", 1762 1762 + "kind": "Secret", 1763 1763 + "data": { 1764 1764 + "userID": self.out_map["ROOK_EXTERNAL_USERNAME"], 1765 1765 + "userKey": self.out_map["ROOK_EXTERNAL_USER_SECRET"], 1766 1766 + }, 1767 1767 + }, 1768 1768 + ] 1769 1769 + 1770 1770 + # if 'MONITORING_ENDPOINT' exists, then only add 'monitoring-endpoint' to Cluster 1771 1771 + if ( 1772 1772 + self.out_map["MONITORING_ENDPOINT"] 1773 1773 + and self.out_map["MONITORING_ENDPOINT_PORT"] 1774 1774 + ): 1775 1775 + json_out.append( 1776 1776 + { 1777 1777 + "name": "monitoring-endpoint", 1778 1778 + "kind": "CephCluster", 1779 1779 + "data": { 1780 1780 + "MonitoringEndpoint": self.out_map["MONITORING_ENDPOINT"], 1781 1781 + "MonitoringPort": self.out_map["MONITORING_ENDPOINT_PORT"], 1782 1782 + }, 1783 1783 + } 1784 1784 + ) 1785 1785 + 1786 1786 + # if 'CSI_RBD_NODE_SECRET' exists, then only add 'rook-csi-rbd-provisioner' Secret 1787 1787 + if ( 1788 1788 + self.out_map["CSI_RBD_NODE_SECRET"] 1789 1789 + and self.out_map["CSI_RBD_NODE_SECRET_NAME"] 1790 1790 + ): 1791 1791 + json_out.append( 1792 1792 + { 1793 1793 + "name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}", 1794 1794 + "kind": "Secret", 1795 1795 + "data": { 1796 1796 + "userID": self.out_map["CSI_RBD_NODE_SECRET_NAME"], 1797 1797 + "userKey": self.out_map["CSI_RBD_NODE_SECRET"], 1798 1798 + }, 1799 1799 + } 1800 1800 + ) 1801 1801 + # if 'CSI_RBD_PROVISIONER_SECRET' exists, then only add 'rook-csi-rbd-provisioner' Secret 1802 1802 + if ( 1803 1803 + self.out_map["CSI_RBD_PROVISIONER_SECRET"] 1804 1804 + and self.out_map["CSI_RBD_PROVISIONER_SECRET_NAME"] 1805 1805 + ): 1806 1806 + json_out.append( 1807 1807 + { 1808 1808 + "name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", 1809 1809 + "kind": "Secret", 1810 1810 + "data": { 1811 1811 + "userID": self.out_map["CSI_RBD_PROVISIONER_SECRET_NAME"], 1812 1812 + "userKey": self.out_map["CSI_RBD_PROVISIONER_SECRET"], 1813 1813 + }, 1814 1814 + } 1815 1815 + ) 1816 1816 + # if 'CSI_CEPHFS_PROVISIONER_SECRET' exists, then only add 'rook-csi-cephfs-provisioner' Secret 1817 1817 + if ( 1818 1818 + self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"] 1819 1819 + and self.out_map["CSI_CEPHFS_PROVISIONER_SECRET_NAME"] 1820 1820 + ): 1821 1821 + json_out.append( 1822 1822 + { 1823 1823 + "name": f"rook-{self.out_map['CSI_CEPHFS_PROVISIONER_SECRET_NAME']}", 1824 1824 + "kind": "Secret", 1825 1825 + "data": { 1826 1826 + "adminID": self.out_map["CSI_CEPHFS_PROVISIONER_SECRET_NAME"], 1827 1827 + "adminKey": self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"], 1828 1828 + }, 1829 1829 + } 1830 1830 + ) 1831 1831 + # if 'CSI_CEPHFS_NODE_SECRET' exists, then only add 'rook-csi-cephfs-node' Secret 1832 1832 + if ( 1833 1833 + self.out_map["CSI_CEPHFS_NODE_SECRET"] 1834 1834 + and self.out_map["CSI_CEPHFS_NODE_SECRET_NAME"] 1835 1835 + ): 1836 1836 + json_out.append( 1837 1837 + { 1838 1838 + "name": f"rook-{self.out_map['CSI_CEPHFS_NODE_SECRET_NAME']}", 1839 1839 + "kind": "Secret", 1840 1840 + "data": { 1841 1841 + "adminID": self.out_map["CSI_CEPHFS_NODE_SECRET_NAME"], 1842 1842 + "adminKey": self.out_map["CSI_CEPHFS_NODE_SECRET"], 1843 1843 + }, 1844 1844 + } 1845 1845 + ) 1846 1846 + # if 'ROOK_EXTERNAL_DASHBOARD_LINK' exists, then only add 'rook-ceph-dashboard-link' Secret 1847 1847 + if self.out_map["ROOK_EXTERNAL_DASHBOARD_LINK"]: 1848 1848 + json_out.append( 1849 1849 + { 1850 1850 + "name": "rook-ceph-dashboard-link", 1851 1851 + "kind": "Secret", 1852 1852 + "data": { 1853 1853 + "userID": "ceph-dashboard-link", 1854 1854 + "userKey": self.out_map["ROOK_EXTERNAL_DASHBOARD_LINK"], 1855 1855 + }, 1856 1856 + } 1857 1857 + ) 1858 1858 + # if 'RADOS_NAMESPACE' exists, then only add the "RADOS_NAMESPACE" namespace 1859 1859 + if ( 1860 1860 + self.out_map["RADOS_NAMESPACE"] 1861 1861 + and self.out_map["RESTRICTED_AUTH_PERMISSION"] 1862 1862 + and not self.out_map["RBD_METADATA_EC_POOL_NAME"] 1863 1863 + ): 1864 1864 + json_out.append( 1865 1865 + { 1866 1866 + "name": "rados-namespace", 1867 1867 + "kind": "CephBlockPoolRadosNamespace", 1868 1868 + "data": { 1869 1869 + "radosNamespaceName": self.out_map["RADOS_NAMESPACE"], 1870 1870 + "pool": self.out_map["RBD_POOL_NAME"], 1871 1871 + }, 1872 1872 + } 1873 1873 + ) 1874 1874 + json_out.append( 1875 1875 + { 1876 1876 + "name": "ceph-rbd-rados-namespace", 1877 1877 + "kind": "StorageClass", 1878 1878 + "data": { 1879 1879 + "pool": self.out_map["RBD_POOL_NAME"], 1880 1880 + "csi.storage.k8s.io/provisioner-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", 1881 1881 + "csi.storage.k8s.io/controller-expand-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", 1882 1882 + "csi.storage.k8s.io/node-stage-secret-name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}", 1883 1883 + }, 1884 1884 + } 1885 1885 + ) 1886 1886 + else: 1887 1887 + if self.out_map["RBD_METADATA_EC_POOL_NAME"]: 1888 1888 + json_out.append( 1889 1889 + { 1890 1890 + "name": "ceph-rbd", 1891 1891 + "kind": "StorageClass", 1892 1892 + "data": { 1893 1893 + "dataPool": self.out_map["RBD_POOL_NAME"], 1894 1894 + "pool": self.out_map["RBD_METADATA_EC_POOL_NAME"], 1895 1895 + "csi.storage.k8s.io/provisioner-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", 1896 1896 + "csi.storage.k8s.io/controller-expand-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", 1897 1897 + "csi.storage.k8s.io/node-stage-secret-name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}", 1898 1898 + }, 1899 1899 + } 1900 1900 + ) 1901 1901 + else: 1902 1902 + json_out.append( 1903 1903 + { 1904 1904 + "name": "ceph-rbd", 1905 1905 + "kind": "StorageClass", 1906 1906 + "data": { 1907 1907 + "pool": self.out_map["RBD_POOL_NAME"], 1908 1908 + "csi.storage.k8s.io/provisioner-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", 1909 1909 + "csi.storage.k8s.io/controller-expand-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", 1910 1910 + "csi.storage.k8s.io/node-stage-secret-name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}", 1911 1911 + }, 1912 1912 + } 1913 1913 + ) 1914 1914 + 1915 1915 + # if 'TOPOLOGY_POOLS', 'TOPOLOGY_FAILURE_DOMAIN_LABEL', 'TOPOLOGY_FAILURE_DOMAIN_VALUES' exists, 1916 1916 + # then only add 'topology' StorageClass 1917 1917 + if ( 1918 1918 + self.out_map["TOPOLOGY_POOLS"] 1919 1919 + and self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"] 1920 1920 + and self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] 1921 1921 + ): 1922 1922 + json_out.append( 1923 1923 + { 1924 1924 + "name": "ceph-rbd-topology", 1925 1925 + "kind": "StorageClass", 1926 1926 + "data": { 1927 1927 + "topologyFailureDomainLabel": self.out_map[ 1928 1928 + "TOPOLOGY_FAILURE_DOMAIN_LABEL" 1929 1929 + ], 1930 1930 + "topologyFailureDomainValues": self.out_map[ 1931 1931 + "TOPOLOGY_FAILURE_DOMAIN_VALUES" 1932 1932 + ], 1933 1933 + "topologyPools": self.out_map["TOPOLOGY_POOLS"], 1934 1934 + "csi.storage.k8s.io/provisioner-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", 1935 1935 + "csi.storage.k8s.io/controller-expand-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", 1936 1936 + "csi.storage.k8s.io/node-stage-secret-name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}", 1937 1937 + }, 1938 1938 + } 1939 1939 + ) 1940 1940 + 1941 1941 + # if 'CEPHFS_FS_NAME' exists, then only add 'cephfs' StorageClass 1942 1942 + if self.out_map["CEPHFS_FS_NAME"]: 1943 1943 + json_out.append( 1944 1944 + { 1945 1945 + "name": "cephfs", 1946 1946 + "kind": "StorageClass", 1947 1947 + "data": { 1948 1948 + "fsName": self.out_map["CEPHFS_FS_NAME"], 1949 1949 + "pool": self.out_map["CEPHFS_POOL_NAME"], 1950 1950 + "csi.storage.k8s.io/provisioner-secret-name": f"rook-{self.out_map['CSI_CEPHFS_PROVISIONER_SECRET_NAME']}", 1951 1951 + "csi.storage.k8s.io/controller-expand-secret-name": f"rook-{self.out_map['CSI_CEPHFS_PROVISIONER_SECRET_NAME']}", 1952 1952 + "csi.storage.k8s.io/node-stage-secret-name": f"rook-{self.out_map['CSI_CEPHFS_NODE_SECRET_NAME']}", 1953 1953 + }, 1954 1954 + } 1955 1955 + ) 1956 1956 + # if 'RGW_ENDPOINT' exists, then only add 'ceph-rgw' StorageClass 1957 1957 + if self.out_map["RGW_ENDPOINT"]: 1958 1958 + json_out.append( 1959 1959 + { 1960 1960 + "name": "ceph-rgw", 1961 1961 + "kind": "StorageClass", 1962 1962 + "data": { 1963 1963 + "endpoint": self.out_map["RGW_ENDPOINT"], 1964 1964 + "poolPrefix": self.out_map["RGW_POOL_PREFIX"], 1965 1965 + }, 1966 1966 + } 1967 1967 + ) 1968 1968 + json_out.append( 1969 1969 + { 1970 1970 + "name": "rgw-admin-ops-user", 1971 1971 + "kind": "Secret", 1972 1972 + "data": { 1973 1973 + "accessKey": self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"], 1974 1974 + "secretKey": self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"], 1975 1975 + }, 1976 1976 + } 1977 1977 + ) 1978 1978 + # if 'RGW_TLS_CERT' exists, then only add the "ceph-rgw-tls-cert" secret 1979 1979 + if self.out_map["RGW_TLS_CERT"]: 1980 1980 + json_out.append( 1981 1981 + { 1982 1982 + "name": "ceph-rgw-tls-cert", 1983 1983 + "kind": "Secret", 1984 1984 + "data": { 1985 1985 + "cert": self.out_map["RGW_TLS_CERT"], 1986 1986 + }, 1987 1987 + } 1988 1988 + ) 1989 1989 + 1990 1990 + return json.dumps(json_out) + LINESEP 1991 1991 + 1992 1992 + def upgrade_users_permissions(self): 1993 1993 + users = [ 1994 1994 + "client.csi-cephfs-node", 1995 1995 + "client.csi-cephfs-provisioner", 1996 1996 + "client.csi-rbd-node", 1997 1997 + "client.csi-rbd-provisioner", 1998 1998 + "client.healthchecker", 1999 1999 + ] 2000 2000 + if self.run_as_user != "" and self.run_as_user not in users: 2001 2001 + users.append(self.run_as_user) 2002 2002 + for user in users: 2003 2003 + self.upgrade_user_permissions(user) 2004 2004 + 2005 2005 + def get_rgw_pool_name_during_upgrade(self, user, caps): 2006 2006 + if user == "client.healthchecker": 2007 2007 + # when admin has not provided rgw pool name during upgrade, 2008 2008 + # get the rgw pool name from client.healthchecker user which was used during connection 2009 2009 + if not self._arg_parser.rgw_pool_prefix: 2010 2010 + # To get value 'default' which is rgw pool name from 'allow rwx pool=default.rgw.meta' 2011 2011 + pattern = r"pool=(.*?)\.rgw\.meta" 2012 2012 + match = re.search(pattern, caps) 2013 2013 + if match: 2014 2014 + self._arg_parser.rgw_pool_prefix = match.group(1) 2015 2015 + else: 2016 2016 + raise ExecutionFailureException( 2017 2017 + "failed to get rgw pool name for upgrade" 2018 2018 + ) 2019 2019 + 2020 2020 + def upgrade_user_permissions(self, user): 2021 2021 + # check whether the given user exists or not 2022 2022 + cmd_json = {"prefix": "auth get", "entity": f"{user}", "format": "json"} 2023 2023 + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) 2024 2024 + if ret_val != 0 or len(json_out) == 0: 2025 2025 + print(f"user {user} not found for upgrading.") 2026 2026 + return 2027 2027 + existing_caps = json_out[0]["caps"] 2028 2028 + self.get_rgw_pool_name_during_upgrade(user, str(existing_caps)) 2029 2029 + new_cap, _ = self.get_caps_and_entity(user) 2030 2030 + cap_keys = ["mon", "mgr", "osd", "mds"] 2031 2031 + caps = [] 2032 2032 + for eachCap in cap_keys: 2033 2033 + cur_cap_values = existing_caps.get(eachCap, "") 2034 2034 + new_cap_values = new_cap.get(eachCap, "") 2035 2035 + cur_cap_perm_list = [ 2036 2036 + x.strip() for x in cur_cap_values.split(",") if x.strip() 2037 2037 + ] 2038 2038 + new_cap_perm_list = [ 2039 2039 + x.strip() for x in new_cap_values.split(",") if x.strip() 2040 2040 + ] 2041 2041 + # append new_cap_list to cur_cap_list to maintain the order of caps 2042 2042 + cur_cap_perm_list.extend(new_cap_perm_list) 2043 2043 + # eliminate duplicates without using 'set' 2044 2044 + # set re-orders items in the list and we have to keep the order 2045 2045 + new_cap_list = [] 2046 2046 + [new_cap_list.append(x) for x in cur_cap_perm_list if x not in new_cap_list] 2047 2047 + existing_caps[eachCap] = ", ".join(new_cap_list) 2048 2048 + if existing_caps[eachCap]: 2049 2049 + caps.append(eachCap) 2050 2050 + caps.append(existing_caps[eachCap]) 2051 2051 + cmd_json = { 2052 2052 + "prefix": "auth caps", 2053 2053 + "entity": user, 2054 2054 + "caps": caps, 2055 2055 + "format": "json", 2056 2056 + } 2057 2057 + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) 2058 2058 + if ret_val != 0: 2059 2059 + raise ExecutionFailureException( 2060 2060 + f"'auth caps {user}' command failed.\n Error: {err_msg}" 2061 2061 + ) 2062 2062 + print(f"Updated user {user} successfully.") 2063 2063 + 2064 2064 + def main(self): 2065 2065 + generated_output = "" 2066 2066 + if self._arg_parser.upgrade: 2067 2067 + self.upgrade_users_permissions() 2068 2068 + elif self._arg_parser.format == "json": 2069 2069 + generated_output = self.gen_json_out() 2070 2070 + elif self._arg_parser.format == "bash": 2071 2071 + generated_output = self.gen_shell_out() 2072 2072 + else: 2073 2073 + raise ExecutionFailureException( 2074 2074 + f"Unsupported format: {self._arg_parser.format}" 2075 2075 + ) 2076 2076 + print(generated_output) 2077 2077 + if self.output_file and generated_output: 2078 2078 + fOut = open(self.output_file, mode="w", encoding="UTF-8") 2079 2079 + fOut.write(generated_output) 2080 2080 + fOut.close() 2081 2081 + 2082 2082 + 2083 2083 + ################################################ 2084 2084 + ##################### MAIN ##################### 2085 2085 + ################################################ 2086 2086 + if __name__ == "__main__": 2087 2087 + rjObj = RadosJSON() 2088 2088 + try: 2089 2089 + rjObj.main() 2090 2090 + except ExecutionFailureException as err: 2091 2091 + print(f"Execution Failed: {err}") 2092 2092 + raise err 2093 2093 + except KeyError as kErr: 2094 2094 + print(f"KeyError: {kErr}") 2095 2095 + except OSError as osErr: 2096 2096 + print(f"Error while trying to output the data: {osErr}") 2097 2097 + finally: 2098 2098 + rjObj.shutdown()

+284

k8s/media/ceph-import-ext.sh

reviewed

··· 1 1 + #!/bin/bash 2 2 + set -e 3 3 + 4 4 + ############## 5 5 + # VARIABLES # 6 6 + ############# 7 7 + MON_SECRET_NAME=rook-ceph-mon 8 8 + RGW_ADMIN_OPS_USER_SECRET_NAME=rgw-admin-ops-user 9 9 + MON_SECRET_CLUSTER_NAME_KEYNAME=cluster-name 10 10 + MON_SECRET_FSID_KEYNAME=fsid 11 11 + MON_SECRET_ADMIN_KEYRING_KEYNAME=admin-secret 12 12 + MON_SECRET_MON_KEYRING_KEYNAME=mon-secret 13 13 + MON_SECRET_CEPH_USERNAME_KEYNAME=ceph-username 14 14 + MON_SECRET_CEPH_SECRET_KEYNAME=ceph-secret 15 15 + MON_ENDPOINT_CONFIGMAP_NAME=rook-ceph-mon-endpoints 16 16 + ROOK_EXTERNAL_CLUSTER_NAME=$NAMESPACE 17 17 + ROOK_RBD_FEATURES=${ROOK_RBD_FEATURES:-"layering"} 18 18 + ROOK_EXTERNAL_MAX_MON_ID=2 19 19 + ROOK_EXTERNAL_MAPPING={} 20 20 + RBD_STORAGE_CLASS_NAME=ceph-rbd 21 21 + CEPHFS_STORAGE_CLASS_NAME=cephfs 22 22 + ROOK_EXTERNAL_MONITOR_SECRET=mon-secret 23 23 + OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph 24 24 + RBD_PROVISIONER=$OPERATOR_NAMESPACE".rbd.csi.ceph.com" # driver:namespace:operator 25 25 + CEPHFS_PROVISIONER=$OPERATOR_NAMESPACE".cephfs.csi.ceph.com" # driver:namespace:operator 26 26 + CLUSTER_ID_RBD=$NAMESPACE 27 27 + CLUSTER_ID_CEPHFS=$NAMESPACE 28 28 + : "${ROOK_EXTERNAL_ADMIN_SECRET:=admin-secret}" 29 29 + 30 30 + ############# 31 31 + # FUNCTIONS # 32 32 + ############# 33 33 + 34 34 + function checkEnvVars() { 35 35 + if [ -z "$NAMESPACE" ]; then 36 36 + echo "Please populate the environment variable NAMESPACE" 37 37 + exit 1 38 38 + fi 39 39 + if [ -z "$ROOK_RBD_FEATURES" ] || [[ ! "$ROOK_RBD_FEATURES" =~ .*"layering".* ]]; then 40 40 + echo "Please populate the environment variable ROOK_RBD_FEATURES" 41 41 + echo "For a kernel earlier than 5.4 use a value of 'layering'; for 5.4 or later" 42 42 + echo "use 'layering,fast-diff,object-map,deep-flatten,exclusive-lock'" 43 43 + exit 1 44 44 + fi 45 45 + if [ -z "$RBD_POOL_NAME" ]; then 46 46 + echo "Please populate the environment variable RBD_POOL_NAME" 47 47 + exit 1 48 48 + fi 49 49 + if [ -z "$CSI_RBD_NODE_SECRET_NAME" ]; then 50 50 + echo "Please populate the environment variable CSI_RBD_NODE_SECRET_NAME" 51 51 + exit 1 52 52 + fi 53 53 + if [ -z "$CSI_RBD_PROVISIONER_SECRET_NAME" ]; then 54 54 + echo "Please populate the environment variable CSI_RBD_PROVISIONER_SECRET_NAME" 55 55 + exit 1 56 56 + fi 57 57 + if [ -z "$CSI_CEPHFS_NODE_SECRET_NAME" ]; then 58 58 + echo "Please populate the environment variable CSI_CEPHFS_NODE_SECRET_NAME" 59 59 + exit 1 60 60 + fi 61 61 + if [ -z "$CSI_CEPHFS_PROVISIONER_SECRET_NAME" ]; then 62 62 + echo "Please populate the environment variable CSI_CEPHFS_PROVISIONER_SECRET_NAME" 63 63 + exit 1 64 64 + fi 65 65 + if [ -z "$ROOK_EXTERNAL_FSID" ]; then 66 66 + echo "Please populate the environment variable ROOK_EXTERNAL_FSID" 67 67 + exit 1 68 68 + fi 69 69 + if [ -z "$ROOK_EXTERNAL_CEPH_MON_DATA" ]; then 70 70 + echo "Please populate the environment variable ROOK_EXTERNAL_CEPH_MON_DATA" 71 71 + exit 1 72 72 + fi 73 73 + if [[ "$ROOK_EXTERNAL_ADMIN_SECRET" == "admin-secret" ]]; then 74 74 + if [ -z "$ROOK_EXTERNAL_USER_SECRET" ]; then 75 75 + echo "Please populate the environment variable ROOK_EXTERNAL_USER_SECRET" 76 76 + exit 1 77 77 + fi 78 78 + if [ -z "$ROOK_EXTERNAL_USERNAME" ]; then 79 79 + echo "Please populate the environment variable ROOK_EXTERNAL_USERNAME" 80 80 + exit 1 81 81 + fi 82 82 + if [ -z "$CSI_RBD_NODE_SECRET" ]; then 83 83 + echo "Please populate the environment variable CSI_RBD_NODE_SECRET" 84 84 + exit 1 85 85 + fi 86 86 + if [ -z "$CSI_RBD_PROVISIONER_SECRET" ]; then 87 87 + echo "Please populate the environment variable CSI_RBD_PROVISIONER_SECRET" 88 88 + exit 1 89 89 + fi 90 90 + if [ -z "$CSI_CEPHFS_NODE_SECRET" ]; then 91 91 + echo "Please populate the environment variable CSI_CEPHFS_NODE_SECRET" 92 92 + exit 1 93 93 + fi 94 94 + if [ -z "$CSI_CEPHFS_PROVISIONER_SECRET" ]; then 95 95 + echo "Please populate the environment variable CSI_CEPHFS_PROVISIONER_SECRET" 96 96 + exit 1 97 97 + fi 98 98 + fi 99 99 + if [[ "$ROOK_EXTERNAL_ADMIN_SECRET" != "admin-secret" ]] && [ -n "$ROOK_EXTERNAL_USER_SECRET" ] ; then 100 100 + echo "Providing both ROOK_EXTERNAL_ADMIN_SECRET and ROOK_EXTERNAL_USER_SECRET is not supported, choose one only." 101 101 + exit 1 102 102 + fi 103 103 + } 104 104 + 105 105 + function importClusterID() { 106 106 + if [ -n "$RADOS_NAMESPACE" ]; then 107 107 + CLUSTER_ID_RBD=$(kubectl -n "$NAMESPACE" get cephblockpoolradosnamespace.ceph.rook.io/"$RADOS_NAMESPACE" -o jsonpath='{.status.info.clusterID}') 108 108 + fi 109 109 + if [ -n "$SUBVOLUME_GROUP" ]; then 110 110 + CLUSTER_ID_CEPHFS=$(kubectl -n "$NAMESPACE" get cephfilesystemsubvolumegroup.ceph.rook.io/"$SUBVOLUME_GROUP" -o jsonpath='{.status.info.clusterID}') 111 111 + fi 112 112 + } 113 113 + 114 114 + function importSecret() { 115 115 + kubectl -n "$NAMESPACE" \ 116 116 + create \ 117 117 + secret \ 118 118 + generic \ 119 119 + --type="kubernetes.io/rook" \ 120 120 + "$MON_SECRET_NAME" \ 121 121 + --from-literal="$MON_SECRET_CLUSTER_NAME_KEYNAME"="$ROOK_EXTERNAL_CLUSTER_NAME" \ 122 122 + --from-literal="$MON_SECRET_FSID_KEYNAME"="$ROOK_EXTERNAL_FSID" \ 123 123 + --from-literal="$MON_SECRET_ADMIN_KEYRING_KEYNAME"="$ROOK_EXTERNAL_ADMIN_SECRET" \ 124 124 + --from-literal="$MON_SECRET_MON_KEYRING_KEYNAME"="$ROOK_EXTERNAL_MONITOR_SECRET" \ 125 125 + --from-literal="$MON_SECRET_CEPH_USERNAME_KEYNAME"="$ROOK_EXTERNAL_USERNAME" \ 126 126 + --from-literal="$MON_SECRET_CEPH_SECRET_KEYNAME"="$ROOK_EXTERNAL_USER_SECRET" 127 127 + } 128 128 + 129 129 + function importConfigMap() { 130 130 + kubectl -n "$NAMESPACE" \ 131 131 + create \ 132 132 + configmap \ 133 133 + "$MON_ENDPOINT_CONFIGMAP_NAME" \ 134 134 + --from-literal=data="$ROOK_EXTERNAL_CEPH_MON_DATA" \ 135 135 + --from-literal=mapping="$ROOK_EXTERNAL_MAPPING" \ 136 136 + --from-literal=maxMonId="$ROOK_EXTERNAL_MAX_MON_ID" 137 137 + } 138 138 + 139 139 + function importCsiRBDNodeSecret() { 140 140 + kubectl -n "$NAMESPACE" \ 141 141 + create \ 142 142 + secret \ 143 143 + generic \ 144 144 + --type="kubernetes.io/rook" \ 145 145 + "rook-""$CSI_RBD_NODE_SECRET_NAME" \ 146 146 + --from-literal=userID="$CSI_RBD_NODE_SECRET_NAME" \ 147 147 + --from-literal=userKey="$CSI_RBD_NODE_SECRET" 148 148 + } 149 149 + 150 150 + function importCsiRBDProvisionerSecret() { 151 151 + kubectl -n "$NAMESPACE" \ 152 152 + create \ 153 153 + secret \ 154 154 + generic \ 155 155 + --type="kubernetes.io/rook" \ 156 156 + "rook-""$CSI_RBD_PROVISIONER_SECRET_NAME" \ 157 157 + --from-literal=userID="$CSI_RBD_PROVISIONER_SECRET_NAME" \ 158 158 + --from-literal=userKey="$CSI_RBD_PROVISIONER_SECRET" 159 159 + } 160 160 + 161 161 + function importCsiCephFSNodeSecret() { 162 162 + kubectl -n "$NAMESPACE" \ 163 163 + create \ 164 164 + secret \ 165 165 + generic \ 166 166 + --type="kubernetes.io/rook" \ 167 167 + "rook-""$CSI_CEPHFS_NODE_SECRET_NAME" \ 168 168 + --from-literal=adminID="$CSI_CEPHFS_NODE_SECRET_NAME" \ 169 169 + --from-literal=adminKey="$CSI_CEPHFS_NODE_SECRET" 170 170 + } 171 171 + 172 172 + function importCsiCephFSProvisionerSecret() { 173 173 + kubectl -n "$NAMESPACE" \ 174 174 + create \ 175 175 + secret \ 176 176 + generic \ 177 177 + --type="kubernetes.io/rook" \ 178 178 + "rook-""$CSI_CEPHFS_PROVISIONER_SECRET_NAME" \ 179 179 + --from-literal=adminID="$CSI_CEPHFS_PROVISIONER_SECRET_NAME" \ 180 180 + --from-literal=adminKey="$CSI_CEPHFS_PROVISIONER_SECRET" 181 181 + } 182 182 + 183 183 + function importRGWAdminOpsUser() { 184 184 + kubectl -n "$NAMESPACE" \ 185 185 + create \ 186 186 + secret \ 187 187 + generic \ 188 188 + --type="kubernetes.io/rook" \ 189 189 + "$RGW_ADMIN_OPS_USER_SECRET_NAME" \ 190 190 + --from-literal=accessKey="$RGW_ADMIN_OPS_USER_ACCESS_KEY" \ 191 191 + --from-literal=secretKey="$RGW_ADMIN_OPS_USER_SECRET_KEY" 192 192 + } 193 193 + 194 194 + function createECRBDStorageClass() { 195 195 + cat <<eof | kubectl create -f - 196 196 + apiVersion: storage.k8s.io/v1 197 197 + kind: StorageClass 198 198 + metadata: 199 199 + name: $RBD_STORAGE_CLASS_NAME 200 200 + provisioner: $RBD_PROVISIONER 201 201 + parameters: 202 202 + clusterID: $CLUSTER_ID_RBD 203 203 + pool: $RBD_POOL_NAME 204 204 + dataPool: $RBD_METADATA_EC_POOL_NAME 205 205 + imageFormat: "2" 206 206 + imageFeatures: $ROOK_RBD_FEATURES 207 207 + csi.storage.k8s.io/provisioner-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" 208 208 + csi.storage.k8s.io/provisioner-secret-namespace: $NAMESPACE 209 209 + csi.storage.k8s.io/controller-expand-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" 210 210 + csi.storage.k8s.io/controller-expand-secret-namespace: $NAMESPACE 211 211 + csi.storage.k8s.io/node-stage-secret-name: "rook-$CSI_RBD_NODE_SECRET_NAME" 212 212 + csi.storage.k8s.io/node-stage-secret-namespace: $NAMESPACE 213 213 + csi.storage.k8s.io/fstype: ext4 214 214 + allowVolumeExpansion: true 215 215 + reclaimPolicy: Delete 216 216 + eof 217 217 + } 218 218 + 219 219 + function createRBDStorageClass() { 220 220 + cat <<eof | kubectl create -f - 221 221 + apiVersion: storage.k8s.io/v1 222 222 + kind: StorageClass 223 223 + metadata: 224 224 + name: $RBD_STORAGE_CLASS_NAME 225 225 + provisioner: $RBD_PROVISIONER 226 226 + parameters: 227 227 + clusterID: $CLUSTER_ID_RBD 228 228 + pool: $RBD_POOL_NAME 229 229 + imageFormat: "2" 230 230 + imageFeatures: $ROOK_RBD_FEATURES 231 231 + csi.storage.k8s.io/provisioner-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" 232 232 + csi.storage.k8s.io/provisioner-secret-namespace: $NAMESPACE 233 233 + csi.storage.k8s.io/controller-expand-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" 234 234 + csi.storage.k8s.io/controller-expand-secret-namespace: $NAMESPACE 235 235 + csi.storage.k8s.io/node-stage-secret-name: "rook-$CSI_RBD_NODE_SECRET_NAME" 236 236 + csi.storage.k8s.io/node-stage-secret-namespace: $NAMESPACE 237 237 + csi.storage.k8s.io/fstype: ext4 238 238 + allowVolumeExpansion: true 239 239 + reclaimPolicy: Delete 240 240 + eof 241 241 + } 242 242 + 243 243 + function createCephFSStorageClass() { 244 244 + cat <<eof | kubectl create -f - 245 245 + apiVersion: storage.k8s.io/v1 246 246 + kind: StorageClass 247 247 + metadata: 248 248 + name: $CEPHFS_STORAGE_CLASS_NAME 249 249 + provisioner: $CEPHFS_PROVISIONER 250 250 + parameters: 251 251 + clusterID: $CLUSTER_ID_CEPHFS 252 252 + fsName: $CEPHFS_FS_NAME 253 253 + pool: $CEPHFS_POOL_NAME 254 254 + csi.storage.k8s.io/provisioner-secret-name: "rook-$CSI_CEPHFS_PROVISIONER_SECRET_NAME" 255 255 + csi.storage.k8s.io/provisioner-secret-namespace: $NAMESPACE 256 256 + csi.storage.k8s.io/controller-expand-secret-name: "rook-$CSI_CEPHFS_PROVISIONER_SECRET_NAME" 257 257 + csi.storage.k8s.io/controller-expand-secret-namespace: $NAMESPACE 258 258 + csi.storage.k8s.io/node-stage-secret-name: "rook-$CSI_CEPHFS_NODE_SECRET_NAME" 259 259 + csi.storage.k8s.io/node-stage-secret-namespace: $NAMESPACE 260 260 + allowVolumeExpansion: true 261 261 + reclaimPolicy: Delete 262 262 + eof 263 263 + } 264 264 + 265 265 + ######## 266 266 + # MAIN # 267 267 + ######## 268 268 + checkEnvVars 269 269 + importClusterID 270 270 + importSecret 271 271 + importConfigMap 272 272 + importCsiRBDNodeSecret 273 273 + importCsiRBDProvisionerSecret 274 274 + importCsiCephFSNodeSecret 275 275 + importCsiCephFSProvisionerSecret 276 276 + importRGWAdminOpsUser 277 277 + if [ -n "$RBD_METADATA_EC_POOL_NAME" ]; then 278 278 + createECRBDStorageClass 279 279 + else 280 280 + createRBDStorageClass 281 281 + fi 282 282 + if [ -n "$CEPHFS_FS_NAME" ] && [ -n "$CEPHFS_POOL_NAME" ]; then 283 283 + createCephFSStorageClass 284 284 + fi