this repo has no description
Initial commit
authored by
Kaushik Chakraborty
and committed by
Kaushik Chakraborty
beab2616
+480
+23
README.md
+23
README.md
···
1
+
## Overview
2
+
3
+
This is a simple Ansible playbook to setup a Rook Storage Cluster inside an already available Kubernetes Cluster.
4
+
5
+
## Pre-Requisites
6
+
7
+
Node in the `k8s_master` group of the `inventory/hosts.cfg` is the Kubernetes cluster's master node. It's assumed that `kubectl` command is available and setup there for the tasks in the playbook to work. Along with the Kubernetes cluster, Rook Operator also need to be setup in the same cluster as mentioned 
8
+
9
+
## Approach & Configuration
10
+
11
+
The approach to setup Rook Storage cluster in this playbook is that a particular mount point in all the nodes assigned for storage will be used as the storage cluster's directories. This same information is mentioned in the `templates/rook-cluster-config.ymk.j2`.
12
+
13
+
Along with the Rook cluster, this playbook also sets up Rook Toolbox required to monitor the Rook / Ceph cluster.
14
+
15
+
The `storage_nodes` in the `inventory/hosts.cfg` denotes the storage node details. Other relvant values for the Rook cluster setup is in the `inventory/group_vars/k8s_master.yml`. The `inventory/group_vars/all/vault.yml` contain some sensitive information like the actual IPs and ssh user, please make sure to edit them before executing the playbook.
16
+
17
+
For more details check on ,  and 
18
+
19
+
## Playbook execution
20
+
21
+
```
22
+
$ ansible-playbook -i inventory/hosts.cfg -kK -v --ask-vault-pass playbook.yml
23
+
```
+1
inventory/group_vars/all/vars.yml
+1
inventory/group_vars/all/vars.yml
···
1
+
data_dir_host_path: /var/lib/rook
+22
inventory/group_vars/all/vault.yml
+22
inventory/group_vars/all/vault.yml
···
1
+
$ANSIBLE_VAULT;1.1;AES256
2
+
32636466333731303737353038353039343936376261336430636464646266616136386631366630
3
+
3963393336383030356631373434363662663562646431330a376464306265636361363038346437
4
+
61306164376534313432326239636539373636396462356365323366653737346130613836613130
5
+
3536313664636236320a363834633735666664333038353562623764666665623962366539383731
6
+
64353766343865336664326539313361303036326431353438613037323939386633646561343263
7
+
63663934313562616436323863333734666432393130663664643635643065626663336435356638
8
+
62343864663465643164353139346462393030653861373561303866623362613936343364393462
9
+
62326162333939363933616563363162363263333339323866383666303033363661643364353034
10
+
37613533316533383133353232396532633134303235313361356238383135323232613139396134
11
+
63656531353466376163343336616562396665333131333462666436386365346136373038373664
12
+
32386239643166653439653732343536616335366437386362343966376438653166653839336264
13
+
39336532343436323533636538646535666236623037373161353265386161663338373836393831
14
+
39353132313766396238636162343133393764623361613934636266303861326166666330303461
15
+
36313462623565656563326266336162356463633863353136343332656366636362653239626662
16
+
32613232346363616264666537393233363537353535333965353862376239316262376236383736
17
+
63643937376437366539383933383838653866613363353034653231666330633135633234336633
18
+
64343933313932633534653532313265613964616134333061336566363130303736323539346165
19
+
39623330303330303164346338616234613330653861373563306336643764663536303731333563
20
+
30333565616135353965313061666538666232316664613666376266346132316637313636373933
21
+
35616238303830316165303033393933663765333332316262633433306566353534616630346530
22
+
3561
+11
inventory/group_vars/k8s_master.yml
+11
inventory/group_vars/k8s_master.yml
+6
inventory/group_vars/storage_nodes.yml
+6
inventory/group_vars/storage_nodes.yml
+31
inventory/hosts.cfg
+31
inventory/hosts.cfg
···
1
+
[all]
2
+
ctsc01029721901 ansible_host={{vault_hosts.ips.node1}} ansible_user={{vault_hosts.ssh_user}}
3
+
ctsc01029725101 ansible_host={{vault_hosts.ips.node4}} k8s_host_name=ctsc01029725101 ansible_user={{vault_hosts.ssh_user}}
4
+
ctsc01029725401 ansible_host={{vault_hosts.ips.node5}} k8s_host_name=ctsc01029725401 ansible_user={{vault_hosts.ssh_user}}
5
+
ctsc01029730401 ansible_host={{vault_hosts.ips.node6}} k8s_host_name=ctsc01029730401 ansible_user={{vault_hosts.ssh_user}}
6
+
7
+
8
+
ctsc01029723101 ansible_host={{vault_hosts.ips.node3}} k8s_host_name=ctsc01029723101 ansible_user={{vault_hosts.ssh_user}}
9
+
ctsc01067708301 ansible_host={{vault_hosts.ips.node8}} k8s_host_name=ctsc01067708301 ansible_user={{vault_hosts.ssh_user}}
10
+
ctsc01067732001 ansible_host={{vault_hosts.ips.node9}} k8s_host_name=ctsc01067732001 ansible_user={{vault_hosts.ssh_user}}
11
+
ctsc01067709301 ansible_host={{vault_hosts.ips.node10}} k8s_host_name=ctsc01067709301 ansible_user={{vault_hosts.ssh_user}}
12
+
ctsc01067709801 ansible_host={{vault_hosts.ips.node11}} k8s_host_name=ctsc01067709801 ansible_user={{vault_hosts.ssh_user}}
13
+
ctsc01067720001 ansible_host={{vault_hosts.ips.node12}} k8s_host_name=ctsc01067720001 ansible_user={{vault_hosts.ssh_user}}
14
+
ctsc01029722401 ansible_host={{vault_hosts.ips.node2}} k8s_host_name=ctsc01029722401 ansible_user={{vault_hosts.ssh_user}}
15
+
16
+
[k8s_master]
17
+
ctsc01029721901
18
+
19
+
[storage_nodes]
20
+
ctsc01029725101
21
+
ctsc01029725401
22
+
ctsc01029730401
23
+
24
+
[other_nodes]
25
+
ctsc01029723101
26
+
ctsc01067708301
27
+
ctsc01067732001
28
+
ctsc01067709301
29
+
ctsc01067709801
30
+
ctsc01067720001
31
+
ctsc01029722401
+114
playbook.yml
+114
playbook.yml
···
1
+
---
2
+
- hosts: all
3
+
tasks:
4
+
- name: remove any old rook data directories
5
+
become: yes
6
+
become_method: sudo
7
+
file:
8
+
path: "{{ data_dir_host_path }}"
9
+
state: absent
10
+
force: yes
11
+
tags: cleanup
12
+
13
+
- hosts: storage_nodes
14
+
tasks:
15
+
- name: cleanup data directories
16
+
become: yes
17
+
become_method: sudo
18
+
with_items: "{{ storageconfig.directory.paths }}"
19
+
shell: "rm -rf {{ item }}/*"
20
+
tags: cleanup
21
+
22
+
- hosts: k8s_master
23
+
tasks:
24
+
- name: ensure no rook operator is setup yet
25
+
shell: "kubectl get po -n {{ rook_operator_namespace }} -o jsonpath='{.items[*]}' | wc -l | tr -d ' '"
26
+
register: norookoperator
27
+
failed_when: (norookoperator.stdout | int) > 0
28
+
changed_when: false
29
+
30
+
- name: ensure no rook cluster is setup yet
31
+
shell: "kubectl get po -n {{ rook_cluster_namespace }} -o jsonpath='{.items[*]}' | wc -l | tr -d ' '"
32
+
register: norookcluster
33
+
failed_when: (norookcluster.stdout | int) > 0
34
+
changed_when: false
35
+
when: norookoperator | succeeded
36
+
37
+
38
+
- name: copy rook operator config
39
+
become: yes
40
+
become_method: sudo
41
+
template:
42
+
src: rook-operator-config.yml.j2
43
+
dest: "{{ rook_cluster_config_dir }}/rook-operator-config.yml"
44
+
register: rook_operator_config
45
+
when: norookcluster | succeeded
46
+
47
+
- name: copy rook cluster config
48
+
become: yes
49
+
become_method: sudo
50
+
template:
51
+
src: rook-cluster-config.yml.j2
52
+
dest: "{{ rook_cluster_config_dir }}/rook-cluster-config.yml"
53
+
register: rook_cluster_config
54
+
when: norookcluster | succeeded
55
+
56
+
57
+
- name: create rook operator
58
+
shell: "kubectl create -f {{ rook_cluster_config_dir }}/rook-operator-config.yml"
59
+
tags: operator
60
+
61
+
- name: ensure rook operator is up and running
62
+
shell: >
63
+
timeout -k 300s 300s
64
+
kubectl get po -n "{{ rook_operator_namespace }}"
65
+
-o jsonpath='{range .items[*]}{.metadata.name} : {.status.phase}{"\n"}{end}'
66
+
| grep -vic Running
67
+
register: rook_operator_created
68
+
changed_when: false
69
+
failed_when: (rook_operator_created.stdout | int) > 0
70
+
retries: 10
71
+
tags: operator, cluster
72
+
73
+
- name: create rook cluster
74
+
shell: "kubectl create -f {{ rook_cluster_config_dir }}/rook-cluster-config.yml"
75
+
when: rook_operator_created | succeeded
76
+
tags: cluster
77
+
78
+
- name: ensure rook cluster is up and running
79
+
shell: >
80
+
timeout -k 300s 300s
81
+
kubectl get po -n "{{ rook_cluster_namespace }}"
82
+
-o jsonpath='{range .items[*]}{.metadata.name} : {.status.phase}{"\n"}{end}'
83
+
| grep -vic Running
84
+
register: rook_cluster_created
85
+
changed_when: false
86
+
failed_when: (rook_cluster_created.stdout | int) > 0
87
+
retries: 10
88
+
tags: cluster
89
+
90
+
- name: copy rook toolbox pod config
91
+
become: yes
92
+
become_method: sudo
93
+
template:
94
+
src: rook-toolbox-config.yml.j2
95
+
dest: "{{ rook_cluster_config_dir }}/rook-toolbox-config.yml"
96
+
register: rook_toolbox_config
97
+
# when: rook_cluster_created | succeeded
98
+
tags: toolbox
99
+
100
+
- name: launch rook toolbox pod
101
+
shell: "kubectl create -f {{ rook_cluster_config_dir }}/rook-toolbox-config.yml"
102
+
tags: toolbox
103
+
104
+
- name: ensure rook toolbox pod is up and running
105
+
shell: >
106
+
kubectl get po -n "{{ rook_cluster_namespace }}" rook-tools
107
+
-o jsonpath='{range .items[*]}{.metadata.name} : {.status.phase}{"\n"}{end}'
108
+
| grep -vic Running
109
+
register: rook_toolbox_launched
110
+
changed_when: false
111
+
failed_when: (rook_toolbox_launched.stdout | int) > 0
112
+
retries: 10
113
+
tags: toolbox
114
+
+50
templates/rook-cluster-config.yml.j2
+50
templates/rook-cluster-config.yml.j2
···
1
+
apiVersion: v1
2
+
kind: Namespace
3
+
metadata:
4
+
name: {{ rook_cluster_namespace }}
5
+
---
6
+
apiVersion: rook.io/v1alpha1
7
+
kind: Cluster
8
+
metadata:
9
+
name: rook
10
+
namespace: {{ rook_cluster_namespace }}
11
+
spec:
12
+
# The path on the host where configuration files will be persisted. If not specified, a kubernetes emptyDir will be created (not recommended).
13
+
# Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
14
+
dataDirHostPath: {{data_dir_host_path}}
15
+
placement:
16
+
all:
17
+
nodeAffinity:
18
+
requiredDuringSchedulingIgnoredDuringExecution:
19
+
nodeSelectorTerms:
20
+
- matchExpressions:
21
+
- key: node-type
22
+
operator: In
23
+
values:
24
+
- storage-node
25
+
26
+
# toggle to use hostNetwork
27
+
hostNetwork: false
28
+
# set the amount of mons to be started
29
+
monCount: 3
30
+
storage: # cluster level storage configuration and selection
31
+
useAllNodes: false
32
+
useAllDevices: false
33
+
deviceFilter:
34
+
metadataDevice:
35
+
location:
36
+
storeConfig:
37
+
storeType: {{storageconfig.storetype | default("bluestore")}}
38
+
databaseSizeMB: {{storageconfig.databasesize | default(1024)}} # this value can be removed for environments with normal sized disks (100 GB or larger)
39
+
journalSizeMB: {{storageconfig.journalsize | default(1024)}} # this value can be removed for environments with normal sized disks (20 GB or larger)
40
+
41
+
directories:
42
+
{% for d in storageconfig.directory.paths %}
43
+
- path: "{{ d }}"
44
+
{% endfor %}
45
+
46
+
nodes:
47
+
{% for n in groups['storage_nodes'] %}
48
+
- name: "{{ hostvars[n]['k8s_host_name'] }}"
49
+
50
+
{% endfor %}
+166
templates/rook-operator-config.yml.j2
+166
templates/rook-operator-config.yml.j2
···
1
+
apiVersion: v1
2
+
kind: Namespace
3
+
metadata:
4
+
name: rook-system
5
+
---
6
+
kind: ClusterRole
7
+
apiVersion: rbac.authorization.k8s.io/v1beta1
8
+
metadata:
9
+
name: rook-operator
10
+
rules:
11
+
- apiGroups:
12
+
- ""
13
+
resources:
14
+
- namespaces
15
+
- serviceaccounts
16
+
- secrets
17
+
- pods
18
+
- services
19
+
- nodes
20
+
- nodes/proxy
21
+
- configmaps
22
+
- events
23
+
- persistentvolumes
24
+
- persistentvolumeclaims
25
+
verbs:
26
+
- get
27
+
- list
28
+
- watch
29
+
- patch
30
+
- create
31
+
- update
32
+
- delete
33
+
- apiGroups:
34
+
- extensions
35
+
resources:
36
+
- thirdpartyresources
37
+
- deployments
38
+
- daemonsets
39
+
- replicasets
40
+
verbs:
41
+
- get
42
+
- list
43
+
- watch
44
+
- create
45
+
- update
46
+
- delete
47
+
- apiGroups:
48
+
- apiextensions.k8s.io
49
+
resources:
50
+
- customresourcedefinitions
51
+
verbs:
52
+
- get
53
+
- list
54
+
- watch
55
+
- create
56
+
- delete
57
+
- apiGroups:
58
+
- rbac.authorization.k8s.io
59
+
resources:
60
+
- clusterroles
61
+
- clusterrolebindings
62
+
- roles
63
+
- rolebindings
64
+
verbs:
65
+
- get
66
+
- list
67
+
- watch
68
+
- create
69
+
- update
70
+
- delete
71
+
- apiGroups:
72
+
- storage.k8s.io
73
+
resources:
74
+
- storageclasses
75
+
verbs:
76
+
- get
77
+
- list
78
+
- watch
79
+
- delete
80
+
- apiGroups:
81
+
- rook.io
82
+
resources:
83
+
- "*"
84
+
verbs:
85
+
- "*"
86
+
---
87
+
apiVersion: v1
88
+
kind: ServiceAccount
89
+
metadata:
90
+
name: rook-operator
91
+
namespace: rook-system
92
+
---
93
+
kind: ClusterRoleBinding
94
+
apiVersion: rbac.authorization.k8s.io/v1beta1
95
+
metadata:
96
+
name: rook-operator
97
+
namespace: rook-system
98
+
roleRef:
99
+
apiGroup: rbac.authorization.k8s.io
100
+
kind: ClusterRole
101
+
name: rook-operator
102
+
subjects:
103
+
- kind: ServiceAccount
104
+
name: rook-operator
105
+
namespace: rook-system
106
+
---
107
+
apiVersion: apps/v1beta1
108
+
kind: Deployment
109
+
metadata:
110
+
name: rook-operator
111
+
namespace: rook-system
112
+
spec:
113
+
replicas: 1
114
+
template:
115
+
metadata:
116
+
labels:
117
+
app: rook-operator
118
+
spec:
119
+
affinity:
120
+
nodeAffinity:
121
+
requiredDuringSchedulingIgnoredDuringExecution:
122
+
nodeSelectorTerms:
123
+
- matchExpressions:
124
+
- key: node-type
125
+
operator: In
126
+
values:
127
+
- storage-node
128
+
129
+
serviceAccountName: rook-operator
130
+
containers:
131
+
- name: rook-operator
132
+
image: rook/rook:master
133
+
args: ["operator"]
134
+
env:
135
+
# To disable RBAC, uncomment the following:
136
+
# - name: RBAC_ENABLED
137
+
# value: "false"
138
+
# Rook Agent toleration. Will tolerate all taints with all keys.
139
+
# Choose between NoSchedule, PreferNoSchedule and NoExecute:
140
+
# - name: AGENT_TOLERATION
141
+
# value: "NoSchedule"
142
+
# (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate
143
+
# - name: AGENT_TOLERATION_KEY
144
+
# value: "<KeyOfTheTaintToTolerate>"
145
+
# Set the path where the Rook agent can find the flex volumes
146
+
# - name: FLEXVOLUME_DIR_PATH
147
+
# value: "<PathToFlexVolumes>"
148
+
# The interval to check if every mon is in the quorum.
149
+
- name: ROOK_MON_HEALTHCHECK_INTERVAL
150
+
value: "45s"
151
+
# The duration to wait before trying to failover or remove/replace the
152
+
# current mon with a new mon (useful for compensating flapping network).
153
+
- name: ROOK_MON_OUT_TIMEOUT
154
+
value: "300s"
155
+
- name: NODE_NAME
156
+
valueFrom:
157
+
fieldRef:
158
+
fieldPath: spec.nodeName
159
+
- name: POD_NAME
160
+
valueFrom:
161
+
fieldRef:
162
+
fieldPath: metadata.name
163
+
- name: POD_NAMESPACE
164
+
valueFrom:
165
+
fieldRef:
166
+
fieldPath: metadata.namespace
+56
templates/rook-toolbox-config.yml.j2
+56
templates/rook-toolbox-config.yml.j2
···
1
+
apiVersion: v1
2
+
kind: Pod
3
+
metadata:
4
+
name: rook-tools
5
+
namespace: {{rook_cluster_namespace}}
6
+
spec:
7
+
8
+
affinity:
9
+
nodeAffinity:
10
+
requiredDuringSchedulingIgnoredDuringExecution:
11
+
nodeSelectorTerms:
12
+
- matchExpressions:
13
+
- key: node-type
14
+
operator: In
15
+
values:
16
+
- storage-node
17
+
18
+
dnsPolicy: ClusterFirstWithHostNet
19
+
containers:
20
+
- name: rook-tools
21
+
image: rook/toolbox:master
22
+
imagePullPolicy: IfNotPresent
23
+
env:
24
+
- name: ROOK_ADMIN_SECRET
25
+
valueFrom:
26
+
secretKeyRef:
27
+
name: rook-ceph-mon
28
+
key: admin-secret
29
+
securityContext:
30
+
privileged: true
31
+
volumeMounts:
32
+
- mountPath: /dev
33
+
name: dev
34
+
- mountPath: /sys/bus
35
+
name: sysbus
36
+
- mountPath: /lib/modules
37
+
name: libmodules
38
+
- name: mon-endpoint-volume
39
+
mountPath: /etc/rook
40
+
hostNetwork: false
41
+
volumes:
42
+
- name: dev
43
+
hostPath:
44
+
path: /dev
45
+
- name: sysbus
46
+
hostPath:
47
+
path: /sys/bus
48
+
- name: libmodules
49
+
hostPath:
50
+
path: /lib/modules
51
+
- name: mon-endpoint-volume
52
+
configMap:
53
+
name: rook-ceph-mon-endpoints
54
+
items:
55
+
- key: data
56
+
path: mon-endpoints