logo头像
Snippet 博客主题

260_k8s-实践

背景介绍

部署架构图

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
jenkins
k8s
设计模式
uml

网络
操作系统
数据结构算法leetcode 剑指offer


ip: 112.74.175.239


--namespace:icql-devops

devops.k8s.icql.work
devops.jenkins.icql.work
devops.frp.icql.work
devops.nexus.icql.work

--namespace:icql-svc

svc.icql.work:3306
svc.icql.work:6379
svc.icql.work:9200
svc.icql.work:5601

--namespace:icql-api
api.icql.work
api.nacos.icql.work
api.gateway.icql.work


--namespace:icql-static
icql.work
file.icql.work
archive.icql.work



efk 收集日志
链路跟踪

搭建 k8s 集群

部署 cluster 资源

1
2
# 创建 namespace
kubctl apply -f k8s-namespace.yaml

k8s-namespace.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# devops资源
---
kind: Namespace
apiVersion: v1
metadata:
name: icql-devops

# 服务资源
---
kind: Namespace
apiVersion: v1
metadata:
name: icql-svc

# 测试资源
---
kind: Namespace
apiVersion: v1
metadata:
name: icql-test

部署 icql-devops 资源

1
2
# 创建 icql-devops 管理员账户
kubectl apply -f icql-devops-serviceaccount.yaml

icql-devops-serviceaccount.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# rbac
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cluster-admin
namespace: icql-devops
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: "icql-devops:cluster-admin" # cluter资源,所以加一个命名空间前缀区分
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
kind: ClusterRole
name: cluster-admin # 绑定集群管理员
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: cluster-admin
namespace: icql-devops

1、jenkins

1
2
# 部署 icql-devops-jenkins
kubectl apply -f icql-devops-jenkins.yaml

1)部署

icql-devops-jenkins.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
# 集群资源:jenkins数据
---
kind: PersistentVolume
apiVersion: v1
metadata:
name: pv-icql-devops-jenkins-data
labels:
app: pv-icql-devops-jenkins-data
spec:
storageClassName: manual
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
hostPath:
path: "/data/icql-devops/jenkins/data"

# pvc绑定pv
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: pvc-jenkins
namespace: icql-devops
labels:
app: pvc-jenkins
spec:
storageClassName: manual
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
selector:
matchLabels:
app: pv-icql-devops-jenkins-data

# deployment
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-jenkins
namespace: icql-devops
labels:
app: deploy-jenkins
spec:
selector:
matchLabels:
app: pod-jenkins
replicas: 1
template:
metadata:
labels:
app: pod-jenkins
spec:
serviceAccountName: cluster-admin
containers:
- name: jenkins
image: jenkins/jenkins:latest
securityContext:
runAsUser: 0 # 设置以ROOT用户运行容器
privileged: true # 拥有特权
ports:
- name: http
containerPort: 8080
- name: jnlp
containerPort: 50000
resources:
limits:
memory: 512Mi
cpu: "200m"
requests:
memory: 256Mi
cpu: "100m"
livenessProbe:
tcpSocket:
port: 8080
initialDelaySeconds: 300
timeoutSeconds: 1
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
readinessProbe:
tcpSocket:
port: 8080
initialDelaySeconds: 5
timeoutSeconds: 1
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
divisor: 1Mi
- name: "JAVA_TOOL_OPTIONS"
value: "
-Dfile.encoding=UTF-8
-Dsun.jnu.encoding=UTF-8
"
- name: "JAVA_OPTS" # 设置变量,指定时区
value: "
-Xmx$(LIMITS_MEMORY)m
-Duser.timezone=Asia/Shanghai
"
volumeMounts:
# 挂载docker.sock 和 docker-bin,以便 jenkins 使用宿主机 docker 环境
- name: jenkins-docker-sock
mountPath: /var/run/docker.sock
- name: jenkins-docker-bin
mountPath: /usr/bin/docker
- name: jenkins-data
mountPath: /var/jenkins_home
volumes:
- name: jenkins-docker-sock
hostPath:
path: /var/run/docker.sock
- name: jenkins-docker-bin
hostPath:
path: /usr/bin/docker
- name: jenkins-data
persistentVolumeClaim:
claimName: pvc-jenkins

# service
---
kind: Service
apiVersion: v1
metadata:
name: svc-jenkins
namespace: icql-devops
labels:
app: svc-jenkins
spec:
ports:
- protocol: TCP
port: 8080
targetPort: 8080
selector:
app: pod-jenkins

# ingress
---
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
name: ingress-jenkins
namespace: icql-devops
labels:
app: ingress-jenkins
spec:
rules:
- host: devops.jenkins.icql.work
http:
paths:
- path: /
backend:
serviceName: svc-jenkins
servicePort: 8080

2)配置

根据提示找到初始化管理员密码,登录,首次启动安装插件建议选择无,跳过插件安装(国内速度奇慢)

(1)解决 jenkins 插件下载慢
1
2
3
4
5
6
(1)系统管理-插件管理-升级地址
https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json

(2)修改var/jenkins_home\updates\default.json
updates.jenkins-ci.org/download 替换为 mirrors.tuna.tsinghua.edu.cn/jenkins
www.google.com 替换为 www.baidu.com
(2)汉化,安装插件

Locale,Localization: Chinese (Simplified)

(3)安装pipeline

pipeline

2、frp

1
2
# 部署 icql-devops-frp
kubectl apply -f icql-devops-frp.yaml

1)部署

icql-devops-frp.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
# configmap
---
kind: ConfigMap
apiVersion: v1
metadata:
name: configmap-frp-conf
namespace: icql-devops
labels:
app: configmap-frp-conf
data:
frps.ini: |
[common]
token = 123456
bind_port = 8201
bind_udp_port = 8202
tcp_mux = false
log_file = /log/frps.log
log_level = info
log_max_days = 3
dashboard_port = 8200
dashboard_user = root
dashboard_pwd = 123456
allow_ports = 8203-8210

# deployment
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-frp
namespace: icql-devops
labels:
app: deploy-frp
spec:
selector:
matchLabels:
app: pod-frp
replicas: 1
template:
metadata:
labels:
app: pod-frp
spec:
serviceAccountName: cluster-admin
containers:
- name: frp
image: icql/frps:0.33.0
securityContext:
runAsUser: 0 # 设置以ROOT用户运行容器
privileged: true # 拥有特权
ports:
- containerPort: 8200
- containerPort: 8201
- containerPort: 8202
- containerPort: 8203
- containerPort: 8204
- containerPort: 8205
- containerPort: 8206
- containerPort: 8207
- containerPort: 8208
- containerPort: 8209
- containerPort: 8210
resources:
limits:
memory: 64Mi
cpu: "50m"
requests:
memory: 32Mi
cpu: "25m"
livenessProbe:
tcpSocket:
port: 8200
initialDelaySeconds: 300
timeoutSeconds: 1
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
readinessProbe:
tcpSocket:
port: 8200
initialDelaySeconds: 5
timeoutSeconds: 1
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
volumeMounts:
- name: frp-conf
mountPath: /conf/frps.ini
subPath: frps.ini
volumes:
- name: frp-conf
configMap:
name: configmap-frp-conf

# service
---
kind: Service
apiVersion: v1
metadata:
name: svc-frp
namespace: icql-devops
labels:
app: svc-frp
spec:
ports:
- name: dashboard
protocol: TCP
port: 8200
targetPort: 8200
- name: bind-tcp-port
protocol: TCP
port: 8201
targetPort: 8201
- name: bind-udp-port
protocol: UDP
port: 8202
targetPort: 8202
- name: port-8203
protocol: TCP
port: 8203
targetPort: 8203
- name: port-8204
protocol: TCP
port: 8204
targetPort: 8204
- name: port-8205
protocol: TCP
port: 8205
targetPort: 8205
- name: port-8206
protocol: TCP
port: 8206
targetPort: 8206
- name: port-8207
protocol: TCP
port: 8207
targetPort: 8207
- name: port-8208
protocol: TCP
port: 8208
targetPort: 8208
- name: port-8209
protocol: TCP
port: 8209
targetPort: 8209
- name: port-8210
protocol: TCP
port: 8210
targetPort: 8210
selector:
app: pod-frp

# ingress
---
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
name: ingress-frp
namespace: icql-devops
labels:
app: ingress-frp
spec:
rules:
- host: devops.frp.icql.work
http:
paths:
- path: /
backend:
serviceName: svc-frp
servicePort: 8200

部署 icql-svc 资源

1
2
# 创建 icql-svc 管理员账户
kubectl apply -f icql-svc-serviceaccount.yaml

icql-svc-serviceaccount.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# rbac
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cluster-admin
namespace: icql-svc
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: "icql-svc:cluster-admin" # cluter资源,所以加一个命名空间前缀区分
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
kind: ClusterRole
name: cluster-admin # 绑定集群管理员
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: cluster-admin
namespace: icql-svc

1、mysql

icql-svc-mysql.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
# 集群资源:mysql数据
---
kind: PersistentVolume
apiVersion: v1
metadata:
name: pv-icql-svc-mysql-data
labels:
app: pv-icql-svc-mysql-data
spec:
storageClassName: manual
capacity:
storage: 2Gi
accessModes:
- ReadWriteOnce
hostPath:
path: "/data/icql-svc/mysql/data"

# pvc绑定pv
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: pvc-mysql
namespace: icql-svc
labels:
app: pvc-mysql
spec:
storageClassName: manual
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 2Gi
selector:
matchLabels:
app: pv-icql-svc-mysql-data

# configmap
---
kind: ConfigMap
apiVersion: v1
metadata:
name: configmap-mysql-conf
namespace: icql-svc
labels:
app: configmap-mysql-conf
data:
my.cnf: |
[mysqld]
character-set-server=utf8
default-time-zone='+08:00'
[client]
default-character-set=utf8
[mysql]
default-character-set=utf8


# secret
---
kind: Secret
apiVersion: v1
metadata:
name: secret-mysql
namespace: icql-svc
labels:
app: secret-mysql
type: Opaque
data:
MYSQL_ROOT_PASSWORD: Q3FsamlhMTk5MzA2MTg=

# deployment
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-mysql
namespace: icql-svc
labels:
app: deploy-mysql
spec:
selector:
matchLabels:
app: pod-mysql
replicas: 1
template:
metadata:
labels:
app: pod-mysql
spec:
serviceAccountName: cluster-admin
containers:
- name: mysql
image: mysql:5.7
securityContext:
runAsUser: 0 # 设置以ROOT用户运行容器
privileged: true # 拥有特权
ports:
- containerPort: 3306
protocol: TCP
resources:
limits:
memory: 640Mi
cpu: "200m"
requests:
memory: 256Mi
cpu: "100m"
env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: secret-mysql
key: MYSQL_ROOT_PASSWORD
volumeMounts:
- name: mysql-data
mountPath: /var/lib/mysql
- name: mysql-conf
mountPath: /etc/mysql/my.cnf
subPath: my.cnf
volumes:
- name: mysql-data
persistentVolumeClaim:
claimName: pvc-mysql
- name: mysql-conf
configMap:
name: configmap-mysql-conf

# service
---
kind: Service
apiVersion: v1
metadata:
name: svc-mysql
namespace: icql-svc
labels:
app: svc-mysql
spec:
ports:
- protocol: TCP
port: 3306
targetPort: 3306
selector:
app: pod-mysql

部署 icql-static 资源:nginx

部署 icql-api 资源:spring-cloud

微信打赏

赞赏是不耍流氓的鼓励