Kubernetes+GitLab+Jenkins使用Pipeline构建镜像

GitLab和Jenkins都是作为Pod部署在kubernetes上的,Jenkins通过创建的pipeline流水线任务从gitlab拉取构建文件以进行构建。

注:Pod都使用了持久化存储来部署,同时使用的是私有仓库镜像。

部署GitLab

部署Redis

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# vim redis.yaml 
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: redis
namespace: default
spec:
serviceName: redis
replicas: 1
updateStrategy:
type: RollingUpdate
template:
metadata:
name: redis
spec:
terminationGracePeriodSeconds: 30
containers:
- name: redis
image: 192.168.100.100/library/redis:4
ports:
- containerPort: 6379
resources:
requests:
memory: "1Gi"
cpu: "1000m"
limits:
memory: "2Gi"
cpu: "2000m"

livenessProbe:
exec:
command:
- redis-cli
- ping
initialDelaySeconds: 30
timeoutSeconds: 5
readinessProbe:
exec:
command:
- redis-cli
- ping
initialDelaySeconds: 5
timeoutSeconds: 1

volumeMounts:
- name: redis
mountPath: /var/lib/redis
volumeClaimTemplates:
- metadata:
name: redis
annotations:
volume.beta.kubernetes.io/storage-class: "ceph-rbd"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 10Gi
---
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: default
spec:
ports:
- name: redis
port: 6379
targetPort: redis
selector:
name: redis
# kubectl create -f redis.yaml
statefulset.apps/redis created
service/redis created

部署PostgreSQL

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# vim postgresql.yaml 
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: postgresql
namespace: default
spec:
serviceName: postgresql
replicas: 1
updateStrategy:
type: RollingUpdate
template:
metadata:
name: postgresql
spec:
terminationGracePeriodSeconds: 30
containers:
- name: postgresql
image: 192.168.100.100/library/postgres:11
ports:
- containerPort: 5432
resources:
requests:
memory: "1Gi"
cpu: "1000m"
limits:
memory: "2Gi"
cpu: "2000m"

env:
- name: DB_USER
value: gitlab
- name: DB_PASS
value: passw0rd
- name: DB_NAME
value: gitlab_production
- name: DB_EXTENSION
value: pg_trgm

livenessProbe:
exec:
command:
- pg_isready
- -h
- localhost
- -U
- postgres
initialDelaySeconds: 30
timeoutSeconds: 5
readinessProbe:
exec:
command:
- pg_isready
- -h
- localhost
- -U
- postgres
initialDelaySeconds: 5
timeoutSeconds: 1

volumeMounts:
- name: postgresql
mountPath: /var/lib/postgresql
volumeClaimTemplates:
- metadata:
name: postgresql
annotations:
volume.beta.kubernetes.io/storage-class: "ceph-rbd"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 10Gi
---
apiVersion: v1
kind: Service
metadata:
name: postgresql
namespace: default
spec:
ports:
- name: postgresql
port: 5432
targetPort: postgresql
selector:
name: postgresql
# kubectl create -f postgresql.yaml
statefulset.apps/postgresql created
service/postgresql created

部署GitLab

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
# vim gitlab.yaml 
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: gitlab
namespace: default
spec:
serviceName: gitlab
replicas: 1
updateStrategy:
type: RollingUpdate
template:
metadata:
name: gitlab
labels:
name: gitlab
spec:
terminationGracePeriodSeconds: 30
containers:
- name: gitlab
image: 192.168.100.100/gitlab/gitlab-ce:11.7.5-ce
resources:
requests:
memory: "1Gi"
cpu: "1000m"
limits:
memory: "4Gi"
cpu: "2000m"

env:
- name: TZ
value: Asia/BeiJing
- name: GITLAB_TIMEZONE
value: BeiJing

- name: GITLAB_SECRETS_DB_KEY_BASE
value: rVhTJmMMnn3RCwxXV7HCTNTkJXWLPjJhNHM3sWTgKwnqnNVTwNCVJ7zPLzKgVz7z
- name: GITLAB_SECRETS_SECRET_KEY_BASE
value: VkT7bxtXRsgvLpqRgbVnJKfjTMs7TsWrkXfmntPvbNWRLc4dxK9cWkLNpnNNvmsM
- name: GITLAB_SECRETS_OTP_KEY_BASE
value: Xgksv7WKTvWVrNm9vcvbdgMnVqXLPb3F9XvbFLXNLTTXtzcvJgp7nTrdbxN444Jt

- name: GITLAB_ROOT_PASSWORD
value: wangzhijian
- name: GITLAB_ROOT_EMAIL
value: wangzhijiansd@qq.com

- name: GITLAB_HOST
value: git.default
- name: GITLAB_PORT
value: "80"
- name: GITLAB_SSH_PORT
value: "22"

- name: GITLAB_NOTIFY_ON_BROKEN_BUILDS
value: "true"
- name: GITLAB_NOTIFY_PUSHER
value: "false"

- name: GITLAB_BACKUP_SCHEDULE
value: daily
- name: GITLAB_BACKUP_TIME
value: 01:00

- name: DB_TYPE
value: postgresql
- name: DB_HOST
value: postgresql
- name: DB_PORT
value: "5432"
- name: DB_USER
value: gitlab
- name: DB_PASS
value: passw0rd
- name: DB_NAME
value: gitlab_production

- name: REDIS_HOST
value: redis
- name: REDIS_PORT
value: "6379"

- name: SMTP_ENABLED
value: "true"
- name: SMTP_DOMAIN
value: www.flywzj.com
- name: SMTP_HOST
value: smtp.qq.com
- name: SMTP_PORT
value: "465"
- name: SMTP_USER
value: wangzhijiansd@qq.com
- name: SMTP_PASS
value: "********"
- name: SMTP_STARTTLS
value: "true"
- name: SMTP_AUTHENTICATION
value: login

- name: IMAP_ENABLED
value: "false"
- name: IMAP_HOST
value: imap.gmail.com
- name: IMAP_PORT
value: "993"
- name: IMAP_USER
value: mailer@example.com
- name: IMAP_PASS
value: password
- name: IMAP_SSL
value: "true"
- name: IMAP_STARTTLS
value: "false"

ports:
- name: http
containerPort: 80
- name: ssh
containerPort: 22

volumeMounts:
- name: gitlab-data
mountPath: /var/opt/gitlab
- name: gitlab-config
mountPath: /etc/gitlab
- name: gitlab-logs
mountPath: /var/log/gitlab
volumeClaimTemplates:
- metadata:
name: gitlab-data
annotations:
volume.beta.kubernetes.io/storage-class: "ceph-rbd"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 10Gi
- metadata:
name: gitlab-config
annotations:
volume.beta.kubernetes.io/storage-class: "ceph-rbd"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 100Mi
- metadata:
name: gitlab-logs
annotations:
volume.beta.kubernetes.io/storage-class: "ceph-rbd"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 100Mi
# vim gitlab-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: gitlab
namespace: default
spec:
type: LoadBalancer
ports:
- name: http
port: 80
targetPort: http
- name: ssh
port: 22
targetPort: ssh
selector:
name: gitlab
# kubectl create -f gitlab.yaml
statefulset.apps/gitlab created
# kubectl create -f gitlab-svc.yaml
service/gitlab created

关于变量的赋予请参考:https://github.com/sameersbn/docker-gitlab

如需进行特殊配置,请参考gitlab相关文档将相应配置写入gitlab.rb并使用configmap将其挂载到/etc/gitlab/目录下

部署Jenkins

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# vim jenkins.yaml
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: jenkins
spec:
serviceName: jenkins
replicas: 1
updateStrategy:
type: RollingUpdate
template:
metadata:
name: jenkins
labels:
name: jenkins
spec:
terminationGracePeriodSeconds: 10
serviceAccountName: jenkins
containers:
- name: jenkins
image: 192.168.100.100/jenkins/jenkins:lts
imagePullPolicy: Always
ports:
- containerPort: 8080
- containerPort: 50000
resources:
limits:
cpu: 2
memory: 2Gi
requests:
cpu: 1
memory: 1Gi
env:
- name: LIMITS_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
divisor: 1Mi
- name: JAVA_OPTS
value: -Duser.timezone=Asia/Shanghai -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
volumeMounts:
- name: jenkins-home
mountPath: /var/jenkins_home
livenessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12 # ~2 minutes
securityContext:
fsGroup: 1000
volumeClaimTemplates:
- metadata:
name: jenkins-home
annotations:
volume.beta.kubernetes.io/storage-class: "ceph-rbd"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 10Gi
---
apiVersion: v1
kind: Service
metadata:
name: jenkins
namespace: default
spec:
type: LoadBalancer
ports:
- name: http
port: 8080
targetPort: 8080
- name: agent
port: 50000
targetPort: 50000
selector:
name: jenkins
# kubectl apply -f jenkins.yaml
statefulset.apps/jenkins configured
service/jenkins configured

配置RBAC

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# vim jenkins-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: jenkins
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: jenkins
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get","list","watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: jenkins
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: jenkins
subjects:
- kind: ServiceAccount
name: jenkins
# kubectl apply -f jenkins-rbac.yaml
serviceaccount/jenkins configured
role.rbac.authorization.k8s.io/jenkins configured
rolebinding.rbac.authorization.k8s.io/jenkins configured
# kubectl create clusterrolebinding cluster-admin-jenkins --clusterrole=cluster-admin --serviceaccount=default:default

查看相关部署情况

查看 StatefulSet

1
2
3
4
5
6
# kubectl get statefulset
NAME READY AGE
gitlab 1/1 41h
jenkins 1/1 41h
postgresql 1/1 41h
redis 1/1 41h

查看 PV和PVC

1
2
3
4
5
6
7
8
9
10
11
12
# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-18ec78b1-4ee5-11e9-b373-000c292a7b79 10Gi RWO Delete Bound default/redis-redis-0 ceph-rbd 41h
pvc-48a7f8c4-4ee9-11e9-b373-000c292a7b79 10Gi RWO Delete Bound default/postgresql-postgresql-0 ceph-rbd 41h
pvc-49035b17-4eeb-11e9-b373-000c292a7b79 10Gi RWO Delete Bound default/gitlab-gitlab-0 ceph-rbd 41h
pvc-b5bbbd9b-4af6-11e9-9bd3-000c292a7b79 10Gi RWO Delete Bound default/jenkins-home-jenkins-0 ceph-rbd 41h
# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
gitlab-gitlab-0 Bound pvc-49035b17-4eeb-11e9-b373-000c292a7b79 10Gi RWO ceph-rbd 41h
jenkins-home-jenkins-0 Bound pvc-b5bbbd9b-4af6-11e9-9bd3-000c292a7b79 10Gi RWO ceph-rbd 41h
postgresql-postgresql-0 Bound pvc-48a7f8c4-4ee9-11e9-b373-000c292a7b79 10Gi RWO ceph-rbd 41h
redis-redis-0 Bound pvc-18ec78b1-4ee5-11e9-b373-000c292a7b79 10Gi RWO ceph-rbd 41h

查看 Pod

1
2
3
4
5
6
# kubectl get pod
NAME READY STATUS RESTARTS AGE
gitlab-0 1/1 Running 0 41h
jenkins-0 1/1 Running 0 41h
postgresql-0 1/1 Running 0 41h
redis-0 1/1 Running 0 41h

查看 Service

1
2
3
4
5
6
# kubectl get services
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
gitlab LoadBalancer 10.96.131.243 <pending> 80:30284/TCP,22:32377/TCP 41h
jenkins LoadBalancer 10.106.240.93 <pending> 8080:31906/TCP,50000:31611/TCP 41h
postgresql ClusterIP 10.106.35.107 <none> 5432/TCP 41h
redis ClusterIP 10.99.125.252 <none> 6379/TCP 41h

配置Jenkins

当您使用nodeport方式打开Jenkins的时候,首先会出现“Unlocking Jenkins”,这里的“Administrator password”存放在logs中,你可以通过查看日志获取,也可通过如下命令获取:

1
2
# kubectl exec -it jenkins-0 cat /var/jenkins_home/secrets/initialAdminPassword
1f297e8179664848a3e0b672ceafce1f

在之后会提示您安装插件(这里我安装了推荐插件)和配置管理员账号,请自行安装和配置。

登录后,在“系统管理”===>“插件管理”===>“available”(可用插件)===>安装Kubernetes和GitLab插件

关于Gitlab插件的配置

在“系统管理”===>“系统设置”下进行设置

ops01

配置完成后点击“Test Connection”进行测试,如出现“Success”即为成功

关于GitLab API token

登录相关Gitlab账户,在该用户的“Setting”===>“Access Tokens”下添加个人访问令牌的名称、到期时间以及授予的权限,如下所示:
ops03

之后会生成个人访问令牌,请注意保存,之后将无法再查看该token。将该token复制到Jenkins以添加凭据
ops04

关于kubernetes插件的配置

在“系统管理”===>“系统设置”===>拖到最后找到“云”===>“新增一个云”

ops17

点击“Test Connection”,如出现“Connection test successful”则说明 Jenkins已经可以和 Kubernetes 通信了

关于添加凭据

Jenkins支持添加多种类型凭据,除了上面介绍的“Gitlab API token”还有“SSH Username with private key”、“Certificate”以及常见的用户名密码类型,这里说的就是用户名密码类型,这里添加harbor镜像仓库以及Gitlab的用户名密码以备使用。

ops18
ops19

使用 GitLab 新建项目

登录Gitlab

使用nodeport方式打开Gitlab,然后使用root账户登录,在“Admin Area”的“Settings”下配置“Network”,使其允许从hooks和 services向local network发出请求。

ops05

新建Gitlab账户

ops06

新建项目

ops07

写一个简单的dockerfile

ops08

使用 Jenkins 新建任务

新建pipeline任务

ops09

写一个pipeline

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
def label = "docker-${UUID.randomUUID().toString()}"

podTemplate(label: label, yaml: """
apiVersion: v1
kind: Pod
spec:
containers:
- name: docker
image: 192.168.100.100/docker/docker:18
command: ['cat']
tty: true
volumeMounts:
- name: dockersock
mountPath: /var/run/docker.sock
volumes:
- name: dockersock
hostPath:
path: /var/run/docker.sock
"""
) {

def image = "192.168.100.100/library/alpine:test"
node(label) {
stage('Git') {
git credentialsId: 'gitlab',url:'http://gitlab/zhi/alpine.git'
}
stage('Build Docker image')
withCredentials([
usernamePassword(
credentialsId: 'harbor',
passwordVariable: 'PASSWORD',
usernameVariable: 'USERNAME')]) {
container('docker') {
sh "docker build -t ${image} ."
sh "docker login -u '$USERNAME' -p '$PASSWORD' 192.168.100.100"
sh "docker push ${image}"
sh "docker rmi ${image}"
}
}
}
}

注意:该流水线是参照jenkinsci-kubernetes-plugin而来。

关于该流水线的简单解释:

  • podTemplate: 用于创建代理的pod的模板
  • label: pod的标签。设置唯一值以避免跨构建的冲突
  • containers: 用于创建pod的容器模板
  • containerTemplate: 将添加到pod中的容器模板
  • name: 容器名
  • image: 容器镜像
  • command: 容器将执行的命令
  • tty: true,启用TTY
  • volumes: 为pod挂载相应卷
  • node: 选择相应标签
  • stage: 构建阶段,这将显性的显示在“Stage View”下
  • withCredentials: 将凭据绑定到变量
  • usernamePassword: 将一个变量设置为用户名,将另一个变量设置为凭据中给出的密码
  • credentialsId: 设置凭据ID
  • passwordVariable: 在构建期间要设置为密码的环境变量的名称
  • usernameVariable: 构建期间要设置为用户名的环境变量的名称

进行构建

点击“立即构建”进行构建

ops10

定时构建

ops11

构建语法

  • 第一个*表示分钟,取值0~59
  • 第二个*表示小时,取值0~23
  • 第三个*表示一个月的第几天,取值1~31
  • 第四个*表示第几月,取值1~12
  • 第五个*表示一周中的第几天,取值0~7,其中0和7代表的都是周日

触发构建

在该任务下启用触发器

勾选将更改推送到GitLab时构建,点击高级生成“Secret token”

ops12

在Gitlab下添加Webhooks

在该项目下设置“Integrations”,将Jenkins的配置配置在“URL”和“Secret Token”项下,根据情况勾选相应触发项

ops13

在Gitlab下更新该项目

ops14

查看Jenkins是否触发了构建

ops15

关于在Jenkins中使用kubectl命令

注: 在上面的Jenkins的部署中已经做好了相关权限的分配

重构jnlp-slave

注: docker安装包和kubectl命令请自行下载至构建目录下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#  mkdir jnlp && cd jnlp
# cp .kube/config .
# ls
config docker-18.06.1-ce.tgz Dockerfile kubectl
# cat Dockerfile
FROM jenkins/jnlp-slave:3.27-1-alpine
MAINTAINER zhi <wangzhijiansd@qq.com>

USER root
ARG DOCKER_GID=994
ENV DOCKER_VERSION=18.06.1-ce

COPY docker-${DOCKER_VERSION}.tgz /var/tmp/
RUN tar --strip-components=1 -xvzf /var/tmp/docker-${DOCKER_VERSION}.tgz -C /usr/local/bin \
&& rm -rf /var/tmp/docker-${DOCKER_VERSION}.tgz \
&& chmod -R 775 /usr/local/bin/docker

COPY kubectl /usr/local/bin/

RUN mkdir -p /root/.kube/
COPY config /root/.kube/

RUN addgroup -g ${DOCKER_GID} docker && adduser jenkins docker

USER jenkins:${DOCKER_GID}
# docker build -t 192.168.100.100/jenkins/jnlp-slave:latest .
# docker push 192.168.100.100/jenkins/jnlp-slave:latest

编写一个pipeline

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
podTemplate(label: 'mypod', cloud: 'kubernetes', containers: [
containerTemplate(
name: 'jnlp',
image: '192.168.100.100/jenkins/jnlp-slave:latest',
alwaysPullImage: true,
args: '${computer.jnlpmac} ${computer.name}'),
],
volumes: [
hostPathVolume(
mountPath: '/var/run/docker.sock',
hostPath: '/var/run/docker.sock'),
],)
{
node('mypod') {
stage('Run shell') {
sh 'kubectl get pod -n kube-system'
}
}
}

进行构建

ops16

附:

ZhiJian wechat
欢迎您扫一扫上面的二维码,订阅我的微信公众号!
-------------本文结束,感谢您的阅读-------------