基于k8s、docker、jenkins构建springboot服务

Jenkins + github + docker + k8s + springboot

本文介绍基于k8s、docker、jenkins、springboot构建docker服务。

环境准备

server-1 k8s-master Centos7 ip地址10.12.5.110server-2 k8s-node Centos7 ip地址10.12.5.115

两台服务执行如下命令

$ setenforce 0$ systemctl stop firewalld$ systemctl disable firewalld

server-1 k8s-master 安装

k8s-master节点安装kubernets、docker、etcd、git、maven等软件。

安装docker

$ yum install docker// 修改docker配置文件 /etc/sysconfig/dockerOPTIONS='--registry-mirror=https://docker.mirrors.ustc.edu.cn --selinux-enabled --log-driver=journald --signature-verification=false'ADD_REGISTRY='--add-registry 10.12.5.110:5000'INSECURE_REGISTRY='--insecure-registry 10.12.5.110:5000'$ systemctl enable docker$ service docker start

安装registry

$ docker pull registry$ mkdir -p /data/docker/registry$ docker run -d -p 5000:5000 -v /data/docker/registry:/var/lib/registry registry

安装etcd

$ yum install etcd$ vi /etc/etcd/etcd.conf// 修改如下配置ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"$ systemctl enable etcd$ systemctl start etcd

安装kubernetes

$ yum install kubernetes// 修改配置文件apiserver$ vi /etc/kubernetes/apiserver// 修改如下配置KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"KUBE_API_PORT="--port=8080"KUBELET_PORT="--kubelet-port=10250"KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ResourceQuota"// 启动服务$ systemctl enable kube-apiserver kube-controller-manager kube-scheduler$ systemctl start kube-apiserver kube-controller-manager kube-scheduler

安装git和mvn

$ yum install git$ yum install maven

server-2 k8s-node 安装

k8s-master节点安装kubernets、docker等软件。

安装docker

$ yum install docker// 修改docker配置文件 /etc/sysconfig/dockerADD_REGISTRY='--add-registry 10.12.5.110:5000'INSECURE_REGISTRY='--insecure-registry 10.12.5.110:5000'$ systemctl enable docker$ systemctl start docker

安装kubernetes

$ yum install kubernetes// 修改配置文件kubelet$ vi /etc/kubernetes/kubelet// 修改如下配置KUBELET_ADDRESS="--address=0.0.0.0"KUBELET_PORT="--port=10250"KUBELET_HOSTNAME="--hostname-override=10.12.5.115"KUBELET_API_SERVER="--api-servers=http://10.12.5.110:8080"// 修改配置文件config$ vi /etc/kubernetes/config// 修改如下配置KUBE_MASTER="--master=http://10.12.5.110:8080"// 启动服务$ systemctl enable kube-proxy kubelet$ systemctl start kube-proxy kubelet

检查k8s集群运行情况

// 在k8s-master节点执行$ kubectl get nodesNAME          STATUS    AGE10.12.5.115   Ready     3m

在k8s-master节点builder springcloudenv镜像

* 下载jdk8$ mdkir -p /home/docker/docker-jdk8$ wget http://javadl.oracle.com/webapps/download/AutoDL?BundleId=233162_512cd62ec5174c3487ac17c61aaa89e8 -O jre-8u161-linux-x64.tar.gz$ tar zxvf jre-8u161-linux-x64.tar.gz* 删除jdk中用不到的文件，尽量减少docker镜像文件大小$ cd jre1.8.0_171$ rm -rf lib/plugin.jar lib/ext/jfxrt.jar bin/javaws lib/javaws.jar lib/desktop plugin lib/deploy* lib/*javafx* lib/*jfx* lib/amd64/libdecora_sse.so lib/amd64/libprism_*.so lib/amd64/libfxplugins.so lib/amd64/libglass.so lib/amd64/libgstreamer-lite.so lib/amd64/libjavafx*.so lib/amd64/libjfx*.so* 编写Dockerfile文件$ cd /home/docker/docker-jdk8$ vi DockerfileFROM centosMAINTAINER by wangtwRUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \    echo 'Asia/Shanghai' >/etc/timezone && \    yum -y install kde-l10n-Chinese && \    localedef -c -f UTF-8 -i zh_CN zh_CN.utf8 && \    mkdir -p /usr/javaCOPY jre1.8.0_171 /usr/java/jre1.8.0_171ENV LANG zh_CN.UTF-8ENV JAVA_HOME /usr/java/jre1.8.0_171ENV PATH $JAVA_HOME/bin:$PATH* 创建镜像springcloudenv，并上传到registry，确保docker registry已经启动$ docker build -t springcloudenv .$ docker tag springcloudenv springcloudenv:v1$ docker push springcloudenv:v1// 查看registry中的镜像$ curl http://10.12.5.110:5000/v2/_catalog{"repositories":["springcloudenv"]}

安装nfs

在k8s-master节点安装nfs server

$ yum install -y nfs-utils rpcbind$ mkdir -p /data/mysql-pv$ chmod 777 /data/mysql-pv/$ mkdir -p /data/nfs$ chmod 666 /data/nfs/$ vi /etc/exports// 增加如下行/data/nfs 10.12.0.0/16(rw,no_root_squash,no_all_squash,sync)/data/mysql-pv 10.12.0.0/16(rw,no_root_squash,no_all_squash,sync)$ systemctl enable rpcbind nfs$ systemctl start rpcbind nfs$ mkdir -p /data/mysql$ mount -t nfs 10.12.5.110:/data/nfs /data/mysql/ -o proto=tcp -o nolock

在k8s-node节点安装nfs

$ yum install -y nfs-utils$ mkdir -p /data/mysql$ mount -t nfs 10.12.5.110:/data/nfs /data/mysql/ -o proto=tcp -o nolock

安装mysql docker

以持久化方式运行myql

在k8s-master节点创建相关文件

$ mkdir -p /home/k8s/yml/services/mysql$ cd /home/k8s/yml/services/mysql$ vi mysql-pv.yamlapiVersion: v1kind: PersistentVolumemetadata:    name: mysql-pvspec:    accessModes:    - ReadWriteOnce    capacity:    storage: 1Gi    persistentVolumeReclaimPolicy: Retain#storageClassName: nfs    nfs:    path: /data/mysql-pv    server: 10.12.5.110$ vi mysql-pvc.yamlapiVersion: v1kind: PersistentVolumeClaimmetadata:    name: mysql-pvcspec:  accessModes:    - ReadWriteOnce  resources:    requests:      storage: 1Gi  #storageClassName: nfs$ vi mysql.yamlapiVersion: v1kind: Servicemetadata:  name: mysqlspec:  ports:  - port: 3306  selector:    app: mysql---apiVersion: extensions/v1beta1kind: Deploymentmetadata:  name: mysqlspec:  replicas: 1  selector:    matchLabels:      app: mysql  template:    metadata:      labels:        app: mysql    spec:      containers:      - name: mysql        image: mysql:5.6        env:        - name: MYSQL_ROOT_PASSWORD          value: password        ports:        - containerPort: 3306          name: mysql        volumeMounts:         - name: mysql-persistent-storage          mountPath: /var/lib/mysql      volumes:      - name: mysql-persistent-storage        persistentVolumeClaim:          claimName: mysql-pvc

创建mysql service和pod

$ cd /home/k8s/yml/services/mysql$ kubectl create -f mysq-pv.yaml$ kubectl get pvNAME       CAPACITY   ACCESSMODES   RECLAIMPOLICY   STATUS    CLAIM               REASON    AGEmysql-pv   1Gi        RWO           Retain          Bound     default/mysql-pvc             2d$ kubectl create -f mysq-pvc.yaml$ kubectl get pvcNAME        STATUS    VOLUME     CAPACITY   ACCESSMODES   AGEmysql-pvc   Bound     mysql-pv   1Gi        RWO           2d$ kubectl create -f mysq.yaml$ kubectl get serviceNAME         CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGEmysql        10.254.151.113   <none>        3306/TCP   2d$ kubectl get podNAME                           READY     STATUS             RESTARTS   AGEmysql-3827607452-hd5ct         1/1       Running            80         19h

k8s-master节点安装和配置jenkins

$ mkdir /home/jenkins$ cd$ vi .bash_profileexport JENKINS_HOME=/home/jenkins$ cd /home/jenkins$ wget http://mirrors.jenkins.io/war-stable/latest/jenkins.war$ java -jar jenkins.war --httpPort=9090&浏览器打开地址：http://10.12.5.110:9090把文件/home/jenkins/secrets/initialAdminPassword的内容复制到安装页面解决插件安装离线问题打开地址：http://10.12.5.110:9090/pluginManager/advanced把https://updates.jenkins.io/update-center.json修改为http://updates.jenkins.io/update-center.json安装插件：Publish Over SSH

创建jenkins任务,源代码在github上

* 任务名称：springboottest* 任务类型：自由风格* 创建github访问CredentialCredential类型：username with password* 源码管理 选择gitRepository URL 值为https://github.com/wangtiewu/springboottest.gitCredentials 值为上一步创建的 Credential* 创建构建脚本mvn clean packagetag=$( date +'%Y%m%d%H%M' )app=springcloudtestecho $tag > version_$app.txtdocker_name=$appdocker build -t $docker_name:$tag .docker push $docker_name:$tag* 创建ssh server系统管理-〉系统设置菜单-〉publish over ssh* 增加构建后操作，Send build artifacts over SSHSSH Server:name 值为k8s masterTransfer Set:Source files 值为version_springcloudtest.txtTransfer Set:Remote directory   值为springcloudtestTransfer Set:Exec command 值为cd /home/k8s/yml/services/springcloudtestapp=springcloudtestdeploy_file=$app.yamlversion_file=version_$app.txtpatch_file=$app.patchnew_version=$( cat $version_file )old_version=$( cat $deploy_file | grep image | awk -F ":" '{print $3}' )sed -i "s/$old_version/$new_version/" $deploy_filedeployment_num=$( /usr/bin/kubectl get deployment | grep springcloudtest | wc -l )if [ $deployment_num -eq 1 ];then    p_old_version=$( cat $patch_file | jq '.spec.template.spec.containers[0].image' )    p_new_version=\"$app:$new_version\"    sed -i "s/$p_old_version/$p_new_version/" $patch_file    patch=$( cat $patch_file )    /usr/bin/kubectl patch deployment $app -p $patchelse    /usr/bin/kubectl create -f $deploy_filefi

创建springclouttest service和pod

$ mkdir -p /home/k8s/yml/services/springcloudtest$ cd /home/k8s/yml/services/springcloudtest$ vi springcloudtest-svr.yamlapiVersion: v1kind: Servicemetadata:  name: springcloudtestspec:  ports:  - name: springcloudtest-svr    port: 9091    nodePort: 30000    targetPort: 9091  selector:    app: springcloudtest  type: NodePort$ vi springcloudtest.yamlapiVersion: extensions/v1beta1kind: Deploymentmetadata:  name: springcloudtestspec:  replicas: 1  template:    metadata:      labels:        app: springcloudtest    spec:      containers:      - name: springclouttest        image: springcloudtest:201805272108        ports:         - containerPort: 9091          protocol: TCP$ vi springcloudtest.patch{"spec":{"template":{"spec":{"containers":[{"name":"springclouttest","image":"springcloudtest:201805052039"}]}}}}

构建jenkins任务：springclouttest

构建完成后，kubectl get pod 查看springcloudtest-1053936621-w19sh   1/1       Running   1          1d

创建springclouttest service

$ cd /home/k8s/yml/services/springcloudtest$ kubectl create -f springcloudtest-svr.yaml查看服务状态$ kubectl get servicespringcloudtest   10.254.126.13   <nodes>       9091:30000/TCP   45m在k8s-node查看容器情况$ docker psCONTAINER ID        IMAGE                                                        COMMAND                  CREATED             STATUS              PORTS               NAMES269366fa1b49        mysql:5.6                                                    "docker-entrypoint..."   About an hour ago   Up About an hour                        k8s_mysql.affc4af4_mysql-3827607452-lp1cp_default_15ace557-60ed-11e8-aa6f-0800276c68b7_404d6f6a2ff21fb61984        springcloudtest:201805272108                                 "/bin/sh -c 'java ..."   About an hour ago   Up About an hour                        k8s_springclouttest.c0ee0b5d_springcloudtest-1053936621-w19sh_default_a320f883-61b2-11e8-aa6f-0800276c68b7_a99e1b3788e9ba47faf1        registry.access.redhat.com/rhel7/pod-infrastructure:latest   "/usr/bin/pod"           About an hour ago   Up About an hour                        k8s_POD.389b0ddb_mysql-3827607452-lp1cp_default_15ace557-60ed-11e8-aa6f-0800276c68b7_7387fdc6130a60acd94d        registry.access.redhat.com/rhel7/pod-infrastructure:latest   "/usr/bin/pod"           About an hour ago   Up About an hour                        k8s_POD.2aaa0bac_springcloudtest-1053936621-w19sh_default_a320f883-61b2-11e8-aa6f-0800276c68b7_fa8aba88解决 NodePort方式下,Node所在节点可以访问服务,节点外客户端请求服务都会失败问题$ iptables -P FORWARD ACCEPT

创建数据库和表

$ docker exec -it 269366fa1b49 /bin/bash$ mysql -uroot -p输入密码mysql>create database test1 default charset utf8 COLLATE utf8_general_ci;mysql>use test1mysql>create table HELLO(ID bigint, NAME varchar(128), primary key (ID));

测试服务

浏览器打开 http://10.12.5.115:30000/浏览器显示 Hello World!$ curl --data '{"id":1,"name":"Hello World"}' -H "Content-type: Application/json" http://10.12.5.115:30000/create/hello