k8s部署canal-1.1.6版本實現MySQL資料庫資料同步

---
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: canal-deployer
  name: canal-deployer
  namespace: canal-mysql
spec:
  replicas: 1
  minReadySeconds: 10
  selector:
    matchLabels:
      app: canal-deployer
  template:
    metadata:
      name: canal-deployer
      labels:
        app: canal-deployer
    spec:
      volumes:
        - name: conf
          configMap:
            defaultMode: 493
            name: canal-deployer-configmap
      containers:
      - name: canal-deployer
        image: canal/canal-server:latest
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - mountPath: /home/admin/canal-server/conf/canal.properties
          name: conf
          subPath: canal.properties
        - mountPath: /home/admin/canal-server/conf/example/instance.properties
          name: conf
          subPath: instance.properties
        - mountPath: /home/admin/app.sh
          name: conf
          subPath: app.sh
        ports:
        - containerPort: 11111
          protocol: TCP
        - containerPort: 11112
          protocol: TCP

---
kind: Service
apiVersion: v1
metadata:
  name: canal-deployer
  namespace: canal-mysql
  labels:
    app: canal-deployer
spec:
  ports:
  - port: 11111
    name: deployer
  - port: 11112
    name: metrics
  clusterIP: None
  selector:
    app: canal-deployer

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: canal-deployer-configmap
  namespace: canal-mysql
data:
  app.sh: |-
    #!/bin/bash
    set -e

    source /etc/profile
    export JAVA_HOME=/usr/java/latest
    export PATH=$JAVA_HOME/bin:$PATH
    touch /tmp/start.log
    host=`hostname -i`

    # waitterm
    #   wait TERM/INT signal.
    #   see: http://veithen.github.io/2014/11/16/sigterm-propagation.html
    waitterm() {
            local PID
            # any process to block
            tail -f /dev/null &
            PID="$!"
            # setup trap, could do nothing, or just kill the blocker
            trap "kill -TERM ${PID}" TERM INT
            # wait for signal, ignore wait exit code
            wait "${PID}" || true
            # clear trap
            trap - TERM INT
            # wait blocker, ignore blocker exit code
            wait "${PID}" 2>/dev/null || true
    }

    # waittermpid "${PIDFILE}".
    #   monitor process by pidfile && wait TERM/INT signal.
    #   if the process disappeared, return 1, means exit with ERROR.
    #   if TERM or INT signal received, return 0, means OK to exit.
    waittermpid() {
            local PIDFILE PID do_run error
            PIDFILE="${1?}"
            do_run=true
            error=0
            trap "do_run=false" TERM INT
            while "${do_run}" ; do
                    PID="$(cat "${PIDFILE}")"
                    if ! ps -p "${PID}" >/dev/null 2>&1 ; then
                            do_run=false
                            error=1
                    else
                            sleep 1
                    fi
            done
            trap - TERM INT
            return "${error}"
    }

    function checkStart() {
        local name=$1
        local cmd=$2
        local timeout=$3
        cost=5
        while [ $timeout -gt 0 ]; do
            ST=`eval $cmd`
            if [ "$ST" == "0" ]; then
                sleep 1
                let timeout=timeout-1
                let cost=cost+1
            elif [ "$ST" == "" ]; then
                sleep 1
                let timeout=timeout-1
                let cost=cost+1
            else
                break
            fi
        done
        echo "start $name successful"
    }

    function start_canal() {
        echo "start canal ..."
        managerAddress=`perl -le 'print $ENV{"canal.admin.manager"}'`
        if [ ! -z "$managerAddress" ] ; then
            # canal_local.properties mode
            adminPort=`perl -le 'print $ENV{"canal.admin.port"}'`
            if [ -z "$adminPort" ] ; then
                adminPort=11110
            fi

            su root -c 'cd /home/admin/canal-server/bin/ && sh restart.sh local 1>>/tmp/start.log 2>&1'
            sleep 5
            #check start
            checkStart "canal" "nc 127.0.0.1 $adminPort -w 1 -z | wc -l" 30
        else
            metricsPort=`perl -le 'print $ENV{"canal.metrics.pull.port"}'`
            if [ -z "$metricsPort" ] ; then
                metricsPort=11112
            fi

            destination=`perl -le 'print $ENV{"canal.destinations"}'`
            if [[ "$destination" =~ ',' ]]; then
                echo "multi destination:$destination is not support"
                exit 1;
            else
                if [ "$destination" != "" ] && [ "$destination" != "example" ] ; then
                    if [ -d /home/admin/canal-server/conf/example ]; then
                        mv /home/admin/canal-server/conf/example /home/admin/canal-server/conf/$destination
                    fi
                fi 
            fi

            su root -c 'cd /home/admin/canal-server/bin/ && sh restart.sh 1>>/tmp/start.log 2>&1'
            sleep 5
            #check start
            checkStart "canal" "nc 127.0.0.1 $metricsPort -w 1 -z | wc -l" 30
        fi  
    }

    function stop_canal() {
        echo "stop canal"
        su root -c 'cd /home/admin/canal-server/bin/ && sh stop.sh 1>>/tmp/start.log 2>&1'
        echo "stop canal successful ..."
    }

    function start_exporter() {
        su root -c 'cd /home/admin/node_exporter && ./node_exporter 1>>/tmp/start.log 2>&1 &'
    }

    function stop_exporter() {
        su root -c 'killall node_exporter'
    }

    echo "==> START ..."

    start_exporter
    start_canal

    echo "==> START SUCCESSFUL ..."

    tail -f /dev/null &
    # wait TERM signal
    waitterm

    echo "==> STOP"

    stop_canal
    stop_exporter

    echo "==> STOP SUCCESSFUL ..."
  canal.properties: >-
    canal.ip =
    canal.register.ip =
    canal.port = 11111
    canal.metrics.pull.port = 11112
    # 修改點: 此處值修改為部署的zookeeper服務的地址
    canal.zkServers = zookeeper:2181
    canal.withoutNetty = false
    canal.serverMode = tcp
    canal.destinations = example
    canal.auto.scan = true
    
    canal.instance.detecting.enable = false
    canal.instance.detecting.sql = select 1
    canal.instance.detecting.interval.time = 3
    canal.instance.detecting.retry.threshold = 3
    canal.instance.detecting.heartbeatHaEnable = false
    
    canal.instance.transaction.size =  1024
    canal.instance.fallbackIntervalInSeconds = 60
    
    canal.instance.network.receiveBufferSize = 16384
    canal.instance.network.sendBufferSize = 16384
    canal.instance.network.soTimeout = 30
    
    canal.instance.filter.druid.ddl = true
    canal.instance.filter.query.dcl = false
    canal.instance.filter.query.dml = false
    canal.instance.filter.query.ddl = false
    canal.instance.filter.table.error = false
    canal.instance.filter.rows = false
    canal.instance.filter.transaction.entry = false
    canal.instance.filter.dml.insert = false
    canal.instance.filter.dml.update = false
    canal.instance.filter.dml.delete = false
    
    canal.conf.dir = ../conf
    canal.instance.binlog.format = ROW,STATEMENT,MIXED
    canal.instance.binlog.image = FULL,MINIMAL,NOBLOB
    canal.instance.get.ddl.isolation = false
    canal.instance.parser.parallel = true
    canal.instance.parser.parallelThreadSize = 16
    
    canal.instance.tsdb.enable = false
    canal.instance.tsdb.dir = ${canal.conf.dir}/tsdb
    canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
    canal.instance.tsdb.dbUsername = canal
    canal.instance.tsdb.dbPassword = canal
    canal.instance.tsdb.snapshot.interval = 24
    canal.instance.tsdb.snapshot.expire = 360
    canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
    
    canal.instance.global.mode = spring
    canal.instance.global.lazy = false
    canal.instance.global.spring.xml = classpath:spring/default-instance.xml
  instance.properties: |-
    canal.instance.tsdb.enable = true
    canal.instance.gtidon = false
    # 修改點：將此處值修改為源庫的地址
    canal.instance.master.address = 192.168.91.131:3306
    canal.instance.master.journal.name =
    canal.instance.master.position =
    canal.instance.master.timestamp=
    canal.instance.master.gtid=
    # 修改點: 將此處的使用者名稱、密碼修改為源庫對應的使用者名稱、密碼
    canal.instance.dbUsername = root
    canal.instance.dbPassword = root@123456
    canal.instance.connectionCharset = UTF-8
    canal.instance.enableDruid = false
    canal.instance.filter.regex=.*\\..*
    # 過濾控制檯列印與設定表無關的紀錄檔資訊
    canal.instance.filter.black.regex=mysql\\.slave_.*
    canal.mq.topic = example
    canal.mq.partition = 0

---
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: canal-adapter
  name: canal-adapter
  namespace: canal-mysql
spec:
  replicas: 1
  minReadySeconds: 10
  selector:
    matchLabels:
      app: canal-adapter
  template:
    metadata:
      name: canal-adapter
      labels:
        app: canal-adapter
    spec:
      volumes:
      - name: conf
        defaultMode: 420
        configMap:
          name: canal-adapter-configmap
      containers:
      - name: canal-adapter
        image: funnyzak/canal-adapter:1.1.6
        imagePullPolicy: IfNotPresent
        volumeMounts:
          - mountPath: /opt/canal/canal-adapter/conf/application.yml
            name: conf
            subPath: application.yml
          - mountPath: /opt/canal/canal-adapter/conf/bootstrap.yml
            name: conf
            subPath: bootstrap.yml
          - mountPath: /opt/canal/canal-adapter/conf/rdb/monitoralter.yml
            name: conf
            subPath: monitoralter.yml
        ports:
        - containerPort: 8081
          protocol: TCP
 
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: canal-adapter-configmap
  namespace: canal-mysql
data:
  application.yml: |-
    server:
      port: 8081
    logging:
      level:
        org.springframework: WARN
        com.alibaba.otter.canal.client.adapter.rdb: WARN
    spring:
      jackson:
        date-format: yyyy-MM-dd HH:mm:ss
        time-zone: GMT+8
        default-property-inclusion: non_null
    canal.conf:
      mode: tcp
      flatMessage: true
      # 修改點：zookeeper服務對應的地址
      zookeeperHosts: zookeeper:2181
      syncBatchSize: 1000
      retries: 0
      timeout:
      accessKey:
      secretKey:
      consumerProperties:
        # 修改點：由於採用tcp方式，所以這裡設定就是tcp方式下canal deployer對應的地址
        canal.tcp.server.host: canal-deployer:11111
        canal.tcp.zookeeper.hosts:
        canal.tcp.batch.size: 500
        canal.tcp.username:
        canal.tcp.password:
      # 修改點：源庫資料來源設定資訊
      srcDataSources:
        monitorAlterDS:
          url: jdbc:mysql://192.168.91.131:3306/monitor_alert?useUnicode=true&characterEncoding=utf8&autoReconnect=true&useSSL=false
          username: root
          password: root@123456
      # 修改點：目標庫資料來源設定資訊
      canalAdapters:
      - instance: example
        groups:
        - groupId: g1
          outerAdapters:
          - name: logger
          - name: rdb
            key: mysql1
            properties:
              jdbc.driverClassName: com.mysql.jdbc.Driver
              jdbc.url: jdbc:mysql://192.168.91.135:3306/monitor_alert?useUnicode=true&characterEncoding=utf8&autoReconnect=true&useSSL=false
              jdbc.username: root
              jdbc.password: admin@4321
  bootstrap.yml:
  # 修改點：映象方式同步組態檔
  monitoralter.yml: |-
    dataSourceKey: monitorAlterDS
    destination: example
    groupId: g1
    outerAdapterKey: mysql1
    concurrent: true
    dbMapping:
      mirrorDb: true
      database: monitor_alert

## 例如：需要再同步product資料庫下的表資料
# Deployment檔案修改如下：
# 1、canal adapter的Deployment檔案新增掛載的product資料庫資訊
containers:
- name: canal-adapter
  image: funnyzak/canal-adapter:1.1.6
  imagePullPolicy: IfNotPresent
  volumeMounts:
    - mountPath: /opt/canal/canal-adapter/conf/application.yml
      name: conf
      subPath: application.yml
    - mountPath: /opt/canal/canal-adapter/conf/bootstrap.yml
      name: conf
      subPath: bootstrap.yml
    - mountPath: /opt/canal/canal-adapter/conf/rdb/monitoralter.yml
      name: conf
      subPath: monitoralter.yml
    - mountPath: /opt/canal/canal-adapter/conf/rdb/product.yml
      name: conf
      subPath: product.yml

# configMap修改如下：
# 1、application.yml檔案新增源庫、目標庫的地址資訊
# 源庫資訊
srcDataSources:
  monitorAlterDS:
    url: jdbc:mysql://192.168.91.131:3306/monitor_alert?useUnicode=true&characterEncoding=utf8&autoReconnect=true&useSSL=false
    username: root
    password: root@123456
  productDS:
    url: jdbc:mysql://192.168.91.131:3306/product?useUnicode=true&characterEncoding=utf8&autoReconnect=true&useSSL=false
    username: root
    password: root@123456
# 目標庫資訊
canalAdapters:
- instance: example
  groups:
  - groupId: g1
    outerAdapters:
    - name: logger
    - name: rdb
      key: mysql1
      properties:
        jdbc.driverClassName: com.mysql.jdbc.Driver
        jdbc.url: jdbc:mysql://192.168.91.135:3306/monitor_alert?useUnicode=true&characterEncoding=utf8&autoReconnect=true&useSSL=false
        jdbc.username: root
        jdbc.password: admin@4321
     - name: rdb
       key: mysql2
       properties:
         jdbc.driverClassName: com.mysql.jdbc.Driver
         jdbc.url: jdbc:mysql://192.168.91.135:3306/product?useUnicode=true&characterEncoding=utf8&autoReconnect=true&useSSL=false
         jdbc.username: root
         jdbc.password: admin@4321
# 2、新增對應的掛載組態檔product.yml
product.yml: |-
  dataSourceKey: productDS
  destination: example
  groupId: g1
  outerAdapterKey: mysql2
  concurrent: true
  dbMapping:
    mirrorDb: true
    database: product           

---
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: zk-deployment
  name: zookeeper
  namespace: canal-mysql
spec:
  replicas: 2
  selector:
    matchLabels:
      app: zk
  template:
    metadata:
      name: zk
      labels:
        app: zk
    spec:
      volumes:
      - name: localtime
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
      containers:
      - name: zookeeper
        image: zookeeper:3.6.2
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: localtime
          mountPath: /etc/localtime
          readOnly: true

---
kind: Service
apiVersion: v1
metadata:
  name: zookeeper
  namespace: canal-mysql
  labels:
    app: zk
spec:
  ports:
  - port: 2181
    name: client
  clusterIP: None
  selector:
    app: zk