es 定時備份 刪除 與數據恢復

1.建立快照倉庫

$ curl -u elastic:123456 -XPOST '192.168.43.104:9200/_snapshot/hsbc_backup' -H  'Content-Type: application/json' -d '{ "type": "fs",  "settings": { "location": "/opt/elasticsearch/elasticsearch-6.8.0/repo", "compress": true, "chunk_size": "1g", "max_snapshot_bytes_per_sec": "50m", "max_restore_bytes_per_sec": "50m"}}'

2.備份 並且保留60個備份記錄

#!/bin/bash

## define vars
es_url="http://192.168.43.104:9200"
es_bak_repo="hsbc_backup"
es_user="elastic"
es_pass="123456 "
es_bak_date=`date +'%Y-%m-%d-%H'`

echo -e "\n =========== $es_bak_date ============"

## bak
curl -u ${es_user}:${es_pass} -XPUT "${es_url}/_snapshot/${es_bak_repo}/es_bak_${es_bak_date}?wait_for_completion=true"
echo ""

##sleep
sleep 10

##del 最後的60: 定時任務 每4小時備份一次 一天備份6次  數據保留10天
es_del_snapshots=`curl -u ${es_user}:${es_pass} -s -XGET "${es_url}/_snapshot/${es_bak_repo}/_all" | python -m json.tool| python -c 'import json; import sys; input = sys.stdin.read(); result = [x["snapshot"] for x in json.loads(input)["snapshots"]]; print("\n".join(result[:-60]));'`
for es_del_snapshot in ${es_del_snapshots}; do
    echo "===========$es_del_snapshot del"
    curl -u ${es_user}:${es_pass} -s -XDELETE "${es_url}/_snapshot/${es_bak_repo}/${es_del_snapshot}"
    echo ""
done

3.建立定時任務

$ crontab -e
0 */4 * ** /bin/bash /opt/elasticsearch/scripts/es_crontab.sh >> /appvol/logs/crontab/es_bak/crontab.log 2>&1

4.將備份的數據恢復
在原來的機器(192.168.43.104)操作
(1)備份數據

# 檢視索引
$ curl -s -u elastic:123456 192.168.43.104:9200/_cat/indices?v |grep -v index| awk '{print $3}'
![檢視索引](https://s3.ap-northeast-1.wasabisys.com/img.tw511.com/202008/202008131431023st03luua33q.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N1bl94dWVnYW5n,size_16,color_FFFFFF,t_70#pic_center)

# 數據備份, 會將所有索引都備份
$ curl -u elastic:123456 -XPUT "http://192.168.43.104:9200/_snapshot/hsbc_backup/es_bak_20200811?wait_for_completion=true"

# 檢視備份檔案包含的索引
$ curl -s -u elastic:123456 -XGET http://192.168.43.104:9200/_snapshot/hsbc_backup/es_bak_20200811
![備份檔案包含的索引](https://s3.ap-northeast-1.wasabisys.com/img.tw511.com/202008/20200813143145165vvam3exqlum.png#pic_center)

(2)將備份檔案打包拷貝到192.168.43.105, 然後在192.168.43.105上解壓到es指定的repo.path下

在新的機器(192.168.43.105)操作

# 檢視存在的索引
![檢視已經存在的索引](https://s3.ap-northeast-1.wasabisys.com/img.tw511.com/202008/20200813143227123is5dadvhrr3.png#pic_center)


# 執行shell 進行數據恢復
$ sh es_restore.sh

# 再次檢視索引
![數據恢復後檢視索引](https://s3.ap-northeast-1.wasabisys.com/img.tw511.com/202008/20200813143249413wvpy105x3yv.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N1bl94dWVnYW5n,size_16,color_FFFFFF,t_70#pic_center)



##############  shell 指令碼
#!/bin/bash

set -e
## define vars
es_url="http://192.168.43.105:9200"
es_bak_repo="hsbc_backup"
es_restore_snapshots="es_bak_20200811"
es_user="elastic"
es_pass="123456"

##(1)獲取備份檔案中存在的索引
es_get_snapshots_index=`curl -u ${es_user}:${es_pass} -s -XGET "${es_url}/_snapshot/${es_bak_repo}/${es_restore_snapshots}" | python -m json.tool| python -c 'import json; import sys; input = sys.stdin.read(); result=json.loads(input); indices = result["snapshots"][0]["indices"]; sys.stdout.write("  ".join([i for i in indices]));'`

##(2)將備份檔案中存在的索引刪除  然後恢復
for index in ${es_get_snapshots_index}; do
    if [ ${index} == ".security-6" ]; then # 如果索引爲 .security-6 則不能刪除
        continue
    fi
    # 判斷索引是否存在
    res=`curl -s -u ${es_user}:${es_pass} -XGET "${es_url}/${index}"| grep "error" |grep 404|awk '{print $1}'`
    if [ -z ${res} ] ; then   #如果存在則刪除索引
        curl -s -u ${es_user}:${es_pass} -XDELETE ${es_url}/${index}
        echo -e "\ndelete index data : ${index} "
    fi

    #數據恢復
    echo -e "data restore : ${index}\n"
    curl -s -u ${es_user}:${es_pass} -XPOST "${es_url}/_snapshot/${es_bak_repo}/${es_restore_snapshots}/_restore" -H 'Content-Type: application/json' -d '{"indices": "'${index}'"}'
done