kong+elasticsearch監控所有請求
2020-08-12 14:32:39
kong+elasticsearch監控所有請求
Kong文章系列
上面是寫了一些Kong基礎使用的文章，如果對Kong還不太熟悉可以參考閱讀。
本篇文章的內容是使用elasticsearch記錄Kong的所有請求，最終目的是通過elasticsearch + kibana對所有請求做進一步分析，不過這篇文章暫時沒有kibana部分，因爲筆者還沒有研究，所以這篇文章主要是將kong的請求資訊轉存到elasticsearch.
目標
- 通過Kong提供的外掛【HTTP Log】，將指定的Service或者Route的請求轉發到指定的HTTP URL
- 構建Web服務，接收Kong轉發的請求資訊，並附加其他資訊儲存(如果是對Kong比較熟，可以直接定製HTTP Log外掛)
- 將構建的請求資訊儲存到elsticsearch
- 通過kibana對數據進行分析
原理
操作步驟
設定HttpLog外掛
給指定的service設定http-log
curl -X POST http://localhost:8001/services/c9beb237-7e85-49b3-8561-0082fc8086ea/plugins \
--data "name=http-log" \
--data "config.http_endpoint=http://172.11.22.10:10150/http_log/post" \ # 設定的代理地址
--data "config.method=POST" \
--data "config.timeout=1000" \
--data "config.keepalive=1000"
請求發送的json數據爲
{
"request": {
"method": "GET",
"uri": "/get",
"url": "http://httpbin.org:8000/get",
"size": "75",
"querystring": {},
"headers": {
"accept": "*/*",
"host": "httpbin.org",
"user-agent": "curl/7.37.1"
},
"tls": {
"version": "TLSv1.2",
"cipher": "ECDHE-RSA-AES256-GCM-SHA384",
"supported_client_ciphers": "ECDHE-RSA-AES256-GCM-SHA384",
"client_verify": "NONE"
}
},
"upstream_uri": "/",
"response": {
"status": 200,
"size": "434",
"headers": {
"Content-Length": "197",
"via": "kong/0.3.0",
"Connection": "close",
"access-control-allow-credentials": "true",
"Content-Type": "application/json",
"server": "nginx",
"access-control-allow-origin": "*"
}
},
"tries": [
{
"state": "next",
"code": 502,
"ip": "127.0.0.1",
"port": 8000
},
{
"ip": "127.0.0.1",
"port": 8000
}
],
"authenticated_entity": {
"consumer_id": "80f74eef-31b8-45d5-c525-ae532297ea8e",
"id": "eaa330c0-4cff-47f5-c79e-b2e4f355207e"
},
"route": {
"created_at": 1521555129,
"hosts": null,
"id": "75818c5f-202d-4b82-a553-6a46e7c9a19e",
"methods": null,
"paths": [
"/example-path"
],
"preserve_host": false,
"protocols": [
"http",
"https"
],
"regex_priority": 0,
"service": {
"id": "0590139e-7481-466c-bcdf-929adcaaf804"
},
"strip_path": true,
"updated_at": 1521555129
},
"service": {
"connect_timeout": 60000,
"created_at": 1521554518,
"host": "example.com",
"id": "0590139e-7481-466c-bcdf-929adcaaf804",
"name": "myservice",
"path": "/",
"port": 80,
"protocol": "http",
"read_timeout": 60000,
"retries": 5,
"updated_at": 1521554518,
"write_timeout": 60000
},
"workspaces": [
{
"id":"b7cac81a-05dc-41f5-b6dc-b87e29b6c3a3",
"name": "default"
}
],
"consumer": {
"username": "demo",
"created_at": 1491847011000,
"id": "35b03bfc-7a5b-4a23-a594-aa350c585fa8"
},
"latencies": {
"proxy": 1430,
"kong": 9,
"request": 1921
},
"client_ip": "127.0.0.1",
"started_at": 1433209822425
}
構建Web服務
自定義Http服務使用Python開發，用於處理HTTPLog轉發的數據，並且在代理服務做了三個事情
- 根據請求ip，使用高德地圖獲取請求使用者的地理位置
- 根據請求頭中的資訊，獲取實際請求使用者的資訊（可根據自己的實際情況來定製）
- 根據設定的請求資訊，使用route_id+method確定具體的功能描述
部署服務
docker run -p 10150:5000 -tid \
--env REDIS_HOST="127.0.0.1" \
--env REDIS_PORT=6379 \
--env REDIS_PASS="password" \
--env REDIS_DB=0 \
--env AMAP_KEY="高德地圖的amapkey" \
--env ELASTICSEARCH_URL="ELASTISEARCH的請求地址" \
--env USER_KEY="我業務中User資訊儲存的快取key" \
--name kong_tools kong_tools
附加資訊後的數據
{
"request": {
"method": "GET",
"uri": "/get",
"url": "http://httpbin.org:8000/get",
"size": "75",
"querystring": {},
"headers": {
"accept": "*/*",
"host": "httpbin.org",
"user-agent": "curl/7.37.1"
"x-real-ip": "113.116.0.162",
},
"tls": {
"version": "TLSv1.2",
"cipher": "ECDHE-RSA-AES256-GCM-SHA384",
"supported_client_ciphers": "ECDHE-RSA-AES256-GCM-SHA384",
"client_verify": "NONE"
}
},
"upstream_uri": "/",
"response": {
"status": 200,
"size": "434",
"headers": {
"Content-Length": "197",
"via": "kong/0.3.0",
"Connection": "close",
"access-control-allow-credentials": "true",
"Content-Type": "application/json",
"server": "nginx",
"access-control-allow-origin": "*"
}
},
"tries": [
{
"state": "next",
"code": 502,
"ip": "127.0.0.1",
"port": 8000
},
{
"ip": "127.0.0.1",
"port": 8000
}
],
"authenticated_entity": {
"consumer_id": "80f74eef-31b8-45d5-c525-ae532297ea8e",
"id": "eaa330c0-4cff-47f5-c79e-b2e4f355207e"
},
"route": {
"created_at": 1521555129,
"hosts": null,
"id": "75818c5f-202d-4b82-a553-6a46e7c9a19e",
"methods": null,
"paths": [
"/example-path"
],
"preserve_host": false,
"protocols": [
"http",
"https"
],
"regex_priority": 0,
"service": {
"id": "0590139e-7481-466c-bcdf-929adcaaf804"
},
"strip_path": true,
"updated_at": 1521555129
},
"service": {
"connect_timeout": 60000,
"created_at": 1521554518,
"host": "example.com",
"id": "0590139e-7481-466c-bcdf-929adcaaf804",
"name": "myservice",
"path": "/",
"port": 80,
"protocol": "http",
"read_timeout": 60000,
"retries": 5,
"updated_at": 1521554518,
"write_timeout": 60000
},
"workspaces": [
{
"id":"b7cac81a-05dc-41f5-b6dc-b87e29b6c3a3",
"name": "default"
}
],
"consumer": {
"username": "demo",
"created_at": 1491847011000,
"id": "35b03bfc-7a5b-4a23-a594-aa350c585fa8"
},
"latencies": {
"proxy": 1430,
"kong": 9,
"request": 1921
},
"client_ip": "127.0.0.1",
"started_at": 1433209822425,
"localtion": { # 附加數據1：根據x-real-ip使用高德獲取具體的地址
"status": "1",
"info": "OK",
"infocode": "10000",
"province": "北京市",
"city": "北京市",
"adcode": "110000",
"rectangle": "116.0119343,39.66127144;116.7829835,40.2164962"
},
"user": { # 附加數據2：使用者資訊，這數據是根據請求頭中的token獲取原生的使用者資訊，可根據實際情況地址中介軟體處理
"user_id": 111,
"nickname": "測試數據",
},
"action": "查詢首頁資訊流" # 附加數據3 根據route_id+method設定具體的功能，然後附加數據處理
}
原始碼地址
關注戰渣渣回覆 回復"kong轉發"
elasticsearch檢視
curl http://localhost:9200/idx_kong_req_log/_search
檢視結果
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 10000,
"relation": "gte"
},
"max_score": 1.0,
"hits": [{
"_index": "idx_kong_req_log",
"_type": "_doc",
"_id": "T8B9nXMByMHbZczOfmM5",
"_score": 1.0,
"_source": {
"latencies": {
"request": 111,
"kong": 0,
"proxy": 111
},
"service": {
"host": "test.example.com",
"created_at": 1550307791,
"connect_timeout": 60000,
"id": "c83db79a-2cc8-4fae-85ce-191008130fbb",
"protocol": "http",
"name": "test",
"read_timeout": 60000,
"port": 80,
"path": null,
"updated_at": 1550307791,
"retries": 5,
"write_timeout": 60000
},
"request": {
"querystring": {},
"size": "738",
"uri": "/test/example/index",
"url": "http://test.example.com:8000/test/example/index",
"headers": {
"host": "test.example.com",
"content-type": "application/json",
"x-real-ip": "111.111.11.11",
"accept": "*/*",
"sec-fetch-mode": "cors",
"user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 wechatdevtools/1.03.2005140 MicroMessenger/7.0.4 Language/zh_CN webview/",
"accept-encoding": "gzip, deflate, br",
"pragma": "no-cache",
"x-access-token": "260eaffa-d145-11ea-8deb-00163e32a9ea",
"x-scheme": "https"
},
"method": "GET"
},
"tries": [{
"balancer_latency": 0,
"port": 30001,
"ip": "171.11.81.111"
}],
"client_ip": "111.111.11.11",
"api": {},
"upstream_uri": "/test/example/index",
"response": {
"headers": {
"content-type": "application/json; charset=UTF-8",
"date": "Thu, 30 Jul 2020 02:11:53 GMT",
"x-kong-proxy-latency": "0",
"content-length": "1316",
"server": "TornadoServer/5.1",
"via": "kong/0.13.1",
"x-kong-upstream-latency": "111",
"etag": "\"069cbf092f18f1f3219def8502b79b6f75bb2d39\"",
"connection": "close"
},
"status": 200,
"size": "1615"
},
"route": {
"created_at": 1591238329,
"strip_path": false,
"hosts": ["test.example.com"],
"preserve_host": false,
"regex_priority": 0,
"updated_at": 1591238329,
"paths": ["/test/example/index"],
"service": {
"id": "c83db79a-2cc8-4fae-85ce-191008130fbb"
},
"methods": ["GET"],
"protocols": ["http", "https"],
"id": "4fe6f2b7-fb16-4ecb-9d9d-47260d9dfe16"
},
"started_at": 1596075113847,
"localtion": {
"status": "1",
"info": "OK",
"infocode": "10000",
"province": "北京市",
"city": "北京市",
"adcode": "110000",
"rectangle": "116.0119343,39.66127144;116.7829835,40.2164962"
},
"user": {
"user_id": 111,
"nickname": "測試",
},
"action": "測試"
}
}]
}
}