import base64 import hashlib import hmac import sys import re import os import urllib import time import requests abspath = os.path.abspath(__file__) root_path = re.sub(r"tendata-warehouse.*", "tendata-warehouse", abspath) sys.path.append(root_path) from dw_base.utils.config_utils import parse_args from dw_base.spark.spark_sql import SparkSQL import json spark = SparkSQL(udf_files=['dw_base/spark/udf/contacts/ctc_common.py', 'dw_base/spark/udf/spark_id_generate_udf.py']) def send_dingtalk_notification(msg): headers = {"Content-Type": "application/json"} data = { "msgtype": "text", "text": {"content": msg} } json_data = json.dumps(data) url = 'https://oapi.dingtalk.com/robot/send?access_token=5183dfe1ecbe06261bcac7b45c1a6b5ae101fec67877d74120a6a95c88d1f917' # url = 'https://oapi.dingtalk.com/robot/send?access_token=c4086d8ba377fdade2dff869e71063733095bc718d3bafdfbe8be0966aa050d6' # url = 'https://oapi.dingtalk.com/robot/send?access_token=bee997dbf61e839a17de087830ffef6e864c3109fef62a956703bdfe043b0e10' response = requests.post(url=url, data=json_data, headers=headers) response.raise_for_status() # shh非核心业务调用数 def get_shh_non_core_interface_cnt(dt): sql = f''' SELECT sum(cnt) cnt FROM (SELECT count(1) cnt FROM ent_raw.interface_base WHERE topic = "ent_monitor_interface" AND dt = "{dt}" AND GET_JSON_OBJECT(ori_json, "$.type") != "EXPORT" AND GET_JSON_OBJECT(ori_json, "$.source")= 'CONTACT' UNION ALL SELECT count(1) cnt FROM ent_raw.interface_base WHERE topic = "ent_shh_bizr_interface" AND dt = "{dt}" AND GET_JSON_OBJECT(ori_json, "$.type") IN("ROOT", "COMPANY_COUNT") AND GET_JSON_OBJECT(ori_json, "$.source")= 'BIZR' UNION ALL SELECT count(1) cnt FROM ent_raw.interface_base WHERE topic = "ent_shh_mecs_interface" AND dt = "{dt}" AND GET_JSON_OBJECT(ori_json, "$.type") IN("CORP", "SITE") AND GET_JSON_OBJECT(ori_json, "$.source")= 'MECS' UNION ALL SELECT count(1) cnt FROM ent_raw.interface_base WHERE topic = "ent_shh_interface" AND dt = "{dt}" AND GET_JSON_OBJECT(ori_json, "$.type")= "BIZR" AND GET_JSON_OBJECT(ori_json, "$.source")= "BIZR" )t ''' return spark.query(sql)[0].collect()[0]['cnt'] def get_shh_company_interface_cnt(dt): sql = f'select count(1) cnt from ent_ods.ent_shh_api_company_logs where dt = "{dt}" and source != "SCRIPT"' return spark.query(sql)[0].collect()[0]['cnt'] def get_shh_company_interface_script_cnt(dt): sql = f'select count(1) cnt from ent_ods.ent_shh_api_company_logs where dt = "{dt}" and source = "SCRIPT"' return spark.query(sql)[0].collect()[0]['cnt'] def get_shh_contact_interface_cnt(dt): sql = f'select count(1) cnt from ent_raw.interface_base where topic = "ctc_shh_interface" and dt = "{dt}" and GET_JSON_OBJECT(ori_json, "$.source") != "SCRIPT"' return spark.query(sql)[0].collect()[0]['cnt'] def get_shh_contact_interface_script_cnt(dt): sql = f'select count(1) cnt from ent_raw.interface_base where topic = "ctc_shh_interface" and dt = "{dt}" and GET_JSON_OBJECT(ori_json, "$.source") = "SCRIPT"' return spark.query(sql)[0].collect()[0]['cnt'] def get_snv_contact_interface_cnt(dt): sql = f'select count(1) cnt from ent_raw.interface_base where topic = "ctc_snovio_interface" and dt = "{dt}" and GET_JSON_OBJECT(ori_json, "$.source") != "MANUAL_CONSUME" ' return spark.query(sql)[0].collect()[0]['cnt'] def get_snv_contact_interface_script_cnt(dt): sql = f'select count(1) cnt from ent_raw.interface_base where topic = "ctc_snovio_interface" and dt = "{dt}" and GET_JSON_OBJECT(ori_json, "$.source") = "MANUAL_CONSUME" ' return spark.query(sql)[0].collect()[0]['cnt'] if __name__ == '__main__': CONFIG, _ = parse_args(sys.argv[1:]) dts = CONFIG.get('dt').split(',') for dt in dts: format_dt = f'{dt[:4]}-{dt[4:6]}-{dt[6:]}' shh_company_interface_cnt = get_shh_company_interface_cnt(dt) shh_company_interface_script_cnt = get_shh_company_interface_script_cnt(dt) shh_contact_interface_cnt = get_shh_contact_interface_cnt(dt) shh_contact_interface_script_cnt = get_shh_contact_interface_script_cnt(dt) snv_contact_interface_cnt = get_snv_contact_interface_cnt(dt) snv_contact_interface_script_cnt = get_snv_contact_interface_script_cnt(dt) shh_non_core_interface_cnt = get_shh_non_core_interface_cnt(dt) msg = f'''【接口调用量统计】------------------------------------------ 统计日期: {format_dt} 1、单接口调用公司信息次数: {shh_company_interface_cnt + shh_company_interface_script_cnt} ①自然调用次数: {shh_company_interface_cnt} ②脚本调用次数: {shh_company_interface_script_cnt} 2、单接口调用联系人次数: {shh_contact_interface_cnt + shh_contact_interface_script_cnt} ①自然调用次数: {shh_contact_interface_cnt} ②脚本调用次数: {shh_contact_interface_script_cnt} 3、snovio调用联系人次数: {snv_contact_interface_cnt + snv_contact_interface_script_cnt} ①自然调用次数: {snv_contact_interface_cnt} ②脚本调用次数: {snv_contact_interface_script_cnt} 4、单接口非核心业务调用次数:{shh_non_core_interface_cnt} ---------------------------------------------------------------''' print(msg) send_dingtalk_notification(msg)